diff --git a/src/qsched.c b/src/qsched.c
index 16ffe81b1065c82c300df06a4a376b933f8c7af6..4bf2173ffb8b9ead1c88d7ef5db45b3d7bd6c241 100644
--- a/src/qsched.c
+++ b/src/qsched.c
@@ -855,12 +855,12 @@ void qsched_partition_compute_costs( struct qsched *s, idx_t *res_costs)
t = &s->tasks[i];
for(j = 0; j < t->nr_locks; j++)
{
- res_costs[getindex(t->locks[j],s)] = t->cost;
+ res_costs[getindex(t->locks[j],s)] += t->cost;
}
for(j = 0; j < t->nr_uses; j++)
{
- res_costs[getindex(t->uses[j],s)] = t->cost;
+ res_costs[getindex(t->uses[j],s)] += t->cost;
}
}
}
@@ -912,14 +912,14 @@ void qsched_partition_build_nodelist(struct qsched *s, idx_t *nodelist, idx_t *n
{
noderef[node_count] = i;
pos_in_nodelist[i] = node_count;
- nodelist[node_count++] = res_costs[i];
+ nodelist[node_count++] += res_costs[i];
}
}
}else{
//Else set noderef to the index i, and set the cost in the nodelist to the res_cost value.
noderef[node_count] = i;
pos_in_nodelist[i] = node_count;
- nodelist[node_count++] = res_costs[i];
+ nodelist[node_count++] += res_costs[i];
}
}else{
@@ -935,6 +935,208 @@ void qsched_partition_build_nodelist(struct qsched *s, idx_t *nodelist, idx_t *n
*node_count_input = node_count;
}
+void qsched_partition_build_edgelist(struct qsched *s, idx_t **edge_vwgts, idx_t **edge_lists, idx_t *edge_counts, idx_t *edge_sizes,
+ idx_t node_count, idx_t *noderef, idx_t *pos_in_nodelist )
+{
+ int i, j, k, l;
+ struct task *t;
+ long long int *parents = malloc(sizeof(long long int) * s->res_ranks[s->count_ranks]);
+ if(parents == NULL)
+ error("Failed to allocate parents array");
+ for(i = 0; i < s->res_ranks[s->count_ranks]; i++)
+ parents[i] = -1;
+
+ for(i = 0; i < s->task_ranks[s->count_ranks]; i++)
+ {
+ t = &s->tasks[i];
+ /* Loop over all lock pairs.*/
+ for(j = 0; j < t->nr_locks; j++)
+ {
+ struct res *r = &s->res[getindex(t->locks[j],s)];
+ if(parents[getindex(t->locks[j], s)] < 0)
+ {
+ /* If not lets compute it. */
+ struct res *r2 = r;
+ struct res *r3 = r;
+ /* We can probably optimize this by checking if our parents "parents" value is set.*/
+ while(r2->parent != -1){
+ r2 = getres(r2->parent, s);
+ if(r2->num_lockers > 0 || r2->num_users > 0)
+ r3 = r2;
+ }
+ parents[getindex(t->locks[j], s)] = r3->ID;
+ }/*if parents not set*/
+
+ for(k = j+1; k < t->nr_locks; k++)
+ {
+ struct res *r4 = &s->res[getindex(t->locks[k], s)];
+ if(parents[getindex(t->locks[k], s)] < 0)
+ {
+ struct res *r2 = r4;
+ struct res *r3 = r4;
+ while(r2->parent != -1){
+ r2 = getres(r2->parent, s);
+ if(r2->num_lockers > 0 || r2->num_users > 0)
+ r3 = r2;
+ }
+ parents[getindex(t->locks[k], s)] = r3->ID;
+ }
+ /* If the resources aren't "the same" at the highest level.*/
+ if(parents[getindex(t->locks[k], s)] != parents[getindex(t->locks[j], s)])
+ {
+ int index1 = getindex(parents[r->ID], s);
+ int index2 = getindex(parents[getindex(r4->ID, s)], s);
+ int nodepos1 = pos_in_nodelist[index1], nodepos2 = pos_in_nodelist[index2];
+ /* Check if the edge already exists.*/
+ int exists = -1;
+ for(l = 0; l < edge_counts[nodepos1]; l++)
+ {
+ if(edge_lists[nodepos1][l] == nodepos2)
+ {
+ exists = l;
+ break;
+ }
+ }/* l */
+ /* If it exists*/
+ if(exists >=0 )
+ {
+ edge_vwgts[nodepos1][l] += t->cost;
+ for(l = 0; l < edge_counts[nodepos2]; l++)
+ {
+ if(edge_lists[nodepos2][l] == nodepos1)
+ {
+ edge_vwgts[nodepos2][l] += t->cost;
+ break;
+ }
+ }/* l*/
+ }else{
+ /* Check if we need to expand edge lists. */
+ if(edge_counts[nodepos1] == edge_sizes[nodepos1] -1)
+ {
+ edge_sizes[nodepos1] *= 2;
+ idx_t *temp = malloc(sizeof(idx_t) * edge_sizes[nodepos1]);
+ idx_t *temp2 = malloc(sizeof(idx_t) * edge_sizes[nodepos1]);
+ memcpy(temp, edge_lists[nodepos1], sizeof(idx_t) * edge_counts[nodepos1]);
+ memcpy(temp2, edge_vwgts[nodepos1], sizeof(idx_t) * edge_counts[nodepos1]);
+ free(edge_lists[nodepos1]);
+ free(edge_vwgts[nodepos1]);
+ edge_lists[nodepos1] = temp;
+ edge_vwgts[nodepos1] = temp2;
+ }
+
+ if(edge_counts[nodepos2] == edge_sizes[nodepos2] -1)
+ {
+ edge_sizes[nodepos2] *= 2;
+ idx_t *temp = malloc(sizeof(idx_t) * edge_sizes[nodepos2]);
+ idx_t *temp2 = malloc(sizeof(idx_t) * edge_sizes[nodepos2]);
+ memcpy(temp, edge_lists[nodepos2], sizeof(idx_t) * edge_counts[nodepos2]);
+ memcpy(temp2, edge_vwgts[nodepos2], sizeof(idx_t) * edge_counts[nodepos2]);
+ free(edge_lists[nodepos2]);
+ free(edge_vwgts[nodepos2]);
+ edge_lists[nodepos2] = temp;
+ edge_vwgts[nodepos2] = temp2;
+ }
+
+ /* Add the edge and the weights. */
+ edge_lists[nodepos1][edge_counts[nodepos1]] = nodepos2;
+ edge_vwgts[nodepos1][edge_counts[nodepos1]] = t->cost;
+ edge_counts[nodepos1] += 1;
+ edge_lists[nodepos2][edge_counts[nodepos2]] = nodepos1;
+ edge_vwgts[nodepos2][edge_counts[nodepos2]] = t->cost;
+ edge_counts[nodepos2] += 1;
+
+ }/* If edge doesn't exist. */
+ }
+ }/* k */
+//#ifdef NO_COMPILE
+ for(k = 0; k < t->nr_uses; k++)
+ {
+ struct res *r4 = &s->res[getindex(t->uses[k], s)];
+ if(parents[getindex(t->uses[k], s)] < 0)
+ {
+ struct res *r2 = r4;
+ struct res *r3 = r4;
+ while(r2->parent != -1){
+ r2 = getres(r2->parent, s);
+ if(r2->num_lockers > 0 || r2->num_users > 0)
+ r3 = r2;
+ }
+ parents[getindex(t->uses[k], s)] = r3->ID;
+ }
+ /* If the resources aren't "the same" at the highest level.*/
+ if(parents[getindex(t->uses[k], s)] != parents[getindex(t->locks[j], s)])
+ {
+ int index1 = getindex(parents[r->ID], s);
+ int index2 = getindex(parents[getindex(r4->ID, s)], s);
+ int nodepos1 = pos_in_nodelist[index1], nodepos2 = pos_in_nodelist[index2];
+
+ /* Check if the edge already exists.*/
+ int exists = -1;
+ for(l = 0; l < edge_counts[nodepos1]; l++)
+ {
+ if(edge_lists[nodepos1][l] == nodepos2)
+ {
+ exists = l;
+ break;
+ }
+ }/* l */
+ /* If it exists*/
+ if(exists >=0 )
+ {
+ edge_vwgts[nodepos1][l] += t->cost;
+ for(l = 0; l < edge_counts[nodepos2]; l++)
+ {
+ if(edge_lists[nodepos2][l] == nodepos1)
+ {
+ edge_vwgts[nodepos2][l] += t->cost;
+ break;
+ }
+ }/* l*/
+ }else{
+ /* Check if we need to expand edge lists. */
+ if(edge_counts[nodepos1] == edge_sizes[nodepos1] -1)
+ {
+ edge_sizes[nodepos1] *= 2;
+ idx_t *temp = malloc(sizeof(idx_t) * edge_sizes[nodepos1]);
+ idx_t *temp2 = malloc(sizeof(idx_t) * edge_sizes[nodepos1]);
+ memcpy(temp, edge_lists[nodepos1], sizeof(idx_t) * edge_counts[nodepos1]);
+ memcpy(temp2, edge_vwgts[nodepos1], sizeof(idx_t) * edge_counts[nodepos1]);
+ free(edge_lists[nodepos1]);
+ free(edge_vwgts[nodepos1]);
+ edge_lists[nodepos1] = temp;
+ edge_vwgts[nodepos1] = temp2;
+ }
+
+ if(edge_counts[nodepos2] == edge_sizes[nodepos2] -1)
+ {
+ edge_sizes[nodepos2] *= 2;
+ idx_t *temp = malloc(sizeof(idx_t) * edge_sizes[nodepos2]);
+ idx_t *temp2 = malloc(sizeof(idx_t) * edge_sizes[nodepos2]);
+ memcpy(temp, edge_lists[nodepos2], sizeof(idx_t) * edge_counts[nodepos2]);
+ memcpy(temp2, edge_vwgts[nodepos2], sizeof(idx_t) * edge_counts[nodepos2]);
+ free(edge_lists[nodepos2]);
+ free(edge_vwgts[nodepos2]);
+ edge_lists[nodepos2] = temp;
+ edge_vwgts[nodepos2] = temp2;
+ }
+
+ /* Add the edge and the weights. */
+ edge_lists[nodepos1][edge_counts[nodepos1]] = nodepos2;
+ edge_vwgts[nodepos1][edge_counts[nodepos1]] = t->cost;
+ edge_counts[nodepos1] += 1;
+ edge_lists[nodepos2][edge_counts[nodepos2]] = nodepos1;
+ edge_vwgts[nodepos2][edge_counts[nodepos2]] = t->cost;
+ edge_counts[nodepos2] += 1;
+
+ }/* If edge doesn't exist. */
+ }
+ }/* k */
+//#endif
+ }/* j */
+ }/* i */
+}
+
+#ifdef NO_COMPILE
void qsched_partition_build_edgelist(struct qsched *s, idx_t *edgelist, idx_t node_count, idx_t *noderef, idx_t *pos_in_nodelist){
int i, j, k;
struct res *r;
@@ -1145,230 +1347,6 @@ free(parents);
}
-#ifdef NO_COMPILE
-void qsched_partition_build_edgelist(struct qsched *s, idx_t *edgelist, idx_t node_count, idx_t *noderef){
-
- int i, j, k, l;
- struct res *r;
- struct task *t;
-
- /* Loop through local tasks */
- for(i = s->task_ranks[s->rank]; i < s->task_ranks[s->rank+1]; i++)
- {
- t = &s->tasks[i];
- for(j = 0; j < t->nr_locks; j++)
- {
- /* Pull the resource. */
- r = getres(t->locks[j], s);
- if(r->num_lockers == 0)
- error("r with ID %lli has no lockers but we chose it.", r->ID)
- struct res *r2 = r;
- /* Find highest parent of r which is locked.*/
- while(r2->parent != -1)
- {
- r2 = getres(r2->parent, s);
- if(r2->num_lockers > 0)
- r = r2;
- }
- /* Loop over the other resources*/
- for(k = 0; k < t->nr_locks; k++)
- {
- if(j == k)
- continue;
-
- /* Find the highest parent of r which is locked or used. */
- r2 = getres(t->locks[k], s);
- struct res *r3 = r2;
- while(r3->parent != -1)
- {
- r3 = getres(r3->parent, s);
- if(r3->num_lockers > 0 || r3->num_users > 0)
- r2 = r3;
- }
-
- if(r == r2)
- {
- continue;
- }
- int index1 = r - s->res;
- int index2 = r2 - s->res;
-
- int nodepos1 = -1, nodepos2 = -1;
-
- /* Find the nodes in the graph which they refer to.*/
- for(l = 0; l < node_count; l++)
- {
- if(noderef[l] == index1)
- nodepos1 = l;
-
- if(noderef[l] == index2)
- nodepos2 = l;
-
- }
- if(nodepos1 < 0 || nodepos2 < 0){
- error("ERROR NODEPOS NOT VALID, nodepos1 = %i, nodepos2= %i\n", nodepos1, nodepos2);
- }
- if(nodepos1 < nodepos2)
- {
-//[nodepos1][nodepos2] is (node_count * nodepos1) + nodepos2 - ( ( nodepos1 * (nodepos+1) / 2) from https://jamesmccaffrey.wordpress.com/2010/05/14/converting-a-triangular-matrix-to-an-array/
-
- edgelist[ (node_count * nodepos1) + nodepos2 - ( nodepos1 * (nodepos1+1) / 2)] += t->cost;
- }else{
- edgelist[ (node_count * nodepos2) + nodepos1 - ( nodepos2 * (nodepos2 + 1) / 2)] += t->cost;
- }
-
- }
-
- for(k = 0; k < t->nr_uses; k++)
- {
- r2 = getres(t->uses[k], s);
- if(r2->num_users == 0)
- error("ERROR, r2 has no users but t uses it\n");
- if(r->num_lockers == 0)
- error("ERROR, r has no lockers but t locks it\n");
- struct res *r3 = r2;
- while(r3->parent != -1)
- {
- r3 = getres(r3->parent, s);
- if(r3->num_users > 0 || r3->num_lockers > 0)
- r2 = r3;
- }
- if(r == r2)
- {
- continue;
- }
- int index1 = r - s->res;
- int index2 = r2 - s->res;
-
- int nodepos1 = -1, nodepos2 = -1;
-
- for(l = 0; l < node_count; l++)
- {
- if(noderef[l] == index1)
- nodepos1 = l;
-
- if(noderef[l] == index2)
- nodepos2 = l;
-
- }
- if(nodepos1 < 0 || nodepos2 < 0){
- error("ERROR NODEPOS NOT VALID, nodepos1 = %i, index1 = %i, nodepos2= %i, index2 = %i\n", nodepos1, index1, nodepos2, index2);
- }
- if(nodepos1 < nodepos2)
- {
-//[nodepos1][nodepos2] is (node_count * nodepos1) + nodepos2 - ( ( nodepos1 * (nodepos+1) / 2) from https://jamesmccaffrey.wordpress.com/2010/05/14/converting-a-triangular-matrix-to-an-array/
-
- edgelist[ (node_count * nodepos1) + nodepos2 - ( nodepos1 * (nodepos1+1) / 2)] += t->cost;
- }else{
- edgelist[ (node_count * nodepos2) + nodepos1 - ( nodepos2 * (nodepos2 + 1) / 2)] += t->cost;
- }
- }
- }//Locks
-
- for(j = 0; j < t->nr_uses; j++)
- {
- r = getres(t->uses[j], s);
- struct res *r2 = r;
- while(r2->parent != -1)
- {
- r2 = getres(r2->parent, s);
- if(r2->num_users > 0)
- r = r2;
- }
- for(k = 0; k < t->nr_locks; k++)
- {
-
- r2 = getres(t->locks[k], s);
- struct res *r3 = r2;
- while(r3->parent != -1)
- {
- r3 = getres(r3->parent, s);
- if(r3->num_lockers > 0)
- r2 = r3;
- }
-
- if(r == r2)
- {
- continue;
- }
- int index1 = r - s->res;
- int index2 = r2 - s->res;
-
- int nodepos1 = -1, nodepos2 = -1;
-
- for(l = 0; l < node_count; l++)
- {
- if(noderef[l] == index1)
- nodepos1 = l;
-
- if(noderef[l] == index2)
- nodepos2 = l;
-
- }
- if(nodepos1 < 0 || nodepos2 < 0){
- error("ERROR NODEPOS NOT VALID\n");
- }
- if(nodepos1 < nodepos2)
- {
-//[nodepos1][nodepos2] is (node_count * nodepos1) + nodepos2 - ( ( nodepos1 * (nodepos+1) / 2) from https://jamesmccaffrey.wordpress.com/2010/05/14/converting-a-triangular-matrix-to-an-array/
-
- edgelist[ (node_count * nodepos1) + nodepos2 - ( nodepos1 * (nodepos1+1) / 2)] += t->cost;
- }else{
- edgelist[ (node_count * nodepos2) + nodepos1 - ( nodepos2 * (nodepos2 + 1) / 2)] += t->cost;
- }
-
- }
-
- for(k = 0; k < t->nr_uses; k++)
- {
- if(j == k)
- continue;
- r2 = getres(t->uses[k], s);
- struct res *r3 = r2;
- while(r3->parent != -1)
- {
- r3 = getres(r3->parent, s);
- if(r3->num_users > 0)
- r2 = r3;
- }
- if(r == r2)
- {
- continue;
- }
-
- int index1 = r - s->res;
- int index2 = r2 - s->res;
-
- int nodepos1 = -1, nodepos2 = -1;
-
- for(l = 0; l < node_count; l++)
- {
- if(noderef[l] == index1)
- nodepos1 = l;
-
- if(noderef[l] == index2)
- nodepos2 = l;
-
- }
- if(nodepos1 < 0 || nodepos2 < 0){
- error("ERROR NODEPOS NOT VALID\n");
- MPI_Finalize();
- exit(0);
- }
- if(nodepos1 < nodepos2)
- {
-//[nodepos1][nodepos2] is (node_count * nodepos1) + nodepos2 - ( ( nodepos1 * (nodepos+1) / 2) from https://jamesmccaffrey.wordpress.com/2010/05/14/converting-a-triangular-matrix-to-an-array/
-
- edgelist[ (node_count * nodepos1) + nodepos2 - ( nodepos1 * (nodepos1+1) / 2)] += t->cost;
- }else{
- edgelist[ (node_count * nodepos2) + nodepos1 - ( nodepos2 * (nodepos2 + 1) / 2)] += t->cost;
- }
- }
- }//Uses
- }
-}
-
-#endif
void qsched_partition_edgelist_squash( struct qsched *s, idx_t *edgelist_pos, idx_t *edgelist_new, idx_t *edgelist_vwgt, idx_t *edgelist_count_input, idx_t *edgelist, int node_count){
@@ -1395,6 +1373,7 @@ void qsched_partition_edgelist_squash( struct qsched *s, idx_t *edgelist_pos, id
}
*edgelist_count_input = edgelist_count;
}
+#endif
void qsched_partition_topsort( struct qsched *s, long long int *tid)
{
@@ -2847,8 +2826,67 @@ void qsched_partition( struct qsched *s){
toc = getticks();
message("qsched_partition_build_nodelist took %lli (= %e) ticks\n", toc-tic, (float)(toc-tic));
-//Build an edgelist where edges are of weight += task->weight for each task that locks (and eventually uses) both. If noderef doesn't contain, recurse until we find the ones it does contain (yuck). Create an initial "complete" graph then delete unused edges.
+//Build an edgelist where edges are of weight += task->weight for each task that locks both. If noderef doesn't contain, recurse until we find the ones it does contain (yuck). Build a set of adjacency lists.
+
+idx_t **edge_vwgts = malloc(sizeof(idx_t *) * node_count);
+ if(edge_vwgts == NULL)
+ error("Failed to allocate edge_vwgts");
+idx_t **edge_lists = malloc(sizeof(idx_t *) * node_count);
+ if(edge_lists == NULL)
+ error("Failed to allocate edge_lists");
+idx_t *edge_counts = calloc(node_count, sizeof(idx_t));
+ if(edge_counts == NULL)
+ error("Failed to allocate edge_counts");
+idx_t *edge_sizes = calloc(node_count, sizeof(idx_t));
+ if(edge_sizes == NULL)
+ error("Failed to allocate edge_sizes");
+
+#define initial_size 100
+for(i = 0; i < node_count; i++)
+{
+ edge_vwgts[i] = calloc(initial_size, sizeof(idx_t));
+ if(edge_vwgts[i] == NULL)
+ error("Failed to allocate edge_vwgts[%i]", i);
+ edge_lists[i] = calloc(initial_size, sizeof(idx_t));
+ if(edge_lists[i] == NULL)
+ error("Failed to allocate edge_vwgts[%i]", i);
+ edge_sizes[i] = initial_size;
+}
+
+ tic = getticks();
+ qsched_partition_build_edgelist(s, edge_vwgts, edge_lists, edge_counts, edge_sizes, node_count, noderef, pos_in_nodelist);
+ toc = getticks();
+ message("qsched_partition_build_edgelist took %lli (= %e) ticks\n", toc-tic, (float)(toc-tic));
+
+idx_t edgelist_size = 0;
+for(i = 0; i < node_count; i++)
+ edgelist_size += edge_counts[i];
+
+printf("edgelist_size = %i\n", edgelist_size);
+printf("node_count = %i\n", node_count);
+/* Convert the edgelists to METIS version.*/
+ idx_t *edgelist_new = malloc(sizeof(idx_t) * edgelist_size);
+ if(edgelist_new == NULL)
+ error("Failed to allocate edgelist_new");
+ idx_t *edgelist_pos = malloc(sizeof(idx_t) * (node_count + 1));
+ if(edgelist_pos == NULL)
+ error("Failed to allocate edgelist_pos");
+ edgelist_pos[0] = 0;
+ idx_t *edgelist_vwgt = malloc(sizeof(idx_t) * edgelist_size);
+ if(edgelist_vwgt == NULL)
+ error("Failed to allocate edgelist_vwgt");
+ // idx_t edgelist_count = 0;
+
+edgelist_pos[0] = 0;
+for(i = 0; i < node_count; i++)
+{
+ memcpy(&edgelist_new[edgelist_pos[i]], edge_lists[i], sizeof(idx_t) * edge_counts[i] );
+ memcpy(&edgelist_vwgt[edgelist_pos[i]], edge_vwgts[i], sizeof(idx_t) * edge_counts[i] );
+ edgelist_pos[i+1] = edgelist_pos[i] + edge_counts[i];
+}
+
+#ifdef NO_COMPILE
idx_t *edgelist;
edgelist = (idx_t *) calloc( (node_count * (node_count+1))/2, sizeof(idx_t));
@@ -2857,6 +2895,7 @@ void qsched_partition( struct qsched *s){
toc = getticks();
message("qsched_partition_build_edgelist took %lli (= %e) ticks\n", toc-tic, (float)(toc-tic));
+#endif
free(pos_in_nodelist);
//This might work but for safety I've made all nodes compute the edgelist anyway.
@@ -2876,6 +2915,7 @@ void qsched_partition( struct qsched *s){
}*/
//Loop through edgelist and remove empty slots. Count how big it needs to be.
+#ifdef NO_COMPILE
int edgelist_size = 0;
tic = getticks();
for(i = 0; i < ((node_count * (node_count+1))/2); i++)
@@ -2888,8 +2928,8 @@ void qsched_partition( struct qsched *s){
//Make the new contiguous edge list.
- message("Size of new edgelist = %i, ((node_count * (node_count+1))/2) = %i", edgelist_size*2, ((node_count * (node_count+1))/2));
- message("Node_count = %i", node_count);
+ // message("Size of new edgelist = %i, ((node_count * (node_count+1))/2) = %i", edgelist_size*2, ((node_count * (node_count+1))/2));
+// message("Node_count = %i", node_count);
//printf("Size of new edgelist = %i * sizeof(idx_t)\n", edgelist_size*2);
idx_t *edgelist_new = malloc(sizeof(idx_t) * edgelist_size*2);
if(edgelist_new == NULL)
@@ -2907,6 +2947,7 @@ void qsched_partition( struct qsched *s){
qsched_partition_edgelist_squash(s, edgelist_pos, edgelist_new, edgelist_vwgt, &edgelist_count, edgelist, node_count);
toc = getticks();
message("qsched_partition_edgelist_squash took %lli (= %e) ticks\n", toc-tic, (float)(toc-tic));
+#endif
/*#if IDXTYPEWIDTH == 32
printf("edgelist_pos[node_count+1] = %i\n", edgelist_pos[node_count]);
@@ -2946,9 +2987,9 @@ void qsched_partition( struct qsched *s){
for(i = 0; i < node_count; i++)
{
if(nodeIDs[i] == s->rank)
- count_me+= edgelist_vwgt[i];
+ count_me+= nodelist[i];
}
-
+ message("My \"cost\" = %i", count_me);
tic = getticks();
MPI_Request *reqs;