Bug Summary

File:tools/polly/lib/External/isl/isl_scheduler.c
Warning:line 3063, column 2
Value stored to 'nrow' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name isl_scheduler.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-eagerly-assume -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-7/lib/clang/7.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/tools/polly/lib/External -I /build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External -I /build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/pet/include -I /build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/ppcg/include -I /build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/ppcg/imath -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/tools/polly/lib/External/ppcg -I /build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl -I /build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/include -I /build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/imath -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/tools/polly/lib/External/isl -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/tools/polly/include -I /usr/include/jsoncpp -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/tools/polly/lib/External/isl/include -I /build/llvm-toolchain-snapshot-7~svn338205/tools/polly/include -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/include -I /build/llvm-toolchain-snapshot-7~svn338205/include -U NDEBUG -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-7/lib/clang/7.0.0/include -internal-externc-isystem /usr/lib/gcc/x86_64-linux-gnu/8/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-comment -std=gnu99 -fconst-strings -fdebug-compilation-dir /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/tools/polly/lib/External -fdebug-prefix-map=/build/llvm-toolchain-snapshot-7~svn338205=. -ferror-limit 19 -fmessage-length 0 -stack-protector 2 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-07-29-043837-17923-1 -x c /build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c -faddrsig
1/*
2 * Copyright 2011 INRIA Saclay
3 * Copyright 2012-2014 Ecole Normale Superieure
4 * Copyright 2015-2016 Sven Verdoolaege
5 * Copyright 2016 INRIA Paris
6 * Copyright 2017 Sven Verdoolaege
7 *
8 * Use of this software is governed by the MIT license
9 *
10 * Written by Sven Verdoolaege, INRIA Saclay - Ile-de-France,
11 * Parc Club Orsay Universite, ZAC des vignes, 4 rue Jacques Monod,
12 * 91893 Orsay, France
13 * and Ecole Normale Superieure, 45 rue d'Ulm, 75230 Paris, France
14 * and Centre de Recherche Inria de Paris, 2 rue Simone Iff - Voie DQ12,
15 * CS 42112, 75589 Paris Cedex 12, France
16 */
17
18#include <isl_ctx_private.h>
19#include <isl_map_private.h>
20#include <isl_space_private.h>
21#include <isl_aff_private.h>
22#include <isl/hash.h>
23#include <isl/id.h>
24#include <isl/constraint.h>
25#include <isl/schedule.h>
26#include <isl_schedule_constraints.h>
27#include <isl/schedule_node.h>
28#include <isl_mat_private.h>
29#include <isl_vec_private.h>
30#include <isl/set.h>
31#include <isl_union_set_private.h>
32#include <isl_seq.h>
33#include <isl_tab.h>
34#include <isl_dim_map.h>
35#include <isl/map_to_basic_set.h>
36#include <isl_sort.h>
37#include <isl_options_private.h>
38#include <isl_tarjan.h>
39#include <isl_morph.h>
40#include <isl/ilp.h>
41#include <isl_val_private.h>
42
43/*
44 * The scheduling algorithm implemented in this file was inspired by
45 * Bondhugula et al., "Automatic Transformations for Communication-Minimized
46 * Parallelization and Locality Optimization in the Polyhedral Model".
47 *
48 * For a detailed description of the variant implemented in isl,
49 * see Verdoolaege and Janssens, "Scheduling for PPCG" (2017).
50 */
51
52
53/* Internal information about a node that is used during the construction
54 * of a schedule.
55 * space represents the original space in which the domain lives;
56 * that is, the space is not affected by compression
57 * sched is a matrix representation of the schedule being constructed
58 * for this node; if compressed is set, then this schedule is
59 * defined over the compressed domain space
60 * sched_map is an isl_map representation of the same (partial) schedule
61 * sched_map may be NULL; if compressed is set, then this map
62 * is defined over the uncompressed domain space
63 * rank is the number of linearly independent rows in the linear part
64 * of sched
65 * the rows of "vmap" represent a change of basis for the node
66 * variables; the first rank rows span the linear part of
67 * the schedule rows; the remaining rows are linearly independent
68 * the rows of "indep" represent linear combinations of the schedule
69 * coefficients that are non-zero when the schedule coefficients are
70 * linearly independent of previously computed schedule rows.
71 * start is the first variable in the LP problem in the sequences that
72 * represents the schedule coefficients of this node
73 * nvar is the dimension of the (compressed) domain
74 * nparam is the number of parameters or 0 if we are not constructing
75 * a parametric schedule
76 *
77 * If compressed is set, then hull represents the constraints
78 * that were used to derive the compression, while compress and
79 * decompress map the original space to the compressed space and
80 * vice versa.
81 *
82 * scc is the index of SCC (or WCC) this node belongs to
83 *
84 * "cluster" is only used inside extract_clusters and identifies
85 * the cluster of SCCs that the node belongs to.
86 *
87 * coincident contains a boolean for each of the rows of the schedule,
88 * indicating whether the corresponding scheduling dimension satisfies
89 * the coincidence constraints in the sense that the corresponding
90 * dependence distances are zero.
91 *
92 * If the schedule_treat_coalescing option is set, then
93 * "sizes" contains the sizes of the (compressed) instance set
94 * in each direction. If there is no fixed size in a given direction,
95 * then the corresponding size value is set to infinity.
96 * If the schedule_treat_coalescing option or the schedule_max_coefficient
97 * option is set, then "max" contains the maximal values for
98 * schedule coefficients of the (compressed) variables. If no bound
99 * needs to be imposed on a particular variable, then the corresponding
100 * value is negative.
101 * If not NULL, then "bounds" contains a non-parametric set
102 * in the compressed space that is bounded by the size in each direction.
103 */
104struct isl_sched_node {
105 isl_space *space;
106 int compressed;
107 isl_setisl_map *hull;
108 isl_multi_aff *compress;
109 isl_multi_aff *decompress;
110 isl_mat *sched;
111 isl_map *sched_map;
112 int rank;
113 isl_mat *indep;
114 isl_mat *vmap;
115 int start;
116 int nvar;
117 int nparam;
118
119 int scc;
120 int cluster;
121
122 int *coincident;
123
124 isl_multi_val *sizes;
125 isl_basic_setisl_basic_map *bounds;
126 isl_vec *max;
127};
128
129static int node_has_tuples(const void *entry, const void *val)
130{
131 struct isl_sched_node *node = (struct isl_sched_node *)entry;
132 isl_space *space = (isl_space *) val;
133
134 return isl_space_has_equal_tuples(node->space, space);
135}
136
137static int node_scc_exactly(struct isl_sched_node *node, int scc)
138{
139 return node->scc == scc;
140}
141
142static int node_scc_at_most(struct isl_sched_node *node, int scc)
143{
144 return node->scc <= scc;
145}
146
147static int node_scc_at_least(struct isl_sched_node *node, int scc)
148{
149 return node->scc >= scc;
150}
151
152/* An edge in the dependence graph. An edge may be used to
153 * ensure validity of the generated schedule, to minimize the dependence
154 * distance or both
155 *
156 * map is the dependence relation, with i -> j in the map if j depends on i
157 * tagged_condition and tagged_validity contain the union of all tagged
158 * condition or conditional validity dependence relations that
159 * specialize the dependence relation "map"; that is,
160 * if (i -> a) -> (j -> b) is an element of "tagged_condition"
161 * or "tagged_validity", then i -> j is an element of "map".
162 * If these fields are NULL, then they represent the empty relation.
163 * src is the source node
164 * dst is the sink node
165 *
166 * types is a bit vector containing the types of this edge.
167 * validity is set if the edge is used to ensure correctness
168 * coincidence is used to enforce zero dependence distances
169 * proximity is set if the edge is used to minimize dependence distances
170 * condition is set if the edge represents a condition
171 * for a conditional validity schedule constraint
172 * local can only be set for condition edges and indicates that
173 * the dependence distance over the edge should be zero
174 * conditional_validity is set if the edge is used to conditionally
175 * ensure correctness
176 *
177 * For validity edges, start and end mark the sequence of inequality
178 * constraints in the LP problem that encode the validity constraint
179 * corresponding to this edge.
180 *
181 * During clustering, an edge may be marked "no_merge" if it should
182 * not be used to merge clusters.
183 * The weight is also only used during clustering and it is
184 * an indication of how many schedule dimensions on either side
185 * of the schedule constraints can be aligned.
186 * If the weight is negative, then this means that this edge was postponed
187 * by has_bounded_distances or any_no_merge. The original weight can
188 * be retrieved by adding 1 + graph->max_weight, with "graph"
189 * the graph containing this edge.
190 */
191struct isl_sched_edge {
192 isl_map *map;
193 isl_union_map *tagged_condition;
194 isl_union_map *tagged_validity;
195
196 struct isl_sched_node *src;
197 struct isl_sched_node *dst;
198
199 unsigned types;
200
201 int start;
202 int end;
203
204 int no_merge;
205 int weight;
206};
207
208/* Is "edge" marked as being of type "type"?
209 */
210static int is_type(struct isl_sched_edge *edge, enum isl_edge_type type)
211{
212 return ISL_FL_ISSET(edge->types, 1 << type)(!!((edge->types) & (1 << type)));
213}
214
215/* Mark "edge" as being of type "type".
216 */
217static void set_type(struct isl_sched_edge *edge, enum isl_edge_type type)
218{
219 ISL_FL_SET(edge->types, 1 << type)((edge->types) |= (1 << type));
220}
221
222/* No longer mark "edge" as being of type "type"?
223 */
224static void clear_type(struct isl_sched_edge *edge, enum isl_edge_type type)
225{
226 ISL_FL_CLR(edge->types, 1 << type)((edge->types) &= ~(1 << type));
227}
228
229/* Is "edge" marked as a validity edge?
230 */
231static int is_validity(struct isl_sched_edge *edge)
232{
233 return is_type(edge, isl_edge_validity);
234}
235
236/* Mark "edge" as a validity edge.
237 */
238static void set_validity(struct isl_sched_edge *edge)
239{
240 set_type(edge, isl_edge_validity);
241}
242
243/* Is "edge" marked as a proximity edge?
244 */
245static int is_proximity(struct isl_sched_edge *edge)
246{
247 return is_type(edge, isl_edge_proximity);
248}
249
250/* Is "edge" marked as a local edge?
251 */
252static int is_local(struct isl_sched_edge *edge)
253{
254 return is_type(edge, isl_edge_local);
255}
256
257/* Mark "edge" as a local edge.
258 */
259static void set_local(struct isl_sched_edge *edge)
260{
261 set_type(edge, isl_edge_local);
262}
263
264/* No longer mark "edge" as a local edge.
265 */
266static void clear_local(struct isl_sched_edge *edge)
267{
268 clear_type(edge, isl_edge_local);
269}
270
271/* Is "edge" marked as a coincidence edge?
272 */
273static int is_coincidence(struct isl_sched_edge *edge)
274{
275 return is_type(edge, isl_edge_coincidence);
276}
277
278/* Is "edge" marked as a condition edge?
279 */
280static int is_condition(struct isl_sched_edge *edge)
281{
282 return is_type(edge, isl_edge_condition);
283}
284
285/* Is "edge" marked as a conditional validity edge?
286 */
287static int is_conditional_validity(struct isl_sched_edge *edge)
288{
289 return is_type(edge, isl_edge_conditional_validity);
290}
291
292/* Is "edge" of a type that can appear multiple times between
293 * the same pair of nodes?
294 *
295 * Condition edges and conditional validity edges may have tagged
296 * dependence relations, in which case an edge is added for each
297 * pair of tags.
298 */
299static int is_multi_edge_type(struct isl_sched_edge *edge)
300{
301 return is_condition(edge) || is_conditional_validity(edge);
302}
303
304/* Internal information about the dependence graph used during
305 * the construction of the schedule.
306 *
307 * intra_hmap is a cache, mapping dependence relations to their dual,
308 * for dependences from a node to itself, possibly without
309 * coefficients for the parameters
310 * intra_hmap_param is a cache, mapping dependence relations to their dual,
311 * for dependences from a node to itself, including coefficients
312 * for the parameters
313 * inter_hmap is a cache, mapping dependence relations to their dual,
314 * for dependences between distinct nodes
315 * if compression is involved then the key for these maps
316 * is the original, uncompressed dependence relation, while
317 * the value is the dual of the compressed dependence relation.
318 *
319 * n is the number of nodes
320 * node is the list of nodes
321 * maxvar is the maximal number of variables over all nodes
322 * max_row is the allocated number of rows in the schedule
323 * n_row is the current (maximal) number of linearly independent
324 * rows in the node schedules
325 * n_total_row is the current number of rows in the node schedules
326 * band_start is the starting row in the node schedules of the current band
327 * root is set to the original dependence graph from which this graph
328 * is derived through splitting. If this graph is not the result of
329 * splitting, then the root field points to the graph itself.
330 *
331 * sorted contains a list of node indices sorted according to the
332 * SCC to which a node belongs
333 *
334 * n_edge is the number of edges
335 * edge is the list of edges
336 * max_edge contains the maximal number of edges of each type;
337 * in particular, it contains the number of edges in the inital graph.
338 * edge_table contains pointers into the edge array, hashed on the source
339 * and sink spaces; there is one such table for each type;
340 * a given edge may be referenced from more than one table
341 * if the corresponding relation appears in more than one of the
342 * sets of dependences; however, for each type there is only
343 * a single edge between a given pair of source and sink space
344 * in the entire graph
345 *
346 * node_table contains pointers into the node array, hashed on the space tuples
347 *
348 * region contains a list of variable sequences that should be non-trivial
349 *
350 * lp contains the (I)LP problem used to obtain new schedule rows
351 *
352 * src_scc and dst_scc are the source and sink SCCs of an edge with
353 * conflicting constraints
354 *
355 * scc represents the number of components
356 * weak is set if the components are weakly connected
357 *
358 * max_weight is used during clustering and represents the maximal
359 * weight of the relevant proximity edges.
360 */
361struct isl_sched_graph {
362 isl_map_to_basic_set *intra_hmap;
363 isl_map_to_basic_set *intra_hmap_param;
364 isl_map_to_basic_set *inter_hmap;
365
366 struct isl_sched_node *node;
367 int n;
368 int maxvar;
369 int max_row;
370 int n_row;
371
372 int *sorted;
373
374 int n_total_row;
375 int band_start;
376
377 struct isl_sched_graph *root;
378
379 struct isl_sched_edge *edge;
380 int n_edge;
381 int max_edge[isl_edge_last + 1];
382 struct isl_hash_table *edge_table[isl_edge_last + 1];
383
384 struct isl_hash_table *node_table;
385 struct isl_trivial_region *region;
386
387 isl_basic_setisl_basic_map *lp;
388
389 int src_scc;
390 int dst_scc;
391
392 int scc;
393 int weak;
394
395 int max_weight;
396};
397
398/* Initialize node_table based on the list of nodes.
399 */
400static int graph_init_table(isl_ctx *ctx, struct isl_sched_graph *graph)
401{
402 int i;
403
404 graph->node_table = isl_hash_table_alloc(ctx, graph->n);
405 if (!graph->node_table)
406 return -1;
407
408 for (i = 0; i < graph->n; ++i) {
409 struct isl_hash_table_entry *entry;
410 uint32_t hash;
411
412 hash = isl_space_get_tuple_hash(graph->node[i].space);
413 entry = isl_hash_table_find(ctx, graph->node_table, hash,
414 &node_has_tuples,
415 graph->node[i].space, 1);
416 if (!entry)
417 return -1;
418 entry->data = &graph->node[i];
419 }
420
421 return 0;
422}
423
424/* Return a pointer to the node that lives within the given space,
425 * an invalid node if there is no such node, or NULL in case of error.
426 */
427static struct isl_sched_node *graph_find_node(isl_ctx *ctx,
428 struct isl_sched_graph *graph, __isl_keep isl_space *space)
429{
430 struct isl_hash_table_entry *entry;
431 uint32_t hash;
432
433 if (!space)
434 return NULL((void*)0);
435
436 hash = isl_space_get_tuple_hash(space);
437 entry = isl_hash_table_find(ctx, graph->node_table, hash,
438 &node_has_tuples, space, 0);
439
440 return entry ? entry->data : graph->node + graph->n;
441}
442
443/* Is "node" a node in "graph"?
444 */
445static int is_node(struct isl_sched_graph *graph,
446 struct isl_sched_node *node)
447{
448 return node && node >= &graph->node[0] && node < &graph->node[graph->n];
449}
450
451static int edge_has_src_and_dst(const void *entry, const void *val)
452{
453 const struct isl_sched_edge *edge = entry;
454 const struct isl_sched_edge *temp = val;
455
456 return edge->src == temp->src && edge->dst == temp->dst;
457}
458
459/* Add the given edge to graph->edge_table[type].
460 */
461static isl_stat graph_edge_table_add(isl_ctx *ctx,
462 struct isl_sched_graph *graph, enum isl_edge_type type,
463 struct isl_sched_edge *edge)
464{
465 struct isl_hash_table_entry *entry;
466 uint32_t hash;
467
468 hash = isl_hash_init()(2166136261u);
469 hash = isl_hash_builtin(hash, edge->src)isl_hash_mem(hash, &edge->src, sizeof(edge->src));
470 hash = isl_hash_builtin(hash, edge->dst)isl_hash_mem(hash, &edge->dst, sizeof(edge->dst));
471 entry = isl_hash_table_find(ctx, graph->edge_table[type], hash,
472 &edge_has_src_and_dst, edge, 1);
473 if (!entry)
474 return isl_stat_error;
475 entry->data = edge;
476
477 return isl_stat_ok;
478}
479
480/* Add "edge" to all relevant edge tables.
481 * That is, for every type of the edge, add it to the corresponding table.
482 */
483static isl_stat graph_edge_tables_add(isl_ctx *ctx,
484 struct isl_sched_graph *graph, struct isl_sched_edge *edge)
485{
486 enum isl_edge_type t;
487
488 for (t = isl_edge_first; t <= isl_edge_last; ++t) {
489 if (!is_type(edge, t))
490 continue;
491 if (graph_edge_table_add(ctx, graph, t, edge) < 0)
492 return isl_stat_error;
493 }
494
495 return isl_stat_ok;
496}
497
498/* Allocate the edge_tables based on the maximal number of edges of
499 * each type.
500 */
501static int graph_init_edge_tables(isl_ctx *ctx, struct isl_sched_graph *graph)
502{
503 int i;
504
505 for (i = 0; i <= isl_edge_last; ++i) {
506 graph->edge_table[i] = isl_hash_table_alloc(ctx,
507 graph->max_edge[i]);
508 if (!graph->edge_table[i])
509 return -1;
510 }
511
512 return 0;
513}
514
515/* If graph->edge_table[type] contains an edge from the given source
516 * to the given destination, then return the hash table entry of this edge.
517 * Otherwise, return NULL.
518 */
519static struct isl_hash_table_entry *graph_find_edge_entry(
520 struct isl_sched_graph *graph,
521 enum isl_edge_type type,
522 struct isl_sched_node *src, struct isl_sched_node *dst)
523{
524 isl_ctx *ctx = isl_space_get_ctx(src->space);
525 uint32_t hash;
526 struct isl_sched_edge temp = { .src = src, .dst = dst };
527
528 hash = isl_hash_init()(2166136261u);
529 hash = isl_hash_builtin(hash, temp.src)isl_hash_mem(hash, &temp.src, sizeof(temp.src));
530 hash = isl_hash_builtin(hash, temp.dst)isl_hash_mem(hash, &temp.dst, sizeof(temp.dst));
531 return isl_hash_table_find(ctx, graph->edge_table[type], hash,
532 &edge_has_src_and_dst, &temp, 0);
533}
534
535
536/* If graph->edge_table[type] contains an edge from the given source
537 * to the given destination, then return this edge.
538 * Otherwise, return NULL.
539 */
540static struct isl_sched_edge *graph_find_edge(struct isl_sched_graph *graph,
541 enum isl_edge_type type,
542 struct isl_sched_node *src, struct isl_sched_node *dst)
543{
544 struct isl_hash_table_entry *entry;
545
546 entry = graph_find_edge_entry(graph, type, src, dst);
547 if (!entry)
548 return NULL((void*)0);
549
550 return entry->data;
551}
552
553/* Check whether the dependence graph has an edge of the given type
554 * between the given two nodes.
555 */
556static isl_bool graph_has_edge(struct isl_sched_graph *graph,
557 enum isl_edge_type type,
558 struct isl_sched_node *src, struct isl_sched_node *dst)
559{
560 struct isl_sched_edge *edge;
561 isl_bool empty;
562
563 edge = graph_find_edge(graph, type, src, dst);
564 if (!edge)
565 return isl_bool_false;
566
567 empty = isl_map_plain_is_empty(edge->map);
568 if (empty < 0)
569 return isl_bool_error;
570
571 return !empty;
572}
573
574/* Look for any edge with the same src, dst and map fields as "model".
575 *
576 * Return the matching edge if one can be found.
577 * Return "model" if no matching edge is found.
578 * Return NULL on error.
579 */
580static struct isl_sched_edge *graph_find_matching_edge(
581 struct isl_sched_graph *graph, struct isl_sched_edge *model)
582{
583 enum isl_edge_type i;
584 struct isl_sched_edge *edge;
585
586 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
587 int is_equal;
588
589 edge = graph_find_edge(graph, i, model->src, model->dst);
590 if (!edge)
591 continue;
592 is_equal = isl_map_plain_is_equal(model->map, edge->map);
593 if (is_equal < 0)
594 return NULL((void*)0);
595 if (is_equal)
596 return edge;
597 }
598
599 return model;
600}
601
602/* Remove the given edge from all the edge_tables that refer to it.
603 */
604static void graph_remove_edge(struct isl_sched_graph *graph,
605 struct isl_sched_edge *edge)
606{
607 isl_ctx *ctx = isl_map_get_ctx(edge->map);
608 enum isl_edge_type i;
609
610 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
611 struct isl_hash_table_entry *entry;
612
613 entry = graph_find_edge_entry(graph, i, edge->src, edge->dst);
614 if (!entry)
615 continue;
616 if (entry->data != edge)
617 continue;
618 isl_hash_table_remove(ctx, graph->edge_table[i], entry);
619 }
620}
621
622/* Check whether the dependence graph has any edge
623 * between the given two nodes.
624 */
625static isl_bool graph_has_any_edge(struct isl_sched_graph *graph,
626 struct isl_sched_node *src, struct isl_sched_node *dst)
627{
628 enum isl_edge_type i;
629 isl_bool r;
630
631 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
632 r = graph_has_edge(graph, i, src, dst);
633 if (r < 0 || r)
634 return r;
635 }
636
637 return r;
638}
639
640/* Check whether the dependence graph has a validity edge
641 * between the given two nodes.
642 *
643 * Conditional validity edges are essentially validity edges that
644 * can be ignored if the corresponding condition edges are iteration private.
645 * Here, we are only checking for the presence of validity
646 * edges, so we need to consider the conditional validity edges too.
647 * In particular, this function is used during the detection
648 * of strongly connected components and we cannot ignore
649 * conditional validity edges during this detection.
650 */
651static isl_bool graph_has_validity_edge(struct isl_sched_graph *graph,
652 struct isl_sched_node *src, struct isl_sched_node *dst)
653{
654 isl_bool r;
655
656 r = graph_has_edge(graph, isl_edge_validity, src, dst);
657 if (r < 0 || r)
658 return r;
659
660 return graph_has_edge(graph, isl_edge_conditional_validity, src, dst);
661}
662
663/* Perform all the required memory allocations for a schedule graph "graph"
664 * with "n_node" nodes and "n_edge" edge and initialize the corresponding
665 * fields.
666 */
667static isl_stat graph_alloc(isl_ctx *ctx, struct isl_sched_graph *graph,
668 int n_node, int n_edge)
669{
670 int i;
671
672 graph->n = n_node;
673 graph->n_edge = n_edge;
674 graph->node = isl_calloc_array(ctx, struct isl_sched_node, graph->n)((struct isl_sched_node *)isl_calloc_or_die(ctx, graph->n,
sizeof(struct isl_sched_node)))
;
675 graph->sorted = isl_calloc_array(ctx, int, graph->n)((int *)isl_calloc_or_die(ctx, graph->n, sizeof(int)));
676 graph->region = isl_alloc_array(ctx,((struct isl_trivial_region *)isl_malloc_or_die(ctx, (graph->
n)*sizeof(struct isl_trivial_region)))
677 struct isl_trivial_region, graph->n)((struct isl_trivial_region *)isl_malloc_or_die(ctx, (graph->
n)*sizeof(struct isl_trivial_region)))
;
678 graph->edge = isl_calloc_array(ctx,((struct isl_sched_edge *)isl_calloc_or_die(ctx, graph->n_edge
, sizeof(struct isl_sched_edge)))
679 struct isl_sched_edge, graph->n_edge)((struct isl_sched_edge *)isl_calloc_or_die(ctx, graph->n_edge
, sizeof(struct isl_sched_edge)))
;
680
681 graph->intra_hmap = isl_map_to_basic_set_alloc(ctx, 2 * n_edge);
682 graph->intra_hmap_param = isl_map_to_basic_set_alloc(ctx, 2 * n_edge);
683 graph->inter_hmap = isl_map_to_basic_set_alloc(ctx, 2 * n_edge);
684
685 if (!graph->node || !graph->region || (graph->n_edge && !graph->edge) ||
686 !graph->sorted)
687 return isl_stat_error;
688
689 for(i = 0; i < graph->n; ++i)
690 graph->sorted[i] = i;
691
692 return isl_stat_ok;
693}
694
695/* Free the memory associated to node "node" in "graph".
696 * The "coincident" field is shared by nodes in a graph and its subgraph.
697 * It therefore only needs to be freed for the original dependence graph,
698 * i.e., one that is not the result of splitting.
699 */
700static void clear_node(struct isl_sched_graph *graph,
701 struct isl_sched_node *node)
702{
703 isl_space_free(node->space);
704 isl_set_free(node->hull);
705 isl_multi_aff_free(node->compress);
706 isl_multi_aff_free(node->decompress);
707 isl_mat_free(node->sched);
708 isl_map_free(node->sched_map);
709 isl_mat_free(node->indep);
710 isl_mat_free(node->vmap);
711 if (graph->root == graph)
712 free(node->coincident);
713 isl_multi_val_free(node->sizes);
714 isl_basic_set_free(node->bounds);
715 isl_vec_free(node->max);
716}
717
718static void graph_free(isl_ctx *ctx, struct isl_sched_graph *graph)
719{
720 int i;
721
722 isl_map_to_basic_set_free(graph->intra_hmap);
723 isl_map_to_basic_set_free(graph->intra_hmap_param);
724 isl_map_to_basic_set_free(graph->inter_hmap);
725
726 if (graph->node)
727 for (i = 0; i < graph->n; ++i)
728 clear_node(graph, &graph->node[i]);
729 free(graph->node);
730 free(graph->sorted);
731 if (graph->edge)
732 for (i = 0; i < graph->n_edge; ++i) {
733 isl_map_free(graph->edge[i].map);
734 isl_union_map_free(graph->edge[i].tagged_condition);
735 isl_union_map_free(graph->edge[i].tagged_validity);
736 }
737 free(graph->edge);
738 free(graph->region);
739 for (i = 0; i <= isl_edge_last; ++i)
740 isl_hash_table_free(ctx, graph->edge_table[i]);
741 isl_hash_table_free(ctx, graph->node_table);
742 isl_basic_set_free(graph->lp);
743}
744
745/* For each "set" on which this function is called, increment
746 * graph->n by one and update graph->maxvar.
747 */
748static isl_stat init_n_maxvar(__isl_take isl_setisl_map *set, void *user)
749{
750 struct isl_sched_graph *graph = user;
751 int nvar = isl_set_dim(set, isl_dim_set);
752
753 graph->n++;
754 if (nvar > graph->maxvar)
755 graph->maxvar = nvar;
756
757 isl_set_free(set);
758
759 return isl_stat_ok;
760}
761
762/* Compute the number of rows that should be allocated for the schedule.
763 * In particular, we need one row for each variable or one row
764 * for each basic map in the dependences.
765 * Note that it is practically impossible to exhaust both
766 * the number of dependences and the number of variables.
767 */
768static isl_stat compute_max_row(struct isl_sched_graph *graph,
769 __isl_keep isl_schedule_constraints *sc)
770{
771 int n_edge;
772 isl_stat r;
773 isl_union_set *domain;
774
775 graph->n = 0;
776 graph->maxvar = 0;
777 domain = isl_schedule_constraints_get_domain(sc);
778 r = isl_union_set_foreach_set(domain, &init_n_maxvar, graph);
779 isl_union_set_free(domain);
780 if (r < 0)
781 return isl_stat_error;
782 n_edge = isl_schedule_constraints_n_basic_map(sc);
783 if (n_edge < 0)
784 return isl_stat_error;
785 graph->max_row = n_edge + graph->maxvar;
786
787 return isl_stat_ok;
788}
789
790/* Does "bset" have any defining equalities for its set variables?
791 */
792static isl_bool has_any_defining_equality(__isl_keep isl_basic_setisl_basic_map *bset)
793{
794 int i, n;
795
796 if (!bset)
797 return isl_bool_error;
798
799 n = isl_basic_set_dim(bset, isl_dim_set);
800 for (i = 0; i < n; ++i) {
801 isl_bool has;
802
803 has = isl_basic_set_has_defining_equality(bset, isl_dim_set, i,
804 NULL((void*)0));
805 if (has < 0 || has)
806 return has;
807 }
808
809 return isl_bool_false;
810}
811
812/* Set the entries of node->max to the value of the schedule_max_coefficient
813 * option, if set.
814 */
815static isl_stat set_max_coefficient(isl_ctx *ctx, struct isl_sched_node *node)
816{
817 int max;
818
819 max = isl_options_get_schedule_max_coefficient(ctx);
820 if (max == -1)
821 return isl_stat_ok;
822
823 node->max = isl_vec_alloc(ctx, node->nvar);
824 node->max = isl_vec_set_si(node->max, max);
825 if (!node->max)
826 return isl_stat_error;
827
828 return isl_stat_ok;
829}
830
831/* Set the entries of node->max to the minimum of the schedule_max_coefficient
832 * option (if set) and half of the minimum of the sizes in the other
833 * dimensions. Round up when computing the half such that
834 * if the minimum of the sizes is one, half of the size is taken to be one
835 * rather than zero.
836 * If the global minimum is unbounded (i.e., if both
837 * the schedule_max_coefficient is not set and the sizes in the other
838 * dimensions are unbounded), then store a negative value.
839 * If the schedule coefficient is close to the size of the instance set
840 * in another dimension, then the schedule may represent a loop
841 * coalescing transformation (especially if the coefficient
842 * in that other dimension is one). Forcing the coefficient to be
843 * smaller than or equal to half the minimal size should avoid this
844 * situation.
845 */
846static isl_stat compute_max_coefficient(isl_ctx *ctx,
847 struct isl_sched_node *node)
848{
849 int max;
850 int i, j;
851 isl_vec *v;
852
853 max = isl_options_get_schedule_max_coefficient(ctx);
854 v = isl_vec_alloc(ctx, node->nvar);
855 if (!v)
856 return isl_stat_error;
857
858 for (i = 0; i < node->nvar; ++i) {
859 isl_int_set_si(v->el[i], max)isl_sioimath_set_si((v->el[i]), max);
860 isl_int_mul_si(v->el[i], v->el[i], 2)isl_sioimath_mul_si((v->el[i]), *(v->el[i]), 2);
861 }
862
863 for (i = 0; i < node->nvar; ++i) {
864 isl_val *size;
865
866 size = isl_multi_val_get_val(node->sizes, i);
867 if (!size)
868 goto error;
869 if (!isl_val_is_int(size)) {
870 isl_val_free(size);
871 continue;
872 }
873 for (j = 0; j < node->nvar; ++j) {
874 if (j == i)
875 continue;
876 if (isl_int_is_neg(v->el[j])(isl_sioimath_sgn(*(v->el[j])) < 0) ||
877 isl_int_gt(v->el[j], size->n)(isl_sioimath_cmp(*(v->el[j]), *(size->n)) > 0))
878 isl_int_set(v->el[j], size->n)isl_sioimath_set((v->el[j]), *(size->n));
879 }
880 isl_val_free(size);
881 }
882
883 for (i = 0; i < node->nvar; ++i)
884 isl_int_cdiv_q_ui(v->el[i], v->el[i], 2)isl_sioimath_cdiv_q_ui((v->el[i]), *(v->el[i]), 2);
885
886 node->max = v;
887 return isl_stat_ok;
888error:
889 isl_vec_free(v);
890 return isl_stat_error;
891}
892
893/* Compute and return the size of "set" in dimension "dim".
894 * The size is taken to be the difference in values for that variable
895 * for fixed values of the other variables.
896 * This assumes that "set" is convex.
897 * In particular, the variable is first isolated from the other variables
898 * in the range of a map
899 *
900 * [i_0, ..., i_dim-1, i_dim+1, ...] -> [i_dim]
901 *
902 * and then duplicated
903 *
904 * [i_0, ..., i_dim-1, i_dim+1, ...] -> [[i_dim] -> [i_dim']]
905 *
906 * The shared variables are then projected out and the maximal value
907 * of i_dim' - i_dim is computed.
908 */
909static __isl_give isl_val *compute_size(__isl_take isl_setisl_map *set, int dim)
910{
911 isl_map *map;
912 isl_local_space *ls;
913 isl_aff *obj;
914 isl_val *v;
915
916 map = isl_set_project_onto_map(set, isl_dim_set, dim, 1);
917 map = isl_map_project_out(map, isl_dim_in, dim, 1);
918 map = isl_map_range_product(map, isl_map_copy(map));
919 map = isl_set_unwrap(isl_map_range(map));
920 set = isl_map_deltas(map);
921 ls = isl_local_space_from_space(isl_set_get_space(set));
922 obj = isl_aff_var_on_domain(ls, isl_dim_set, 0);
923 v = isl_set_max_val(set, obj);
924 isl_aff_free(obj);
925 isl_set_free(set);
926
927 return v;
928}
929
930/* Compute the size of the instance set "set" of "node", after compression,
931 * as well as bounds on the corresponding coefficients, if needed.
932 *
933 * The sizes are needed when the schedule_treat_coalescing option is set.
934 * The bounds are needed when the schedule_treat_coalescing option or
935 * the schedule_max_coefficient option is set.
936 *
937 * If the schedule_treat_coalescing option is not set, then at most
938 * the bounds need to be set and this is done in set_max_coefficient.
939 * Otherwise, compress the domain if needed, compute the size
940 * in each direction and store the results in node->size.
941 * If the domain is not convex, then the sizes are computed
942 * on a convex superset in order to avoid picking up sizes
943 * that are valid for the individual disjuncts, but not for
944 * the domain as a whole.
945 * Finally, set the bounds on the coefficients based on the sizes
946 * and the schedule_max_coefficient option in compute_max_coefficient.
947 */
948static isl_stat compute_sizes_and_max(isl_ctx *ctx, struct isl_sched_node *node,
949 __isl_take isl_setisl_map *set)
950{
951 int j, n;
952 isl_multi_val *mv;
953
954 if (!isl_options_get_schedule_treat_coalescing(ctx)) {
955 isl_set_free(set);
956 return set_max_coefficient(ctx, node);
957 }
958
959 if (node->compressed)
960 set = isl_set_preimage_multi_aff(set,
961 isl_multi_aff_copy(node->decompress));
962 set = isl_set_from_basic_set(isl_set_simple_hull(set));
963 mv = isl_multi_val_zero(isl_set_get_space(set));
964 n = isl_set_dim(set, isl_dim_set);
965 for (j = 0; j < n; ++j) {
966 isl_val *v;
967
968 v = compute_size(isl_set_copy(set), j);
969 mv = isl_multi_val_set_val(mv, j, v);
970 }
971 node->sizes = mv;
972 isl_set_free(set);
973 if (!node->sizes)
974 return isl_stat_error;
975 return compute_max_coefficient(ctx, node);
976}
977
978/* Add a new node to the graph representing the given instance set.
979 * "nvar" is the (possibly compressed) number of variables and
980 * may be smaller than then number of set variables in "set"
981 * if "compressed" is set.
982 * If "compressed" is set, then "hull" represents the constraints
983 * that were used to derive the compression, while "compress" and
984 * "decompress" map the original space to the compressed space and
985 * vice versa.
986 * If "compressed" is not set, then "hull", "compress" and "decompress"
987 * should be NULL.
988 *
989 * Compute the size of the instance set and bounds on the coefficients,
990 * if needed.
991 */
992static isl_stat add_node(struct isl_sched_graph *graph,
993 __isl_take isl_setisl_map *set, int nvar, int compressed,
994 __isl_take isl_setisl_map *hull, __isl_take isl_multi_aff *compress,
995 __isl_take isl_multi_aff *decompress)
996{
997 int nparam;
998 isl_ctx *ctx;
999 isl_mat *sched;
1000 isl_space *space;
1001 int *coincident;
1002 struct isl_sched_node *node;
1003
1004 if (!set)
1005 return isl_stat_error;
1006
1007 ctx = isl_set_get_ctx(set);
1008 nparam = isl_set_dim(set, isl_dim_param);
1009 if (!ctx->opt->schedule_parametric)
1010 nparam = 0;
1011 sched = isl_mat_alloc(ctx, 0, 1 + nparam + nvar);
1012 node = &graph->node[graph->n];
1013 graph->n++;
1014 space = isl_set_get_space(set);
1015 node->space = space;
1016 node->nvar = nvar;
1017 node->nparam = nparam;
1018 node->sched = sched;
1019 node->sched_map = NULL((void*)0);
1020 coincident = isl_calloc_array(ctx, int, graph->max_row)((int *)isl_calloc_or_die(ctx, graph->max_row, sizeof(int)
))
;
1021 node->coincident = coincident;
1022 node->compressed = compressed;
1023 node->hull = hull;
1024 node->compress = compress;
1025 node->decompress = decompress;
1026 if (compute_sizes_and_max(ctx, node, set) < 0)
1027 return isl_stat_error;
1028
1029 if (!space || !sched || (graph->max_row && !coincident))
1030 return isl_stat_error;
1031 if (compressed && (!hull || !compress || !decompress))
1032 return isl_stat_error;
1033
1034 return isl_stat_ok;
1035}
1036
1037/* Construct an identifier for node "node", which will represent "set".
1038 * The name of the identifier is either "compressed" or
1039 * "compressed_<name>", with <name> the name of the space of "set".
1040 * The user pointer of the identifier points to "node".
1041 */
1042static __isl_give isl_id *construct_compressed_id(__isl_keep isl_setisl_map *set,
1043 struct isl_sched_node *node)
1044{
1045 isl_bool has_name;
1046 isl_ctx *ctx;
1047 isl_id *id;
1048 isl_printer *p;
1049 const char *name;
1050 char *id_name;
1051
1052 has_name = isl_set_has_tuple_name(set);
1053 if (has_name < 0)
1054 return NULL((void*)0);
1055
1056 ctx = isl_set_get_ctx(set);
1057 if (!has_name)
1058 return isl_id_alloc(ctx, "compressed", node);
1059
1060 p = isl_printer_to_str(ctx);
1061 name = isl_set_get_tuple_name(set);
1062 p = isl_printer_print_str(p, "compressed_");
1063 p = isl_printer_print_str(p, name);
1064 id_name = isl_printer_get_str(p);
1065 isl_printer_free(p);
1066
1067 id = isl_id_alloc(ctx, id_name, node);
1068 free(id_name);
1069
1070 return id;
1071}
1072
1073/* Add a new node to the graph representing the given set.
1074 *
1075 * If any of the set variables is defined by an equality, then
1076 * we perform variable compression such that we can perform
1077 * the scheduling on the compressed domain.
1078 * In this case, an identifier is used that references the new node
1079 * such that each compressed space is unique and
1080 * such that the node can be recovered from the compressed space.
1081 */
1082static isl_stat extract_node(__isl_take isl_setisl_map *set, void *user)
1083{
1084 int nvar;
1085 isl_bool has_equality;
1086 isl_id *id;
1087 isl_basic_setisl_basic_map *hull;
1088 isl_setisl_map *hull_set;
1089 isl_morph *morph;
1090 isl_multi_aff *compress, *decompress;
1091 struct isl_sched_graph *graph = user;
1092
1093 hull = isl_set_affine_hull(isl_set_copy(set));
1094 hull = isl_basic_set_remove_divs(hull);
1095 nvar = isl_set_dim(set, isl_dim_set);
1096 has_equality = has_any_defining_equality(hull);
1097
1098 if (has_equality < 0)
1099 goto error;
1100 if (!has_equality) {
1101 isl_basic_set_free(hull);
1102 return add_node(graph, set, nvar, 0, NULL((void*)0), NULL((void*)0), NULL((void*)0));
1103 }
1104
1105 id = construct_compressed_id(set, &graph->node[graph->n]);
1106 morph = isl_basic_set_variable_compression_with_id(hull,
1107 isl_dim_set, id);
1108 isl_id_free(id);
1109 nvar = isl_morph_ran_dim(morph, isl_dim_set);
1110 compress = isl_morph_get_var_multi_aff(morph);
1111 morph = isl_morph_inverse(morph);
1112 decompress = isl_morph_get_var_multi_aff(morph);
1113 isl_morph_free(morph);
1114
1115 hull_set = isl_set_from_basic_set(hull);
1116 return add_node(graph, set, nvar, 1, hull_set, compress, decompress);
1117error:
1118 isl_basic_set_free(hull);
1119 isl_set_free(set);
1120 return isl_stat_error;
1121}
1122
1123struct isl_extract_edge_data {
1124 enum isl_edge_type type;
1125 struct isl_sched_graph *graph;
1126};
1127
1128/* Merge edge2 into edge1, freeing the contents of edge2.
1129 * Return 0 on success and -1 on failure.
1130 *
1131 * edge1 and edge2 are assumed to have the same value for the map field.
1132 */
1133static int merge_edge(struct isl_sched_edge *edge1,
1134 struct isl_sched_edge *edge2)
1135{
1136 edge1->types |= edge2->types;
1137 isl_map_free(edge2->map);
1138
1139 if (is_condition(edge2)) {
1140 if (!edge1->tagged_condition)
1141 edge1->tagged_condition = edge2->tagged_condition;
1142 else
1143 edge1->tagged_condition =
1144 isl_union_map_union(edge1->tagged_condition,
1145 edge2->tagged_condition);
1146 }
1147
1148 if (is_conditional_validity(edge2)) {
1149 if (!edge1->tagged_validity)
1150 edge1->tagged_validity = edge2->tagged_validity;
1151 else
1152 edge1->tagged_validity =
1153 isl_union_map_union(edge1->tagged_validity,
1154 edge2->tagged_validity);
1155 }
1156
1157 if (is_condition(edge2) && !edge1->tagged_condition)
1158 return -1;
1159 if (is_conditional_validity(edge2) && !edge1->tagged_validity)
1160 return -1;
1161
1162 return 0;
1163}
1164
1165/* Insert dummy tags in domain and range of "map".
1166 *
1167 * In particular, if "map" is of the form
1168 *
1169 * A -> B
1170 *
1171 * then return
1172 *
1173 * [A -> dummy_tag] -> [B -> dummy_tag]
1174 *
1175 * where the dummy_tags are identical and equal to any dummy tags
1176 * introduced by any other call to this function.
1177 */
1178static __isl_give isl_map *insert_dummy_tags(__isl_take isl_map *map)
1179{
1180 static char dummy;
1181 isl_ctx *ctx;
1182 isl_id *id;
1183 isl_space *space;
1184 isl_setisl_map *domain, *range;
1185
1186 ctx = isl_map_get_ctx(map);
1187
1188 id = isl_id_alloc(ctx, NULL((void*)0), &dummy);
1189 space = isl_space_params(isl_map_get_space(map));
1190 space = isl_space_set_from_params(space);
1191 space = isl_space_set_tuple_id(space, isl_dim_set, id);
1192 space = isl_space_map_from_set(space);
1193
1194 domain = isl_map_wrap(map);
1195 range = isl_map_wrap(isl_map_universe(space));
1196 map = isl_map_from_domain_and_range(domain, range);
1197 map = isl_map_zip(map);
1198
1199 return map;
1200}
1201
1202/* Given that at least one of "src" or "dst" is compressed, return
1203 * a map between the spaces of these nodes restricted to the affine
1204 * hull that was used in the compression.
1205 */
1206static __isl_give isl_map *extract_hull(struct isl_sched_node *src,
1207 struct isl_sched_node *dst)
1208{
1209 isl_setisl_map *dom, *ran;
1210
1211 if (src->compressed)
1212 dom = isl_set_copy(src->hull);
1213 else
1214 dom = isl_set_universe(isl_space_copy(src->space));
1215 if (dst->compressed)
1216 ran = isl_set_copy(dst->hull);
1217 else
1218 ran = isl_set_universe(isl_space_copy(dst->space));
1219
1220 return isl_map_from_domain_and_range(dom, ran);
1221}
1222
1223/* Intersect the domains of the nested relations in domain and range
1224 * of "tagged" with "map".
1225 */
1226static __isl_give isl_map *map_intersect_domains(__isl_take isl_map *tagged,
1227 __isl_keep isl_map *map)
1228{
1229 isl_setisl_map *set;
1230
1231 tagged = isl_map_zip(tagged);
1232 set = isl_map_wrap(isl_map_copy(map));
1233 tagged = isl_map_intersect_domain(tagged, set);
1234 tagged = isl_map_zip(tagged);
1235 return tagged;
1236}
1237
1238/* Return a pointer to the node that lives in the domain space of "map",
1239 * an invalid node if there is no such node, or NULL in case of error.
1240 */
1241static struct isl_sched_node *find_domain_node(isl_ctx *ctx,
1242 struct isl_sched_graph *graph, __isl_keep isl_map *map)
1243{
1244 struct isl_sched_node *node;
1245 isl_space *space;
1246
1247 space = isl_space_domain(isl_map_get_space(map));
1248 node = graph_find_node(ctx, graph, space);
1249 isl_space_free(space);
1250
1251 return node;
1252}
1253
1254/* Return a pointer to the node that lives in the range space of "map",
1255 * an invalid node if there is no such node, or NULL in case of error.
1256 */
1257static struct isl_sched_node *find_range_node(isl_ctx *ctx,
1258 struct isl_sched_graph *graph, __isl_keep isl_map *map)
1259{
1260 struct isl_sched_node *node;
1261 isl_space *space;
1262
1263 space = isl_space_range(isl_map_get_space(map));
1264 node = graph_find_node(ctx, graph, space);
1265 isl_space_free(space);
1266
1267 return node;
1268}
1269
1270/* Refrain from adding a new edge based on "map".
1271 * Instead, just free the map.
1272 * "tagged" is either a copy of "map" with additional tags or NULL.
1273 */
1274static isl_stat skip_edge(__isl_take isl_map *map, __isl_take isl_map *tagged)
1275{
1276 isl_map_free(map);
1277 isl_map_free(tagged);
1278
1279 return isl_stat_ok;
1280}
1281
1282/* Add a new edge to the graph based on the given map
1283 * and add it to data->graph->edge_table[data->type].
1284 * If a dependence relation of a given type happens to be identical
1285 * to one of the dependence relations of a type that was added before,
1286 * then we don't create a new edge, but instead mark the original edge
1287 * as also representing a dependence of the current type.
1288 *
1289 * Edges of type isl_edge_condition or isl_edge_conditional_validity
1290 * may be specified as "tagged" dependence relations. That is, "map"
1291 * may contain elements (i -> a) -> (j -> b), where i -> j denotes
1292 * the dependence on iterations and a and b are tags.
1293 * edge->map is set to the relation containing the elements i -> j,
1294 * while edge->tagged_condition and edge->tagged_validity contain
1295 * the union of all the "map" relations
1296 * for which extract_edge is called that result in the same edge->map.
1297 *
1298 * If the source or the destination node is compressed, then
1299 * intersect both "map" and "tagged" with the constraints that
1300 * were used to construct the compression.
1301 * This ensures that there are no schedule constraints defined
1302 * outside of these domains, while the scheduler no longer has
1303 * any control over those outside parts.
1304 */
1305static isl_stat extract_edge(__isl_take isl_map *map, void *user)
1306{
1307 isl_bool empty;
1308 isl_ctx *ctx = isl_map_get_ctx(map);
1309 struct isl_extract_edge_data *data = user;
1310 struct isl_sched_graph *graph = data->graph;
1311 struct isl_sched_node *src, *dst;
1312 struct isl_sched_edge *edge;
1313 isl_map *tagged = NULL((void*)0);
1314
1315 if (data->type == isl_edge_condition ||
1316 data->type == isl_edge_conditional_validity) {
1317 if (isl_map_can_zip(map)) {
1318 tagged = isl_map_copy(map);
1319 map = isl_set_unwrap(isl_map_domain(isl_map_zip(map)));
1320 } else {
1321 tagged = insert_dummy_tags(isl_map_copy(map));
1322 }
1323 }
1324
1325 src = find_domain_node(ctx, graph, map);
1326 dst = find_range_node(ctx, graph, map);
1327
1328 if (!src || !dst)
1329 goto error;
1330 if (!is_node(graph, src) || !is_node(graph, dst))
1331 return skip_edge(map, tagged);
1332
1333 if (src->compressed || dst->compressed) {
1334 isl_map *hull;
1335 hull = extract_hull(src, dst);
1336 if (tagged)
1337 tagged = map_intersect_domains(tagged, hull);
1338 map = isl_map_intersect(map, hull);
1339 }
1340
1341 empty = isl_map_plain_is_empty(map);
1342 if (empty < 0)
1343 goto error;
1344 if (empty)
1345 return skip_edge(map, tagged);
1346
1347 graph->edge[graph->n_edge].src = src;
1348 graph->edge[graph->n_edge].dst = dst;
1349 graph->edge[graph->n_edge].map = map;
1350 graph->edge[graph->n_edge].types = 0;
1351 graph->edge[graph->n_edge].tagged_condition = NULL((void*)0);
1352 graph->edge[graph->n_edge].tagged_validity = NULL((void*)0);
1353 set_type(&graph->edge[graph->n_edge], data->type);
1354 if (data->type == isl_edge_condition)
1355 graph->edge[graph->n_edge].tagged_condition =
1356 isl_union_map_from_map(tagged);
1357 if (data->type == isl_edge_conditional_validity)
1358 graph->edge[graph->n_edge].tagged_validity =
1359 isl_union_map_from_map(tagged);
1360
1361 edge = graph_find_matching_edge(graph, &graph->edge[graph->n_edge]);
1362 if (!edge) {
1363 graph->n_edge++;
1364 return isl_stat_error;
1365 }
1366 if (edge == &graph->edge[graph->n_edge])
1367 return graph_edge_table_add(ctx, graph, data->type,
1368 &graph->edge[graph->n_edge++]);
1369
1370 if (merge_edge(edge, &graph->edge[graph->n_edge]) < 0)
1371 return isl_stat_error;
1372
1373 return graph_edge_table_add(ctx, graph, data->type, edge);
1374error:
1375 isl_map_free(map);
1376 isl_map_free(tagged);
1377 return isl_stat_error;
1378}
1379
1380/* Initialize the schedule graph "graph" from the schedule constraints "sc".
1381 *
1382 * The context is included in the domain before the nodes of
1383 * the graphs are extracted in order to be able to exploit
1384 * any possible additional equalities.
1385 * Note that this intersection is only performed locally here.
1386 */
1387static isl_stat graph_init(struct isl_sched_graph *graph,
1388 __isl_keep isl_schedule_constraints *sc)
1389{
1390 isl_ctx *ctx;
1391 isl_union_set *domain;
1392 isl_union_map *c;
1393 struct isl_extract_edge_data data;
1394 enum isl_edge_type i;
1395 isl_stat r;
1396
1397 if (!sc)
1398 return isl_stat_error;
1399
1400 ctx = isl_schedule_constraints_get_ctx(sc);
1401
1402 domain = isl_schedule_constraints_get_domain(sc);
1403 graph->n = isl_union_set_n_set(domain);
1404 isl_union_set_free(domain);
1405
1406 if (graph_alloc(ctx, graph, graph->n,
1407 isl_schedule_constraints_n_map(sc)) < 0)
1408 return isl_stat_error;
1409
1410 if (compute_max_row(graph, sc) < 0)
1411 return isl_stat_error;
1412 graph->root = graph;
1413 graph->n = 0;
1414 domain = isl_schedule_constraints_get_domain(sc);
1415 domain = isl_union_set_intersect_params(domain,
1416 isl_schedule_constraints_get_context(sc));
1417 r = isl_union_set_foreach_set(domain, &extract_node, graph);
1418 isl_union_set_free(domain);
1419 if (r < 0)
1420 return isl_stat_error;
1421 if (graph_init_table(ctx, graph) < 0)
1422 return isl_stat_error;
1423 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
1424 c = isl_schedule_constraints_get(sc, i);
1425 graph->max_edge[i] = isl_union_map_n_map(c);
1426 isl_union_map_free(c);
1427 if (!c)
1428 return isl_stat_error;
1429 }
1430 if (graph_init_edge_tables(ctx, graph) < 0)
1431 return isl_stat_error;
1432 graph->n_edge = 0;
1433 data.graph = graph;
1434 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
1435 isl_stat r;
1436
1437 data.type = i;
1438 c = isl_schedule_constraints_get(sc, i);
1439 r = isl_union_map_foreach_map(c, &extract_edge, &data);
1440 isl_union_map_free(c);
1441 if (r < 0)
1442 return isl_stat_error;
1443 }
1444
1445 return isl_stat_ok;
1446}
1447
1448/* Check whether there is any dependence from node[j] to node[i]
1449 * or from node[i] to node[j].
1450 */
1451static isl_bool node_follows_weak(int i, int j, void *user)
1452{
1453 isl_bool f;
1454 struct isl_sched_graph *graph = user;
1455
1456 f = graph_has_any_edge(graph, &graph->node[j], &graph->node[i]);
1457 if (f < 0 || f)
1458 return f;
1459 return graph_has_any_edge(graph, &graph->node[i], &graph->node[j]);
1460}
1461
1462/* Check whether there is a (conditional) validity dependence from node[j]
1463 * to node[i], forcing node[i] to follow node[j].
1464 */
1465static isl_bool node_follows_strong(int i, int j, void *user)
1466{
1467 struct isl_sched_graph *graph = user;
1468
1469 return graph_has_validity_edge(graph, &graph->node[j], &graph->node[i]);
1470}
1471
1472/* Use Tarjan's algorithm for computing the strongly connected components
1473 * in the dependence graph only considering those edges defined by "follows".
1474 */
1475static isl_stat detect_ccs(isl_ctx *ctx, struct isl_sched_graph *graph,
1476 isl_bool (*follows)(int i, int j, void *user))
1477{
1478 int i, n;
1479 struct isl_tarjan_graph *g = NULL((void*)0);
1480
1481 g = isl_tarjan_graph_init(ctx, graph->n, follows, graph);
1482 if (!g)
1483 return isl_stat_error;
1484
1485 graph->scc = 0;
1486 i = 0;
1487 n = graph->n;
1488 while (n) {
1489 while (g->order[i] != -1) {
1490 graph->node[g->order[i]].scc = graph->scc;
1491 --n;
1492 ++i;
1493 }
1494 ++i;
1495 graph->scc++;
1496 }
1497
1498 isl_tarjan_graph_free(g);
1499
1500 return isl_stat_ok;
1501}
1502
1503/* Apply Tarjan's algorithm to detect the strongly connected components
1504 * in the dependence graph.
1505 * Only consider the (conditional) validity dependences and clear "weak".
1506 */
1507static isl_stat detect_sccs(isl_ctx *ctx, struct isl_sched_graph *graph)
1508{
1509 graph->weak = 0;
1510 return detect_ccs(ctx, graph, &node_follows_strong);
1511}
1512
1513/* Apply Tarjan's algorithm to detect the (weakly) connected components
1514 * in the dependence graph.
1515 * Consider all dependences and set "weak".
1516 */
1517static isl_stat detect_wccs(isl_ctx *ctx, struct isl_sched_graph *graph)
1518{
1519 graph->weak = 1;
1520 return detect_ccs(ctx, graph, &node_follows_weak);
1521}
1522
1523static int cmp_scc(const void *a, const void *b, void *data)
1524{
1525 struct isl_sched_graph *graph = data;
1526 const int *i1 = a;
1527 const int *i2 = b;
1528
1529 return graph->node[*i1].scc - graph->node[*i2].scc;
1530}
1531
1532/* Sort the elements of graph->sorted according to the corresponding SCCs.
1533 */
1534static int sort_sccs(struct isl_sched_graph *graph)
1535{
1536 return isl_sort(graph->sorted, graph->n, sizeof(int), &cmp_scc, graph);
1537}
1538
1539/* Return a non-parametric set in the compressed space of "node" that is
1540 * bounded by the size in each direction
1541 *
1542 * { [x] : -S_i <= x_i <= S_i }
1543 *
1544 * If S_i is infinity in direction i, then there are no constraints
1545 * in that direction.
1546 *
1547 * Cache the result in node->bounds.
1548 */
1549static __isl_give isl_basic_setisl_basic_map *get_size_bounds(struct isl_sched_node *node)
1550{
1551 isl_space *space;
1552 isl_basic_setisl_basic_map *bounds;
1553 int i;
1554 unsigned nparam;
1555
1556 if (node->bounds)
1557 return isl_basic_set_copy(node->bounds);
1558
1559 if (node->compressed)
1560 space = isl_multi_aff_get_domain_space(node->decompress);
1561 else
1562 space = isl_space_copy(node->space);
1563 nparam = isl_space_dim(space, isl_dim_param);
1564 space = isl_space_drop_dims(space, isl_dim_param, 0, nparam);
1565 bounds = isl_basic_set_universe(space);
1566
1567 for (i = 0; i < node->nvar; ++i) {
1568 isl_val *size;
1569
1570 size = isl_multi_val_get_val(node->sizes, i);
1571 if (!size)
1572 return isl_basic_set_free(bounds);
1573 if (!isl_val_is_int(size)) {
1574 isl_val_free(size);
1575 continue;
1576 }
1577 bounds = isl_basic_set_upper_bound_val(bounds, isl_dim_set, i,
1578 isl_val_copy(size));
1579 bounds = isl_basic_set_lower_bound_val(bounds, isl_dim_set, i,
1580 isl_val_neg(size));
1581 }
1582
1583 node->bounds = isl_basic_set_copy(bounds);
1584 return bounds;
1585}
1586
1587/* Drop some constraints from "delta" that could be exploited
1588 * to construct loop coalescing schedules.
1589 * In particular, drop those constraint that bound the difference
1590 * to the size of the domain.
1591 * First project out the parameters to improve the effectiveness.
1592 */
1593static __isl_give isl_setisl_map *drop_coalescing_constraints(
1594 __isl_take isl_setisl_map *delta, struct isl_sched_node *node)
1595{
1596 unsigned nparam;
1597 isl_basic_setisl_basic_map *bounds;
1598
1599 bounds = get_size_bounds(node);
1600
1601 nparam = isl_set_dim(delta, isl_dim_param);
1602 delta = isl_set_project_out(delta, isl_dim_param, 0, nparam);
1603 delta = isl_set_remove_divs(delta);
1604 delta = isl_set_plain_gist_basic_set(delta, bounds);
1605 return delta;
1606}
1607
1608/* Given a dependence relation R from "node" to itself,
1609 * construct the set of coefficients of valid constraints for elements
1610 * in that dependence relation.
1611 * In particular, the result contains tuples of coefficients
1612 * c_0, c_n, c_x such that
1613 *
1614 * c_0 + c_n n + c_x y - c_x x >= 0 for each (x,y) in R
1615 *
1616 * or, equivalently,
1617 *
1618 * c_0 + c_n n + c_x d >= 0 for each d in delta R = { y - x | (x,y) in R }
1619 *
1620 * We choose here to compute the dual of delta R.
1621 * Alternatively, we could have computed the dual of R, resulting
1622 * in a set of tuples c_0, c_n, c_x, c_y, and then
1623 * plugged in (c_0, c_n, c_x, -c_x).
1624 *
1625 * If "need_param" is set, then the resulting coefficients effectively
1626 * include coefficients for the parameters c_n. Otherwise, they may
1627 * have been projected out already.
1628 * Since the constraints may be different for these two cases,
1629 * they are stored in separate caches.
1630 * In particular, if no parameter coefficients are required and
1631 * the schedule_treat_coalescing option is set, then the parameters
1632 * are projected out and some constraints that could be exploited
1633 * to construct coalescing schedules are removed before the dual
1634 * is computed.
1635 *
1636 * If "node" has been compressed, then the dependence relation
1637 * is also compressed before the set of coefficients is computed.
1638 */
1639static __isl_give isl_basic_setisl_basic_map *intra_coefficients(
1640 struct isl_sched_graph *graph, struct isl_sched_node *node,
1641 __isl_take isl_map *map, int need_param)
1642{
1643 isl_ctx *ctx;
1644 isl_setisl_map *delta;
1645 isl_map *key;
1646 isl_basic_setisl_basic_map *coef;
1647 isl_maybe_isl_basic_setisl_maybe_isl_basic_map m;
1648 isl_map_to_basic_set **hmap = &graph->intra_hmap;
1649 int treat;
1650
1651 if (!map)
1652 return NULL((void*)0);
1653
1654 ctx = isl_map_get_ctx(map);
1655 treat = !need_param && isl_options_get_schedule_treat_coalescing(ctx);
1656 if (!treat)
1657 hmap = &graph->intra_hmap_param;
1658 m = isl_map_to_basic_set_try_get(*hmap, map);
1659 if (m.valid < 0 || m.valid) {
1660 isl_map_free(map);
1661 return m.value;
1662 }
1663
1664 key = isl_map_copy(map);
1665 if (node->compressed) {
1666 map = isl_map_preimage_domain_multi_aff(map,
1667 isl_multi_aff_copy(node->decompress));
1668 map = isl_map_preimage_range_multi_aff(map,
1669 isl_multi_aff_copy(node->decompress));
1670 }
1671 delta = isl_map_deltas(map);
1672 if (treat)
1673 delta = drop_coalescing_constraints(delta, node);
1674 delta = isl_set_remove_divs(delta);
1675 coef = isl_set_coefficients(delta);
1676 *hmap = isl_map_to_basic_set_set(*hmap, key, isl_basic_set_copy(coef));
1677
1678 return coef;
1679}
1680
1681/* Given a dependence relation R, construct the set of coefficients
1682 * of valid constraints for elements in that dependence relation.
1683 * In particular, the result contains tuples of coefficients
1684 * c_0, c_n, c_x, c_y such that
1685 *
1686 * c_0 + c_n n + c_x x + c_y y >= 0 for each (x,y) in R
1687 *
1688 * If the source or destination nodes of "edge" have been compressed,
1689 * then the dependence relation is also compressed before
1690 * the set of coefficients is computed.
1691 */
1692static __isl_give isl_basic_setisl_basic_map *inter_coefficients(
1693 struct isl_sched_graph *graph, struct isl_sched_edge *edge,
1694 __isl_take isl_map *map)
1695{
1696 isl_setisl_map *set;
1697 isl_map *key;
1698 isl_basic_setisl_basic_map *coef;
1699 isl_maybe_isl_basic_setisl_maybe_isl_basic_map m;
1700
1701 m = isl_map_to_basic_set_try_get(graph->inter_hmap, map);
1702 if (m.valid < 0 || m.valid) {
1703 isl_map_free(map);
1704 return m.value;
1705 }
1706
1707 key = isl_map_copy(map);
1708 if (edge->src->compressed)
1709 map = isl_map_preimage_domain_multi_aff(map,
1710 isl_multi_aff_copy(edge->src->decompress));
1711 if (edge->dst->compressed)
1712 map = isl_map_preimage_range_multi_aff(map,
1713 isl_multi_aff_copy(edge->dst->decompress));
1714 set = isl_map_wrap(isl_map_remove_divs(map));
1715 coef = isl_set_coefficients(set);
1716 graph->inter_hmap = isl_map_to_basic_set_set(graph->inter_hmap, key,
1717 isl_basic_set_copy(coef));
1718
1719 return coef;
1720}
1721
1722/* Return the position of the coefficients of the variables in
1723 * the coefficients constraints "coef".
1724 *
1725 * The space of "coef" is of the form
1726 *
1727 * { coefficients[[cst, params] -> S] }
1728 *
1729 * Return the position of S.
1730 */
1731static int coef_var_offset(__isl_keep isl_basic_setisl_basic_map *coef)
1732{
1733 int offset;
1734 isl_space *space;
1735
1736 space = isl_space_unwrap(isl_basic_set_get_space(coef));
1737 offset = isl_space_dim(space, isl_dim_in);
1738 isl_space_free(space);
1739
1740 return offset;
1741}
1742
1743/* Return the offset of the coefficient of the constant term of "node"
1744 * within the (I)LP.
1745 *
1746 * Within each node, the coefficients have the following order:
1747 * - positive and negative parts of c_i_x
1748 * - c_i_n (if parametric)
1749 * - c_i_0
1750 */
1751static int node_cst_coef_offset(struct isl_sched_node *node)
1752{
1753 return node->start + 2 * node->nvar + node->nparam;
1754}
1755
1756/* Return the offset of the coefficients of the parameters of "node"
1757 * within the (I)LP.
1758 *
1759 * Within each node, the coefficients have the following order:
1760 * - positive and negative parts of c_i_x
1761 * - c_i_n (if parametric)
1762 * - c_i_0
1763 */
1764static int node_par_coef_offset(struct isl_sched_node *node)
1765{
1766 return node->start + 2 * node->nvar;
1767}
1768
1769/* Return the offset of the coefficients of the variables of "node"
1770 * within the (I)LP.
1771 *
1772 * Within each node, the coefficients have the following order:
1773 * - positive and negative parts of c_i_x
1774 * - c_i_n (if parametric)
1775 * - c_i_0
1776 */
1777static int node_var_coef_offset(struct isl_sched_node *node)
1778{
1779 return node->start;
1780}
1781
1782/* Return the position of the pair of variables encoding
1783 * coefficient "i" of "node".
1784 *
1785 * The order of these variable pairs is the opposite of
1786 * that of the coefficients, with 2 variables per coefficient.
1787 */
1788static int node_var_coef_pos(struct isl_sched_node *node, int i)
1789{
1790 return node_var_coef_offset(node) + 2 * (node->nvar - 1 - i);
1791}
1792
1793/* Construct an isl_dim_map for mapping constraints on coefficients
1794 * for "node" to the corresponding positions in graph->lp.
1795 * "offset" is the offset of the coefficients for the variables
1796 * in the input constraints.
1797 * "s" is the sign of the mapping.
1798 *
1799 * The input constraints are given in terms of the coefficients
1800 * (c_0, c_x) or (c_0, c_n, c_x).
1801 * The mapping produced by this function essentially plugs in
1802 * (0, c_i_x^+ - c_i_x^-) if s = 1 and
1803 * (0, -c_i_x^+ + c_i_x^-) if s = -1 or
1804 * (0, 0, c_i_x^+ - c_i_x^-) if s = 1 and
1805 * (0, 0, -c_i_x^+ + c_i_x^-) if s = -1.
1806 * In graph->lp, the c_i_x^- appear before their c_i_x^+ counterpart.
1807 * Furthermore, the order of these pairs is the opposite of that
1808 * of the corresponding coefficients.
1809 *
1810 * The caller can extend the mapping to also map the other coefficients
1811 * (and therefore not plug in 0).
1812 */
1813static __isl_give isl_dim_map *intra_dim_map(isl_ctx *ctx,
1814 struct isl_sched_graph *graph, struct isl_sched_node *node,
1815 int offset, int s)
1816{
1817 int pos;
1818 unsigned total;
1819 isl_dim_map *dim_map;
1820
1821 if (!node || !graph->lp)
1822 return NULL((void*)0);
1823
1824 total = isl_basic_set_total_dim(graph->lp);
1825 pos = node_var_coef_pos(node, 0);
1826 dim_map = isl_dim_map_alloc(ctx, total);
1827 isl_dim_map_range(dim_map, pos, -2, offset, 1, node->nvar, -s);
1828 isl_dim_map_range(dim_map, pos + 1, -2, offset, 1, node->nvar, s);
1829
1830 return dim_map;
1831}
1832
1833/* Construct an isl_dim_map for mapping constraints on coefficients
1834 * for "src" (node i) and "dst" (node j) to the corresponding positions
1835 * in graph->lp.
1836 * "offset" is the offset of the coefficients for the variables of "src"
1837 * in the input constraints.
1838 * "s" is the sign of the mapping.
1839 *
1840 * The input constraints are given in terms of the coefficients
1841 * (c_0, c_n, c_x, c_y).
1842 * The mapping produced by this function essentially plugs in
1843 * (c_j_0 - c_i_0, c_j_n - c_i_n,
1844 * -(c_i_x^+ - c_i_x^-), c_j_x^+ - c_j_x^-) if s = 1 and
1845 * (-c_j_0 + c_i_0, -c_j_n + c_i_n,
1846 * c_i_x^+ - c_i_x^-, -(c_j_x^+ - c_j_x^-)) if s = -1.
1847 * In graph->lp, the c_*^- appear before their c_*^+ counterpart.
1848 * Furthermore, the order of these pairs is the opposite of that
1849 * of the corresponding coefficients.
1850 *
1851 * The caller can further extend the mapping.
1852 */
1853static __isl_give isl_dim_map *inter_dim_map(isl_ctx *ctx,
1854 struct isl_sched_graph *graph, struct isl_sched_node *src,
1855 struct isl_sched_node *dst, int offset, int s)
1856{
1857 int pos;
1858 unsigned total;
1859 isl_dim_map *dim_map;
1860
1861 if (!src || !dst || !graph->lp)
1862 return NULL((void*)0);
1863
1864 total = isl_basic_set_total_dim(graph->lp);
1865 dim_map = isl_dim_map_alloc(ctx, total);
1866
1867 pos = node_cst_coef_offset(dst);
1868 isl_dim_map_range(dim_map, pos, 0, 0, 0, 1, s);
1869 pos = node_par_coef_offset(dst);
1870 isl_dim_map_range(dim_map, pos, 1, 1, 1, dst->nparam, s);
1871 pos = node_var_coef_pos(dst, 0);
1872 isl_dim_map_range(dim_map, pos, -2, offset + src->nvar, 1,
1873 dst->nvar, -s);
1874 isl_dim_map_range(dim_map, pos + 1, -2, offset + src->nvar, 1,
1875 dst->nvar, s);
1876
1877 pos = node_cst_coef_offset(src);
1878 isl_dim_map_range(dim_map, pos, 0, 0, 0, 1, -s);
1879 pos = node_par_coef_offset(src);
1880 isl_dim_map_range(dim_map, pos, 1, 1, 1, src->nparam, -s);
1881 pos = node_var_coef_pos(src, 0);
1882 isl_dim_map_range(dim_map, pos, -2, offset, 1, src->nvar, s);
1883 isl_dim_map_range(dim_map, pos + 1, -2, offset, 1, src->nvar, -s);
1884
1885 return dim_map;
1886}
1887
1888/* Add the constraints from "src" to "dst" using "dim_map",
1889 * after making sure there is enough room in "dst" for the extra constraints.
1890 */
1891static __isl_give isl_basic_setisl_basic_map *add_constraints_dim_map(
1892 __isl_take isl_basic_setisl_basic_map *dst, __isl_take isl_basic_setisl_basic_map *src,
1893 __isl_take isl_dim_map *dim_map)
1894{
1895 int n_eq, n_ineq;
1896
1897 n_eq = isl_basic_set_n_equality(src);
1898 n_ineq = isl_basic_set_n_inequality(src);
1899 dst = isl_basic_set_extend_constraints(dst, n_eq, n_ineq);
1900 dst = isl_basic_set_add_constraints_dim_map(dst, src, dim_map);
1901 return dst;
1902}
1903
1904/* Add constraints to graph->lp that force validity for the given
1905 * dependence from a node i to itself.
1906 * That is, add constraints that enforce
1907 *
1908 * (c_i_0 + c_i_n n + c_i_x y) - (c_i_0 + c_i_n n + c_i_x x)
1909 * = c_i_x (y - x) >= 0
1910 *
1911 * for each (x,y) in R.
1912 * We obtain general constraints on coefficients (c_0, c_x)
1913 * of valid constraints for (y - x) and then plug in (0, c_i_x^+ - c_i_x^-),
1914 * where c_i_x = c_i_x^+ - c_i_x^-, with c_i_x^+ and c_i_x^- non-negative.
1915 * In graph->lp, the c_i_x^- appear before their c_i_x^+ counterpart.
1916 * Note that the result of intra_coefficients may also contain
1917 * parameter coefficients c_n, in which case 0 is plugged in for them as well.
1918 */
1919static isl_stat add_intra_validity_constraints(struct isl_sched_graph *graph,
1920 struct isl_sched_edge *edge)
1921{
1922 int offset;
1923 isl_map *map = isl_map_copy(edge->map);
1924 isl_ctx *ctx = isl_map_get_ctx(map);
1925 isl_dim_map *dim_map;
1926 isl_basic_setisl_basic_map *coef;
1927 struct isl_sched_node *node = edge->src;
1928
1929 coef = intra_coefficients(graph, node, map, 0);
1930
1931 offset = coef_var_offset(coef);
1932
1933 if (!coef)
1934 return isl_stat_error;
1935
1936 dim_map = intra_dim_map(ctx, graph, node, offset, 1);
1937 graph->lp = add_constraints_dim_map(graph->lp, coef, dim_map);
1938
1939 return isl_stat_ok;
1940}
1941
1942/* Add constraints to graph->lp that force validity for the given
1943 * dependence from node i to node j.
1944 * That is, add constraints that enforce
1945 *
1946 * (c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x) >= 0
1947 *
1948 * for each (x,y) in R.
1949 * We obtain general constraints on coefficients (c_0, c_n, c_x, c_y)
1950 * of valid constraints for R and then plug in
1951 * (c_j_0 - c_i_0, c_j_n - c_i_n, -(c_i_x^+ - c_i_x^-), c_j_x^+ - c_j_x^-),
1952 * where c_* = c_*^+ - c_*^-, with c_*^+ and c_*^- non-negative.
1953 * In graph->lp, the c_*^- appear before their c_*^+ counterpart.
1954 */
1955static isl_stat add_inter_validity_constraints(struct isl_sched_graph *graph,
1956 struct isl_sched_edge *edge)
1957{
1958 int offset;
1959 isl_map *map;
1960 isl_ctx *ctx;
1961 isl_dim_map *dim_map;
1962 isl_basic_setisl_basic_map *coef;
1963 struct isl_sched_node *src = edge->src;
1964 struct isl_sched_node *dst = edge->dst;
1965
1966 if (!graph->lp)
1967 return isl_stat_error;
1968
1969 map = isl_map_copy(edge->map);
1970 ctx = isl_map_get_ctx(map);
1971 coef = inter_coefficients(graph, edge, map);
1972
1973 offset = coef_var_offset(coef);
1974
1975 if (!coef)
1976 return isl_stat_error;
1977
1978 dim_map = inter_dim_map(ctx, graph, src, dst, offset, 1);
1979
1980 edge->start = graph->lp->n_ineq;
1981 graph->lp = add_constraints_dim_map(graph->lp, coef, dim_map);
1982 if (!graph->lp)
1983 return isl_stat_error;
1984 edge->end = graph->lp->n_ineq;
1985
1986 return isl_stat_ok;
1987}
1988
1989/* Add constraints to graph->lp that bound the dependence distance for the given
1990 * dependence from a node i to itself.
1991 * If s = 1, we add the constraint
1992 *
1993 * c_i_x (y - x) <= m_0 + m_n n
1994 *
1995 * or
1996 *
1997 * -c_i_x (y - x) + m_0 + m_n n >= 0
1998 *
1999 * for each (x,y) in R.
2000 * If s = -1, we add the constraint
2001 *
2002 * -c_i_x (y - x) <= m_0 + m_n n
2003 *
2004 * or
2005 *
2006 * c_i_x (y - x) + m_0 + m_n n >= 0
2007 *
2008 * for each (x,y) in R.
2009 * We obtain general constraints on coefficients (c_0, c_n, c_x)
2010 * of valid constraints for (y - x) and then plug in (m_0, m_n, -s * c_i_x),
2011 * with each coefficient (except m_0) represented as a pair of non-negative
2012 * coefficients.
2013 *
2014 *
2015 * If "local" is set, then we add constraints
2016 *
2017 * c_i_x (y - x) <= 0
2018 *
2019 * or
2020 *
2021 * -c_i_x (y - x) <= 0
2022 *
2023 * instead, forcing the dependence distance to be (less than or) equal to 0.
2024 * That is, we plug in (0, 0, -s * c_i_x),
2025 * intra_coefficients is not required to have c_n in its result when
2026 * "local" is set. If they are missing, then (0, -s * c_i_x) is plugged in.
2027 * Note that dependences marked local are treated as validity constraints
2028 * by add_all_validity_constraints and therefore also have
2029 * their distances bounded by 0 from below.
2030 */
2031static isl_stat add_intra_proximity_constraints(struct isl_sched_graph *graph,
2032 struct isl_sched_edge *edge, int s, int local)
2033{
2034 int offset;
2035 unsigned nparam;
2036 isl_map *map = isl_map_copy(edge->map);
2037 isl_ctx *ctx = isl_map_get_ctx(map);
2038 isl_dim_map *dim_map;
2039 isl_basic_setisl_basic_map *coef;
2040 struct isl_sched_node *node = edge->src;
2041
2042 coef = intra_coefficients(graph, node, map, !local);
2043
2044 offset = coef_var_offset(coef);
2045
2046 if (!coef)
2047 return isl_stat_error;
2048
2049 nparam = isl_space_dim(node->space, isl_dim_param);
2050 dim_map = intra_dim_map(ctx, graph, node, offset, -s);
2051
2052 if (!local) {
2053 isl_dim_map_range(dim_map, 1, 0, 0, 0, 1, 1);
2054 isl_dim_map_range(dim_map, 4, 2, 1, 1, nparam, -1);
2055 isl_dim_map_range(dim_map, 5, 2, 1, 1, nparam, 1);
2056 }
2057 graph->lp = add_constraints_dim_map(graph->lp, coef, dim_map);
2058
2059 return isl_stat_ok;
2060}
2061
2062/* Add constraints to graph->lp that bound the dependence distance for the given
2063 * dependence from node i to node j.
2064 * If s = 1, we add the constraint
2065 *
2066 * (c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x)
2067 * <= m_0 + m_n n
2068 *
2069 * or
2070 *
2071 * -(c_j_0 + c_j_n n + c_j_x y) + (c_i_0 + c_i_n n + c_i_x x) +
2072 * m_0 + m_n n >= 0
2073 *
2074 * for each (x,y) in R.
2075 * If s = -1, we add the constraint
2076 *
2077 * -((c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x))
2078 * <= m_0 + m_n n
2079 *
2080 * or
2081 *
2082 * (c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x) +
2083 * m_0 + m_n n >= 0
2084 *
2085 * for each (x,y) in R.
2086 * We obtain general constraints on coefficients (c_0, c_n, c_x, c_y)
2087 * of valid constraints for R and then plug in
2088 * (m_0 - s*c_j_0 + s*c_i_0, m_n - s*c_j_n + s*c_i_n,
2089 * s*c_i_x, -s*c_j_x)
2090 * with each coefficient (except m_0, c_*_0 and c_*_n)
2091 * represented as a pair of non-negative coefficients.
2092 *
2093 *
2094 * If "local" is set (and s = 1), then we add constraints
2095 *
2096 * (c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x) <= 0
2097 *
2098 * or
2099 *
2100 * -((c_j_0 + c_j_n n + c_j_x y) + (c_i_0 + c_i_n n + c_i_x x)) >= 0
2101 *
2102 * instead, forcing the dependence distance to be (less than or) equal to 0.
2103 * That is, we plug in
2104 * (-s*c_j_0 + s*c_i_0, -s*c_j_n + s*c_i_n, s*c_i_x, -s*c_j_x).
2105 * Note that dependences marked local are treated as validity constraints
2106 * by add_all_validity_constraints and therefore also have
2107 * their distances bounded by 0 from below.
2108 */
2109static isl_stat add_inter_proximity_constraints(struct isl_sched_graph *graph,
2110 struct isl_sched_edge *edge, int s, int local)
2111{
2112 int offset;
2113 unsigned nparam;
2114 isl_map *map = isl_map_copy(edge->map);
2115 isl_ctx *ctx = isl_map_get_ctx(map);
2116 isl_dim_map *dim_map;
2117 isl_basic_setisl_basic_map *coef;
2118 struct isl_sched_node *src = edge->src;
2119 struct isl_sched_node *dst = edge->dst;
2120
2121 coef = inter_coefficients(graph, edge, map);
2122
2123 offset = coef_var_offset(coef);
2124
2125 if (!coef)
2126 return isl_stat_error;
2127
2128 nparam = isl_space_dim(src->space, isl_dim_param);
2129 dim_map = inter_dim_map(ctx, graph, src, dst, offset, -s);
2130
2131 if (!local) {
2132 isl_dim_map_range(dim_map, 1, 0, 0, 0, 1, 1);
2133 isl_dim_map_range(dim_map, 4, 2, 1, 1, nparam, -1);
2134 isl_dim_map_range(dim_map, 5, 2, 1, 1, nparam, 1);
2135 }
2136
2137 graph->lp = add_constraints_dim_map(graph->lp, coef, dim_map);
2138
2139 return isl_stat_ok;
2140}
2141
2142/* Should the distance over "edge" be forced to zero?
2143 * That is, is it marked as a local edge?
2144 * If "use_coincidence" is set, then coincidence edges are treated
2145 * as local edges.
2146 */
2147static int force_zero(struct isl_sched_edge *edge, int use_coincidence)
2148{
2149 return is_local(edge) || (use_coincidence && is_coincidence(edge));
2150}
2151
2152/* Add all validity constraints to graph->lp.
2153 *
2154 * An edge that is forced to be local needs to have its dependence
2155 * distances equal to zero. We take care of bounding them by 0 from below
2156 * here. add_all_proximity_constraints takes care of bounding them by 0
2157 * from above.
2158 *
2159 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
2160 * Otherwise, we ignore them.
2161 */
2162static int add_all_validity_constraints(struct isl_sched_graph *graph,
2163 int use_coincidence)
2164{
2165 int i;
2166
2167 for (i = 0; i < graph->n_edge; ++i) {
2168 struct isl_sched_edge *edge = &graph->edge[i];
2169 int zero;
2170
2171 zero = force_zero(edge, use_coincidence);
2172 if (!is_validity(edge) && !zero)
2173 continue;
2174 if (edge->src != edge->dst)
2175 continue;
2176 if (add_intra_validity_constraints(graph, edge) < 0)
2177 return -1;
2178 }
2179
2180 for (i = 0; i < graph->n_edge; ++i) {
2181 struct isl_sched_edge *edge = &graph->edge[i];
2182 int zero;
2183
2184 zero = force_zero(edge, use_coincidence);
2185 if (!is_validity(edge) && !zero)
2186 continue;
2187 if (edge->src == edge->dst)
2188 continue;
2189 if (add_inter_validity_constraints(graph, edge) < 0)
2190 return -1;
2191 }
2192
2193 return 0;
2194}
2195
2196/* Add constraints to graph->lp that bound the dependence distance
2197 * for all dependence relations.
2198 * If a given proximity dependence is identical to a validity
2199 * dependence, then the dependence distance is already bounded
2200 * from below (by zero), so we only need to bound the distance
2201 * from above. (This includes the case of "local" dependences
2202 * which are treated as validity dependence by add_all_validity_constraints.)
2203 * Otherwise, we need to bound the distance both from above and from below.
2204 *
2205 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
2206 * Otherwise, we ignore them.
2207 */
2208static int add_all_proximity_constraints(struct isl_sched_graph *graph,
2209 int use_coincidence)
2210{
2211 int i;
2212
2213 for (i = 0; i < graph->n_edge; ++i) {
2214 struct isl_sched_edge *edge = &graph->edge[i];
2215 int zero;
2216
2217 zero = force_zero(edge, use_coincidence);
2218 if (!is_proximity(edge) && !zero)
2219 continue;
2220 if (edge->src == edge->dst &&
2221 add_intra_proximity_constraints(graph, edge, 1, zero) < 0)
2222 return -1;
2223 if (edge->src != edge->dst &&
2224 add_inter_proximity_constraints(graph, edge, 1, zero) < 0)
2225 return -1;
2226 if (is_validity(edge) || zero)
2227 continue;
2228 if (edge->src == edge->dst &&
2229 add_intra_proximity_constraints(graph, edge, -1, 0) < 0)
2230 return -1;
2231 if (edge->src != edge->dst &&
2232 add_inter_proximity_constraints(graph, edge, -1, 0) < 0)
2233 return -1;
2234 }
2235
2236 return 0;
2237}
2238
2239/* Normalize the rows of "indep" such that all rows are lexicographically
2240 * positive and such that each row contains as many final zeros as possible,
2241 * given the choice for the previous rows.
2242 * Do this by performing elementary row operations.
2243 */
2244static __isl_give isl_mat *normalize_independent(__isl_take isl_mat *indep)
2245{
2246 indep = isl_mat_reverse_gauss(indep);
2247 indep = isl_mat_lexnonneg_rows(indep);
2248 return indep;
2249}
2250
2251/* Compute a basis for the rows in the linear part of the schedule
2252 * and extend this basis to a full basis. The remaining rows
2253 * can then be used to force linear independence from the rows
2254 * in the schedule.
2255 *
2256 * In particular, given the schedule rows S, we compute
2257 *
2258 * S = H Q
2259 * S U = H
2260 *
2261 * with H the Hermite normal form of S. That is, all but the
2262 * first rank columns of H are zero and so each row in S is
2263 * a linear combination of the first rank rows of Q.
2264 * The matrix Q can be used as a variable transformation
2265 * that isolates the directions of S in the first rank rows.
2266 * Transposing S U = H yields
2267 *
2268 * U^T S^T = H^T
2269 *
2270 * with all but the first rank rows of H^T zero.
2271 * The last rows of U^T are therefore linear combinations
2272 * of schedule coefficients that are all zero on schedule
2273 * coefficients that are linearly dependent on the rows of S.
2274 * At least one of these combinations is non-zero on
2275 * linearly independent schedule coefficients.
2276 * The rows are normalized to involve as few of the last
2277 * coefficients as possible and to have a positive initial value.
2278 */
2279static int node_update_vmap(struct isl_sched_node *node)
2280{
2281 isl_mat *H, *U, *Q;
2282 int n_row = isl_mat_rows(node->sched);
2283
2284 H = isl_mat_sub_alloc(node->sched, 0, n_row,
2285 1 + node->nparam, node->nvar);
2286
2287 H = isl_mat_left_hermite(H, 0, &U, &Q);
2288 isl_mat_free(node->indep);
2289 isl_mat_free(node->vmap);
2290 node->vmap = Q;
2291 node->indep = isl_mat_transpose(U);
2292 node->rank = isl_mat_initial_non_zero_cols(H);
2293 node->indep = isl_mat_drop_rows(node->indep, 0, node->rank);
2294 node->indep = normalize_independent(node->indep);
2295 isl_mat_free(H);
2296
2297 if (!node->indep || !node->vmap || node->rank < 0)
2298 return -1;
2299 return 0;
2300}
2301
2302/* Is "edge" marked as a validity or a conditional validity edge?
2303 */
2304static int is_any_validity(struct isl_sched_edge *edge)
2305{
2306 return is_validity(edge) || is_conditional_validity(edge);
2307}
2308
2309/* How many times should we count the constraints in "edge"?
2310 *
2311 * We count as follows
2312 * validity -> 1 (>= 0)
2313 * validity+proximity -> 2 (>= 0 and upper bound)
2314 * proximity -> 2 (lower and upper bound)
2315 * local(+any) -> 2 (>= 0 and <= 0)
2316 *
2317 * If an edge is only marked conditional_validity then it counts
2318 * as zero since it is only checked afterwards.
2319 *
2320 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
2321 * Otherwise, we ignore them.
2322 */
2323static int edge_multiplicity(struct isl_sched_edge *edge, int use_coincidence)
2324{
2325 if (is_proximity(edge) || force_zero(edge, use_coincidence))
2326 return 2;
2327 if (is_validity(edge))
2328 return 1;
2329 return 0;
2330}
2331
2332/* How many times should the constraints in "edge" be counted
2333 * as a parametric intra-node constraint?
2334 *
2335 * Only proximity edges that are not forced zero need
2336 * coefficient constraints that include coefficients for parameters.
2337 * If the edge is also a validity edge, then only
2338 * an upper bound is introduced. Otherwise, both lower and upper bounds
2339 * are introduced.
2340 */
2341static int parametric_intra_edge_multiplicity(struct isl_sched_edge *edge,
2342 int use_coincidence)
2343{
2344 if (edge->src != edge->dst)
2345 return 0;
2346 if (!is_proximity(edge))
2347 return 0;
2348 if (force_zero(edge, use_coincidence))
2349 return 0;
2350 if (is_validity(edge))
2351 return 1;
2352 else
2353 return 2;
2354}
2355
2356/* Add "f" times the number of equality and inequality constraints of "bset"
2357 * to "n_eq" and "n_ineq" and free "bset".
2358 */
2359static isl_stat update_count(__isl_take isl_basic_setisl_basic_map *bset,
2360 int f, int *n_eq, int *n_ineq)
2361{
2362 if (!bset)
2363 return isl_stat_error;
2364
2365 *n_eq += isl_basic_set_n_equality(bset);
2366 *n_ineq += isl_basic_set_n_inequality(bset);
2367 isl_basic_set_free(bset);
2368
2369 return isl_stat_ok;
2370}
2371
2372/* Count the number of equality and inequality constraints
2373 * that will be added for the given map.
2374 *
2375 * The edges that require parameter coefficients are counted separately.
2376 *
2377 * "use_coincidence" is set if we should take into account coincidence edges.
2378 */
2379static isl_stat count_map_constraints(struct isl_sched_graph *graph,
2380 struct isl_sched_edge *edge, __isl_take isl_map *map,
2381 int *n_eq, int *n_ineq, int use_coincidence)
2382{
2383 isl_map *copy;
2384 isl_basic_setisl_basic_map *coef;
2385 int f = edge_multiplicity(edge, use_coincidence);
2386 int fp = parametric_intra_edge_multiplicity(edge, use_coincidence);
2387
2388 if (f == 0) {
2389 isl_map_free(map);
2390 return isl_stat_ok;
2391 }
2392
2393 if (edge->src != edge->dst) {
2394 coef = inter_coefficients(graph, edge, map);
2395 return update_count(coef, f, n_eq, n_ineq);
2396 }
2397
2398 if (fp > 0) {
2399 copy = isl_map_copy(map);
2400 coef = intra_coefficients(graph, edge->src, copy, 1);
2401 if (update_count(coef, fp, n_eq, n_ineq) < 0)
2402 goto error;
2403 }
2404
2405 if (f > fp) {
2406 copy = isl_map_copy(map);
2407 coef = intra_coefficients(graph, edge->src, copy, 0);
2408 if (update_count(coef, f - fp, n_eq, n_ineq) < 0)
2409 goto error;
2410 }
2411
2412 isl_map_free(map);
2413 return isl_stat_ok;
2414error:
2415 isl_map_free(map);
2416 return isl_stat_error;
2417}
2418
2419/* Count the number of equality and inequality constraints
2420 * that will be added to the main lp problem.
2421 * We count as follows
2422 * validity -> 1 (>= 0)
2423 * validity+proximity -> 2 (>= 0 and upper bound)
2424 * proximity -> 2 (lower and upper bound)
2425 * local(+any) -> 2 (>= 0 and <= 0)
2426 *
2427 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
2428 * Otherwise, we ignore them.
2429 */
2430static int count_constraints(struct isl_sched_graph *graph,
2431 int *n_eq, int *n_ineq, int use_coincidence)
2432{
2433 int i;
2434
2435 *n_eq = *n_ineq = 0;
2436 for (i = 0; i < graph->n_edge; ++i) {
2437 struct isl_sched_edge *edge = &graph->edge[i];
2438 isl_map *map = isl_map_copy(edge->map);
2439
2440 if (count_map_constraints(graph, edge, map, n_eq, n_ineq,
2441 use_coincidence) < 0)
2442 return -1;
2443 }
2444
2445 return 0;
2446}
2447
2448/* Count the number of constraints that will be added by
2449 * add_bound_constant_constraints to bound the values of the constant terms
2450 * and increment *n_eq and *n_ineq accordingly.
2451 *
2452 * In practice, add_bound_constant_constraints only adds inequalities.
2453 */
2454static isl_stat count_bound_constant_constraints(isl_ctx *ctx,
2455 struct isl_sched_graph *graph, int *n_eq, int *n_ineq)
2456{
2457 if (isl_options_get_schedule_max_constant_term(ctx) == -1)
2458 return isl_stat_ok;
2459
2460 *n_ineq += graph->n;
2461
2462 return isl_stat_ok;
2463}
2464
2465/* Add constraints to bound the values of the constant terms in the schedule,
2466 * if requested by the user.
2467 *
2468 * The maximal value of the constant terms is defined by the option
2469 * "schedule_max_constant_term".
2470 */
2471static isl_stat add_bound_constant_constraints(isl_ctx *ctx,
2472 struct isl_sched_graph *graph)
2473{
2474 int i, k;
2475 int max;
2476 int total;
2477
2478 max = isl_options_get_schedule_max_constant_term(ctx);
2479 if (max == -1)
2480 return isl_stat_ok;
2481
2482 total = isl_basic_set_dim(graph->lp, isl_dim_set);
2483
2484 for (i = 0; i < graph->n; ++i) {
2485 struct isl_sched_node *node = &graph->node[i];
2486 int pos;
2487
2488 k = isl_basic_set_alloc_inequality(graph->lp);
2489 if (k < 0)
2490 return isl_stat_error;
2491 isl_seq_clr(graph->lp->ineq[k], 1 + total);
2492 pos = node_cst_coef_offset(node);
2493 isl_int_set_si(graph->lp->ineq[k][1 + pos], -1)isl_sioimath_set_si((graph->lp->ineq[k][1 + pos]), -1);
2494 isl_int_set_si(graph->lp->ineq[k][0], max)isl_sioimath_set_si((graph->lp->ineq[k][0]), max);
2495 }
2496
2497 return isl_stat_ok;
2498}
2499
2500/* Count the number of constraints that will be added by
2501 * add_bound_coefficient_constraints and increment *n_eq and *n_ineq
2502 * accordingly.
2503 *
2504 * In practice, add_bound_coefficient_constraints only adds inequalities.
2505 */
2506static int count_bound_coefficient_constraints(isl_ctx *ctx,
2507 struct isl_sched_graph *graph, int *n_eq, int *n_ineq)
2508{
2509 int i;
2510
2511 if (isl_options_get_schedule_max_coefficient(ctx) == -1 &&
2512 !isl_options_get_schedule_treat_coalescing(ctx))
2513 return 0;
2514
2515 for (i = 0; i < graph->n; ++i)
2516 *n_ineq += graph->node[i].nparam + 2 * graph->node[i].nvar;
2517
2518 return 0;
2519}
2520
2521/* Add constraints to graph->lp that bound the values of
2522 * the parameter schedule coefficients of "node" to "max" and
2523 * the variable schedule coefficients to the corresponding entry
2524 * in node->max.
2525 * In either case, a negative value means that no bound needs to be imposed.
2526 *
2527 * For parameter coefficients, this amounts to adding a constraint
2528 *
2529 * c_n <= max
2530 *
2531 * i.e.,
2532 *
2533 * -c_n + max >= 0
2534 *
2535 * The variables coefficients are, however, not represented directly.
2536 * Instead, the variable coefficients c_x are written as differences
2537 * c_x = c_x^+ - c_x^-.
2538 * That is,
2539 *
2540 * -max_i <= c_x_i <= max_i
2541 *
2542 * is encoded as
2543 *
2544 * -max_i <= c_x_i^+ - c_x_i^- <= max_i
2545 *
2546 * or
2547 *
2548 * -(c_x_i^+ - c_x_i^-) + max_i >= 0
2549 * c_x_i^+ - c_x_i^- + max_i >= 0
2550 */
2551static isl_stat node_add_coefficient_constraints(isl_ctx *ctx,
2552 struct isl_sched_graph *graph, struct isl_sched_node *node, int max)
2553{
2554 int i, j, k;
2555 int total;
2556 isl_vec *ineq;
2557
2558 total = isl_basic_set_dim(graph->lp, isl_dim_set);
2559
2560 for (j = 0; j < node->nparam; ++j) {
2561 int dim;
2562
2563 if (max < 0)
2564 continue;
2565
2566 k = isl_basic_set_alloc_inequality(graph->lp);
2567 if (k < 0)
2568 return isl_stat_error;
2569 dim = 1 + node_par_coef_offset(node) + j;
2570 isl_seq_clr(graph->lp->ineq[k], 1 + total);
2571 isl_int_set_si(graph->lp->ineq[k][dim], -1)isl_sioimath_set_si((graph->lp->ineq[k][dim]), -1);
2572 isl_int_set_si(graph->lp->ineq[k][0], max)isl_sioimath_set_si((graph->lp->ineq[k][0]), max);
2573 }
2574
2575 ineq = isl_vec_alloc(ctx, 1 + total);
2576 ineq = isl_vec_clr(ineq);
2577 if (!ineq)
2578 return isl_stat_error;
2579 for (i = 0; i < node->nvar; ++i) {
2580 int pos = 1 + node_var_coef_pos(node, i);
2581
2582 if (isl_int_is_neg(node->max->el[i])(isl_sioimath_sgn(*(node->max->el[i])) < 0))
2583 continue;
2584
2585 isl_int_set_si(ineq->el[pos], 1)isl_sioimath_set_si((ineq->el[pos]), 1);
2586 isl_int_set_si(ineq->el[pos + 1], -1)isl_sioimath_set_si((ineq->el[pos + 1]), -1);
2587 isl_int_set(ineq->el[0], node->max->el[i])isl_sioimath_set((ineq->el[0]), *(node->max->el[i]));
2588
2589 k = isl_basic_set_alloc_inequality(graph->lp);
2590 if (k < 0)
2591 goto error;
2592 isl_seq_cpy(graph->lp->ineq[k], ineq->el, 1 + total);
2593
2594 isl_seq_neg(ineq->el + pos, ineq->el + pos, 2);
2595 k = isl_basic_set_alloc_inequality(graph->lp);
2596 if (k < 0)
2597 goto error;
2598 isl_seq_cpy(graph->lp->ineq[k], ineq->el, 1 + total);
2599
2600 isl_seq_clr(ineq->el + pos, 2);
2601 }
2602 isl_vec_free(ineq);
2603
2604 return isl_stat_ok;
2605error:
2606 isl_vec_free(ineq);
2607 return isl_stat_error;
2608}
2609
2610/* Add constraints that bound the values of the variable and parameter
2611 * coefficients of the schedule.
2612 *
2613 * The maximal value of the coefficients is defined by the option
2614 * 'schedule_max_coefficient' and the entries in node->max.
2615 * These latter entries are only set if either the schedule_max_coefficient
2616 * option or the schedule_treat_coalescing option is set.
2617 */
2618static isl_stat add_bound_coefficient_constraints(isl_ctx *ctx,
2619 struct isl_sched_graph *graph)
2620{
2621 int i;
2622 int max;
2623
2624 max = isl_options_get_schedule_max_coefficient(ctx);
2625
2626 if (max == -1 && !isl_options_get_schedule_treat_coalescing(ctx))
2627 return isl_stat_ok;
2628
2629 for (i = 0; i < graph->n; ++i) {
2630 struct isl_sched_node *node = &graph->node[i];
2631
2632 if (node_add_coefficient_constraints(ctx, graph, node, max) < 0)
2633 return isl_stat_error;
2634 }
2635
2636 return isl_stat_ok;
2637}
2638
2639/* Add a constraint to graph->lp that equates the value at position
2640 * "sum_pos" to the sum of the "n" values starting at "first".
2641 */
2642static isl_stat add_sum_constraint(struct isl_sched_graph *graph,
2643 int sum_pos, int first, int n)
2644{
2645 int i, k;
2646 int total;
2647
2648 total = isl_basic_set_dim(graph->lp, isl_dim_set);
2649
2650 k = isl_basic_set_alloc_equality(graph->lp);
2651 if (k < 0)
2652 return isl_stat_error;
2653 isl_seq_clr(graph->lp->eq[k], 1 + total);
2654 isl_int_set_si(graph->lp->eq[k][1 + sum_pos], -1)isl_sioimath_set_si((graph->lp->eq[k][1 + sum_pos]), -1
)
;
2655 for (i = 0; i < n; ++i)
2656 isl_int_set_si(graph->lp->eq[k][1 + first + i], 1)isl_sioimath_set_si((graph->lp->eq[k][1 + first + i]), 1
)
;
2657
2658 return isl_stat_ok;
2659}
2660
2661/* Add a constraint to graph->lp that equates the value at position
2662 * "sum_pos" to the sum of the parameter coefficients of all nodes.
2663 */
2664static isl_stat add_param_sum_constraint(struct isl_sched_graph *graph,
2665 int sum_pos)
2666{
2667 int i, j, k;
2668 int total;
2669
2670 total = isl_basic_set_dim(graph->lp, isl_dim_set);
2671
2672 k = isl_basic_set_alloc_equality(graph->lp);
2673 if (k < 0)
2674 return isl_stat_error;
2675 isl_seq_clr(graph->lp->eq[k], 1 + total);
2676 isl_int_set_si(graph->lp->eq[k][1 + sum_pos], -1)isl_sioimath_set_si((graph->lp->eq[k][1 + sum_pos]), -1
)
;
2677 for (i = 0; i < graph->n; ++i) {
2678 int pos = 1 + node_par_coef_offset(&graph->node[i]);
2679
2680 for (j = 0; j < graph->node[i].nparam; ++j)
2681 isl_int_set_si(graph->lp->eq[k][pos + j], 1)isl_sioimath_set_si((graph->lp->eq[k][pos + j]), 1);
2682 }
2683
2684 return isl_stat_ok;
2685}
2686
2687/* Add a constraint to graph->lp that equates the value at position
2688 * "sum_pos" to the sum of the variable coefficients of all nodes.
2689 */
2690static isl_stat add_var_sum_constraint(struct isl_sched_graph *graph,
2691 int sum_pos)
2692{
2693 int i, j, k;
2694 int total;
2695
2696 total = isl_basic_set_dim(graph->lp, isl_dim_set);
2697
2698 k = isl_basic_set_alloc_equality(graph->lp);
2699 if (k < 0)
2700 return isl_stat_error;
2701 isl_seq_clr(graph->lp->eq[k], 1 + total);
2702 isl_int_set_si(graph->lp->eq[k][1 + sum_pos], -1)isl_sioimath_set_si((graph->lp->eq[k][1 + sum_pos]), -1
)
;
2703 for (i = 0; i < graph->n; ++i) {
2704 struct isl_sched_node *node = &graph->node[i];
2705 int pos = 1 + node_var_coef_offset(node);
2706
2707 for (j = 0; j < 2 * node->nvar; ++j)
2708 isl_int_set_si(graph->lp->eq[k][pos + j], 1)isl_sioimath_set_si((graph->lp->eq[k][pos + j]), 1);
2709 }
2710
2711 return isl_stat_ok;
2712}
2713
2714/* Construct an ILP problem for finding schedule coefficients
2715 * that result in non-negative, but small dependence distances
2716 * over all dependences.
2717 * In particular, the dependence distances over proximity edges
2718 * are bounded by m_0 + m_n n and we compute schedule coefficients
2719 * with small values (preferably zero) of m_n and m_0.
2720 *
2721 * All variables of the ILP are non-negative. The actual coefficients
2722 * may be negative, so each coefficient is represented as the difference
2723 * of two non-negative variables. The negative part always appears
2724 * immediately before the positive part.
2725 * Other than that, the variables have the following order
2726 *
2727 * - sum of positive and negative parts of m_n coefficients
2728 * - m_0
2729 * - sum of all c_n coefficients
2730 * (unconstrained when computing non-parametric schedules)
2731 * - sum of positive and negative parts of all c_x coefficients
2732 * - positive and negative parts of m_n coefficients
2733 * - for each node
2734 * - positive and negative parts of c_i_x, in opposite order
2735 * - c_i_n (if parametric)
2736 * - c_i_0
2737 *
2738 * The constraints are those from the edges plus two or three equalities
2739 * to express the sums.
2740 *
2741 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
2742 * Otherwise, we ignore them.
2743 */
2744static isl_stat setup_lp(isl_ctx *ctx, struct isl_sched_graph *graph,
2745 int use_coincidence)
2746{
2747 int i;
2748 unsigned nparam;
2749 unsigned total;
2750 isl_space *space;
2751 int parametric;
2752 int param_pos;
2753 int n_eq, n_ineq;
2754
2755 parametric = ctx->opt->schedule_parametric;
2756 nparam = isl_space_dim(graph->node[0].space, isl_dim_param);
2757 param_pos = 4;
2758 total = param_pos + 2 * nparam;
2759 for (i = 0; i < graph->n; ++i) {
2760 struct isl_sched_node *node = &graph->node[graph->sorted[i]];
2761 if (node_update_vmap(node) < 0)
2762 return isl_stat_error;
2763 node->start = total;
2764 total += 1 + node->nparam + 2 * node->nvar;
2765 }
2766
2767 if (count_constraints(graph, &n_eq, &n_ineq, use_coincidence) < 0)
2768 return isl_stat_error;
2769 if (count_bound_constant_constraints(ctx, graph, &n_eq, &n_ineq) < 0)
2770 return isl_stat_error;
2771 if (count_bound_coefficient_constraints(ctx, graph, &n_eq, &n_ineq) < 0)
2772 return isl_stat_error;
2773
2774 space = isl_space_set_alloc(ctx, 0, total);
2775 isl_basic_set_free(graph->lp);
2776 n_eq += 2 + parametric;
2777
2778 graph->lp = isl_basic_set_alloc_space(space, 0, n_eq, n_ineq);
2779
2780 if (add_sum_constraint(graph, 0, param_pos, 2 * nparam) < 0)
2781 return isl_stat_error;
2782 if (parametric && add_param_sum_constraint(graph, 2) < 0)
2783 return isl_stat_error;
2784 if (add_var_sum_constraint(graph, 3) < 0)
2785 return isl_stat_error;
2786 if (add_bound_constant_constraints(ctx, graph) < 0)
2787 return isl_stat_error;
2788 if (add_bound_coefficient_constraints(ctx, graph) < 0)
2789 return isl_stat_error;
2790 if (add_all_validity_constraints(graph, use_coincidence) < 0)
2791 return isl_stat_error;
2792 if (add_all_proximity_constraints(graph, use_coincidence) < 0)
2793 return isl_stat_error;
2794
2795 return isl_stat_ok;
2796}
2797
2798/* Analyze the conflicting constraint found by
2799 * isl_tab_basic_set_non_trivial_lexmin. If it corresponds to the validity
2800 * constraint of one of the edges between distinct nodes, living, moreover
2801 * in distinct SCCs, then record the source and sink SCC as this may
2802 * be a good place to cut between SCCs.
2803 */
2804static int check_conflict(int con, void *user)
2805{
2806 int i;
2807 struct isl_sched_graph *graph = user;
2808
2809 if (graph->src_scc >= 0)
2810 return 0;
2811
2812 con -= graph->lp->n_eq;
2813
2814 if (con >= graph->lp->n_ineq)
2815 return 0;
2816
2817 for (i = 0; i < graph->n_edge; ++i) {
2818 if (!is_validity(&graph->edge[i]))
2819 continue;
2820 if (graph->edge[i].src == graph->edge[i].dst)
2821 continue;
2822 if (graph->edge[i].src->scc == graph->edge[i].dst->scc)
2823 continue;
2824 if (graph->edge[i].start > con)
2825 continue;
2826 if (graph->edge[i].end <= con)
2827 continue;
2828 graph->src_scc = graph->edge[i].src->scc;
2829 graph->dst_scc = graph->edge[i].dst->scc;
2830 }
2831
2832 return 0;
2833}
2834
2835/* Check whether the next schedule row of the given node needs to be
2836 * non-trivial. Lower-dimensional domains may have some trivial rows,
2837 * but as soon as the number of remaining required non-trivial rows
2838 * is as large as the number or remaining rows to be computed,
2839 * all remaining rows need to be non-trivial.
2840 */
2841static int needs_row(struct isl_sched_graph *graph, struct isl_sched_node *node)
2842{
2843 return node->nvar - node->rank >= graph->maxvar - graph->n_row;
2844}
2845
2846/* Construct a non-triviality region with triviality directions
2847 * corresponding to the rows of "indep".
2848 * The rows of "indep" are expressed in terms of the schedule coefficients c_i,
2849 * while the triviality directions are expressed in terms of
2850 * pairs of non-negative variables c^+_i - c^-_i, with c^-_i appearing
2851 * before c^+_i. Furthermore,
2852 * the pairs of non-negative variables representing the coefficients
2853 * are stored in the opposite order.
2854 */
2855static __isl_give isl_mat *construct_trivial(__isl_keep isl_mat *indep)
2856{
2857 isl_ctx *ctx;
2858 isl_mat *mat;
2859 int i, j, n, n_var;
2860
2861 if (!indep)
2862 return NULL((void*)0);
2863
2864 ctx = isl_mat_get_ctx(indep);
2865 n = isl_mat_rows(indep);
2866 n_var = isl_mat_cols(indep);
2867 mat = isl_mat_alloc(ctx, n, 2 * n_var);
2868 if (!mat)
2869 return NULL((void*)0);
2870 for (i = 0; i < n; ++i) {
2871 for (j = 0; j < n_var; ++j) {
2872 int nj = n_var - 1 - j;
2873 isl_int_neg(mat->row[i][2 * nj], indep->row[i][j])isl_sioimath_neg((mat->row[i][2 * nj]), *(indep->row[i]
[j]))
;
2874 isl_int_set(mat->row[i][2 * nj + 1], indep->row[i][j])isl_sioimath_set((mat->row[i][2 * nj + 1]), *(indep->row
[i][j]))
;
2875 }
2876 }
2877
2878 return mat;
2879}
2880
2881/* Solve the ILP problem constructed in setup_lp.
2882 * For each node such that all the remaining rows of its schedule
2883 * need to be non-trivial, we construct a non-triviality region.
2884 * This region imposes that the next row is independent of previous rows.
2885 * In particular, the non-triviality region enforces that at least
2886 * one of the linear combinations in the rows of node->indep is non-zero.
2887 */
2888static __isl_give isl_vec *solve_lp(isl_ctx *ctx, struct isl_sched_graph *graph)
2889{
2890 int i;
2891 isl_vec *sol;
2892 isl_basic_setisl_basic_map *lp;
2893
2894 for (i = 0; i < graph->n; ++i) {
2895 struct isl_sched_node *node = &graph->node[i];
2896 isl_mat *trivial;
2897
2898 graph->region[i].pos = node_var_coef_offset(node);
2899 if (needs_row(graph, node))
2900 trivial = construct_trivial(node->indep);
2901 else
2902 trivial = isl_mat_zero(ctx, 0, 0);
2903 graph->region[i].trivial = trivial;
2904 }
2905 lp = isl_basic_set_copy(graph->lp);
2906 sol = isl_tab_basic_set_non_trivial_lexmin(lp, 2, graph->n,
2907 graph->region, &check_conflict, graph);
2908 for (i = 0; i < graph->n; ++i)
2909 isl_mat_free(graph->region[i].trivial);
2910 return sol;
2911}
2912
2913/* Extract the coefficients for the variables of "node" from "sol".
2914 *
2915 * Each schedule coefficient c_i_x is represented as the difference
2916 * between two non-negative variables c_i_x^+ - c_i_x^-.
2917 * The c_i_x^- appear before their c_i_x^+ counterpart.
2918 * Furthermore, the order of these pairs is the opposite of that
2919 * of the corresponding coefficients.
2920 *
2921 * Return c_i_x = c_i_x^+ - c_i_x^-
2922 */
2923static __isl_give isl_vec *extract_var_coef(struct isl_sched_node *node,
2924 __isl_keep isl_vec *sol)
2925{
2926 int i;
2927 int pos;
2928 isl_vec *csol;
2929
2930 if (!sol)
2931 return NULL((void*)0);
2932 csol = isl_vec_alloc(isl_vec_get_ctx(sol), node->nvar);
2933 if (!csol)
2934 return NULL((void*)0);
2935
2936 pos = 1 + node_var_coef_offset(node);
2937 for (i = 0; i < node->nvar; ++i)
2938 isl_int_sub(csol->el[node->nvar - 1 - i],isl_sioimath_sub((csol->el[node->nvar - 1 - i]), *(sol->
el[pos + 2 * i + 1]), *(sol->el[pos + 2 * i]))
2939 sol->el[pos + 2 * i + 1], sol->el[pos + 2 * i])isl_sioimath_sub((csol->el[node->nvar - 1 - i]), *(sol->
el[pos + 2 * i + 1]), *(sol->el[pos + 2 * i]))
;
2940
2941 return csol;
2942}
2943
2944/* Update the schedules of all nodes based on the given solution
2945 * of the LP problem.
2946 * The new row is added to the current band.
2947 * All possibly negative coefficients are encoded as a difference
2948 * of two non-negative variables, so we need to perform the subtraction
2949 * here.
2950 *
2951 * If coincident is set, then the caller guarantees that the new
2952 * row satisfies the coincidence constraints.
2953 */
2954static int update_schedule(struct isl_sched_graph *graph,
2955 __isl_take isl_vec *sol, int coincident)
2956{
2957 int i, j;
2958 isl_vec *csol = NULL((void*)0);
2959
2960 if (!sol)
2961 goto error;
2962 if (sol->size == 0)
2963 isl_die(sol->ctx, isl_error_internal,do { isl_handle_error(sol->ctx, isl_error_internal, "no solution found"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 2964); goto error; } while (0)
2964 "no solution found", goto error)do { isl_handle_error(sol->ctx, isl_error_internal, "no solution found"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 2964); goto error; } while (0)
;
2965 if (graph->n_total_row >= graph->max_row)
2966 isl_die(sol->ctx, isl_error_internal,do { isl_handle_error(sol->ctx, isl_error_internal, "too many schedule rows"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 2967); goto error; } while (0)
2967 "too many schedule rows", goto error)do { isl_handle_error(sol->ctx, isl_error_internal, "too many schedule rows"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 2967); goto error; } while (0)
;
2968
2969 for (i = 0; i < graph->n; ++i) {
2970 struct isl_sched_node *node = &graph->node[i];
2971 int pos;
2972 int row = isl_mat_rows(node->sched);
2973
2974 isl_vec_free(csol);
2975 csol = extract_var_coef(node, sol);
2976 if (!csol)
2977 goto error;
2978
2979 isl_map_free(node->sched_map);
2980 node->sched_map = NULL((void*)0);
2981 node->sched = isl_mat_add_rows(node->sched, 1);
2982 if (!node->sched)
2983 goto error;
2984 pos = node_cst_coef_offset(node);
2985 node->sched = isl_mat_set_element(node->sched,
2986 row, 0, sol->el[1 + pos]);
2987 pos = node_par_coef_offset(node);
2988 for (j = 0; j < node->nparam; ++j)
2989 node->sched = isl_mat_set_element(node->sched,
2990 row, 1 + j, sol->el[1 + pos + j]);
2991 for (j = 0; j < node->nvar; ++j)
2992 node->sched = isl_mat_set_element(node->sched,
2993 row, 1 + node->nparam + j, csol->el[j]);
2994 node->coincident[graph->n_total_row] = coincident;
2995 }
2996 isl_vec_free(sol);
2997 isl_vec_free(csol);
2998
2999 graph->n_row++;
3000 graph->n_total_row++;
3001
3002 return 0;
3003error:
3004 isl_vec_free(sol);
3005 isl_vec_free(csol);
3006 return -1;
3007}
3008
3009/* Convert row "row" of node->sched into an isl_aff living in "ls"
3010 * and return this isl_aff.
3011 */
3012static __isl_give isl_aff *extract_schedule_row(__isl_take isl_local_space *ls,
3013 struct isl_sched_node *node, int row)
3014{
3015 int j;
3016 isl_int v;
3017 isl_aff *aff;
3018
3019 isl_int_init(v)isl_sioimath_init((v));
3020
3021 aff = isl_aff_zero_on_domain(ls);
3022 if (isl_mat_get_element(node->sched, row, 0, &v) < 0)
3023 goto error;
3024 aff = isl_aff_set_constant(aff, v);
3025 for (j = 0; j < node->nparam; ++j) {
3026 if (isl_mat_get_element(node->sched, row, 1 + j, &v) < 0)
3027 goto error;
3028 aff = isl_aff_set_coefficient(aff, isl_dim_param, j, v);
3029 }
3030 for (j = 0; j < node->nvar; ++j) {
3031 if (isl_mat_get_element(node->sched, row,
3032 1 + node->nparam + j, &v) < 0)
3033 goto error;
3034 aff = isl_aff_set_coefficient(aff, isl_dim_in, j, v);
3035 }
3036
3037 isl_int_clear(v)isl_sioimath_clear((v));
3038
3039 return aff;
3040error:
3041 isl_int_clear(v)isl_sioimath_clear((v));
3042 isl_aff_free(aff);
3043 return NULL((void*)0);
3044}
3045
3046/* Convert the "n" rows starting at "first" of node->sched into a multi_aff
3047 * and return this multi_aff.
3048 *
3049 * The result is defined over the uncompressed node domain.
3050 */
3051static __isl_give isl_multi_aff *node_extract_partial_schedule_multi_aff(
3052 struct isl_sched_node *node, int first, int n)
3053{
3054 int i;
3055 isl_space *space;
3056 isl_local_space *ls;
3057 isl_aff *aff;
3058 isl_multi_aff *ma;
3059 int nrow;
3060
3061 if (!node)
3062 return NULL((void*)0);
3063 nrow = isl_mat_rows(node->sched);
Value stored to 'nrow' is never read
3064 if (node->compressed)
3065 space = isl_multi_aff_get_domain_space(node->decompress);
3066 else
3067 space = isl_space_copy(node->space);
3068 ls = isl_local_space_from_space(isl_space_copy(space));
3069 space = isl_space_from_domain(space);
3070 space = isl_space_add_dims(space, isl_dim_out, n);
3071 ma = isl_multi_aff_zero(space);
3072
3073 for (i = first; i < first + n; ++i) {
3074 aff = extract_schedule_row(isl_local_space_copy(ls), node, i);
3075 ma = isl_multi_aff_set_aff(ma, i - first, aff);
3076 }
3077
3078 isl_local_space_free(ls);
3079
3080 if (node->compressed)
3081 ma = isl_multi_aff_pullback_multi_aff(ma,
3082 isl_multi_aff_copy(node->compress));
3083
3084 return ma;
3085}
3086
3087/* Convert node->sched into a multi_aff and return this multi_aff.
3088 *
3089 * The result is defined over the uncompressed node domain.
3090 */
3091static __isl_give isl_multi_aff *node_extract_schedule_multi_aff(
3092 struct isl_sched_node *node)
3093{
3094 int nrow;
3095
3096 nrow = isl_mat_rows(node->sched);
3097 return node_extract_partial_schedule_multi_aff(node, 0, nrow);
3098}
3099
3100/* Convert node->sched into a map and return this map.
3101 *
3102 * The result is cached in node->sched_map, which needs to be released
3103 * whenever node->sched is updated.
3104 * It is defined over the uncompressed node domain.
3105 */
3106static __isl_give isl_map *node_extract_schedule(struct isl_sched_node *node)
3107{
3108 if (!node->sched_map) {
3109 isl_multi_aff *ma;
3110
3111 ma = node_extract_schedule_multi_aff(node);
3112 node->sched_map = isl_map_from_multi_aff(ma);
3113 }
3114
3115 return isl_map_copy(node->sched_map);
3116}
3117
3118/* Construct a map that can be used to update a dependence relation
3119 * based on the current schedule.
3120 * That is, construct a map expressing that source and sink
3121 * are executed within the same iteration of the current schedule.
3122 * This map can then be intersected with the dependence relation.
3123 * This is not the most efficient way, but this shouldn't be a critical
3124 * operation.
3125 */
3126static __isl_give isl_map *specializer(struct isl_sched_node *src,
3127 struct isl_sched_node *dst)
3128{
3129 isl_map *src_sched, *dst_sched;
3130
3131 src_sched = node_extract_schedule(src);
3132 dst_sched = node_extract_schedule(dst);
3133 return isl_map_apply_range(src_sched, isl_map_reverse(dst_sched));
3134}
3135
3136/* Intersect the domains of the nested relations in domain and range
3137 * of "umap" with "map".
3138 */
3139static __isl_give isl_union_map *intersect_domains(
3140 __isl_take isl_union_map *umap, __isl_keep isl_map *map)
3141{
3142 isl_union_set *uset;
3143
3144 umap = isl_union_map_zip(umap);
3145 uset = isl_union_set_from_set(isl_map_wrap(isl_map_copy(map)));
3146 umap = isl_union_map_intersect_domain(umap, uset);
3147 umap = isl_union_map_zip(umap);
3148 return umap;
3149}
3150
3151/* Update the dependence relation of the given edge based
3152 * on the current schedule.
3153 * If the dependence is carried completely by the current schedule, then
3154 * it is removed from the edge_tables. It is kept in the list of edges
3155 * as otherwise all edge_tables would have to be recomputed.
3156 *
3157 * If the edge is of a type that can appear multiple times
3158 * between the same pair of nodes, then it is added to
3159 * the edge table (again). This prevents the situation
3160 * where none of these edges is referenced from the edge table
3161 * because the one that was referenced turned out to be empty and
3162 * was therefore removed from the table.
3163 */
3164static isl_stat update_edge(isl_ctx *ctx, struct isl_sched_graph *graph,
3165 struct isl_sched_edge *edge)
3166{
3167 int empty;
3168 isl_map *id;
3169
3170 id = specializer(edge->src, edge->dst);
3171 edge->map = isl_map_intersect(edge->map, isl_map_copy(id));
3172 if (!edge->map)
3173 goto error;
3174
3175 if (edge->tagged_condition) {
3176 edge->tagged_condition =
3177 intersect_domains(edge->tagged_condition, id);
3178 if (!edge->tagged_condition)
3179 goto error;
3180 }
3181 if (edge->tagged_validity) {
3182 edge->tagged_validity =
3183 intersect_domains(edge->tagged_validity, id);
3184 if (!edge->tagged_validity)
3185 goto error;
3186 }
3187
3188 empty = isl_map_plain_is_empty(edge->map);
3189 if (empty < 0)
3190 goto error;
3191 if (empty) {
3192 graph_remove_edge(graph, edge);
3193 } else if (is_multi_edge_type(edge)) {
3194 if (graph_edge_tables_add(ctx, graph, edge) < 0)
3195 goto error;
3196 }
3197
3198 isl_map_free(id);
3199 return isl_stat_ok;
3200error:
3201 isl_map_free(id);
3202 return isl_stat_error;
3203}
3204
3205/* Does the domain of "umap" intersect "uset"?
3206 */
3207static int domain_intersects(__isl_keep isl_union_map *umap,
3208 __isl_keep isl_union_set *uset)
3209{
3210 int empty;
3211
3212 umap = isl_union_map_copy(umap);
3213 umap = isl_union_map_intersect_domain(umap, isl_union_set_copy(uset));
3214 empty = isl_union_map_is_empty(umap);
3215 isl_union_map_free(umap);
3216
3217 return empty < 0 ? -1 : !empty;
3218}
3219
3220/* Does the range of "umap" intersect "uset"?
3221 */
3222static int range_intersects(__isl_keep isl_union_map *umap,
3223 __isl_keep isl_union_set *uset)
3224{
3225 int empty;
3226
3227 umap = isl_union_map_copy(umap);
3228 umap = isl_union_map_intersect_range(umap, isl_union_set_copy(uset));
3229 empty = isl_union_map_is_empty(umap);
3230 isl_union_map_free(umap);
3231
3232 return empty < 0 ? -1 : !empty;
3233}
3234
3235/* Are the condition dependences of "edge" local with respect to
3236 * the current schedule?
3237 *
3238 * That is, are domain and range of the condition dependences mapped
3239 * to the same point?
3240 *
3241 * In other words, is the condition false?
3242 */
3243static int is_condition_false(struct isl_sched_edge *edge)
3244{
3245 isl_union_map *umap;
3246 isl_map *map, *sched, *test;
3247 int empty, local;
3248
3249 empty = isl_union_map_is_empty(edge->tagged_condition);
3250 if (empty < 0 || empty)
3251 return empty;
3252
3253 umap = isl_union_map_copy(edge->tagged_condition);
3254 umap = isl_union_map_zip(umap);
3255 umap = isl_union_set_unwrap(isl_union_map_domain(umap));
3256 map = isl_map_from_union_map(umap);
3257
3258 sched = node_extract_schedule(edge->src);
3259 map = isl_map_apply_domain(map, sched);
3260 sched = node_extract_schedule(edge->dst);
3261 map = isl_map_apply_range(map, sched);
3262
3263 test = isl_map_identity(isl_map_get_space(map));
3264 local = isl_map_is_subset(map, test);
3265 isl_map_free(map);
3266 isl_map_free(test);
3267
3268 return local;
3269}
3270
3271/* For each conditional validity constraint that is adjacent
3272 * to a condition with domain in condition_source or range in condition_sink,
3273 * turn it into an unconditional validity constraint.
3274 */
3275static int unconditionalize_adjacent_validity(struct isl_sched_graph *graph,
3276 __isl_take isl_union_set *condition_source,
3277 __isl_take isl_union_set *condition_sink)
3278{
3279 int i;
3280
3281 condition_source = isl_union_set_coalesce(condition_source);
3282 condition_sink = isl_union_set_coalesce(condition_sink);
3283
3284 for (i = 0; i < graph->n_edge; ++i) {
3285 int adjacent;
3286 isl_union_map *validity;
3287
3288 if (!is_conditional_validity(&graph->edge[i]))
3289 continue;
3290 if (is_validity(&graph->edge[i]))
3291 continue;
3292
3293 validity = graph->edge[i].tagged_validity;
3294 adjacent = domain_intersects(validity, condition_sink);
3295 if (adjacent >= 0 && !adjacent)
3296 adjacent = range_intersects(validity, condition_source);
3297 if (adjacent < 0)
3298 goto error;
3299 if (!adjacent)
3300 continue;
3301
3302 set_validity(&graph->edge[i]);
3303 }
3304
3305 isl_union_set_free(condition_source);
3306 isl_union_set_free(condition_sink);
3307 return 0;
3308error:
3309 isl_union_set_free(condition_source);
3310 isl_union_set_free(condition_sink);
3311 return -1;
3312}
3313
3314/* Update the dependence relations of all edges based on the current schedule
3315 * and enforce conditional validity constraints that are adjacent
3316 * to satisfied condition constraints.
3317 *
3318 * First check if any of the condition constraints are satisfied
3319 * (i.e., not local to the outer schedule) and keep track of
3320 * their domain and range.
3321 * Then update all dependence relations (which removes the non-local
3322 * constraints).
3323 * Finally, if any condition constraints turned out to be satisfied,
3324 * then turn all adjacent conditional validity constraints into
3325 * unconditional validity constraints.
3326 */
3327static int update_edges(isl_ctx *ctx, struct isl_sched_graph *graph)
3328{
3329 int i;
3330 int any = 0;
3331 isl_union_set *source, *sink;
3332
3333 source = isl_union_set_empty(isl_space_params_alloc(ctx, 0));
3334 sink = isl_union_set_empty(isl_space_params_alloc(ctx, 0));
3335 for (i = 0; i < graph->n_edge; ++i) {
3336 int local;
3337 isl_union_set *uset;
3338 isl_union_map *umap;
3339
3340 if (!is_condition(&graph->edge[i]))
3341 continue;
3342 if (is_local(&graph->edge[i]))
3343 continue;
3344 local = is_condition_false(&graph->edge[i]);
3345 if (local < 0)
3346 goto error;
3347 if (local)
3348 continue;
3349
3350 any = 1;
3351
3352 umap = isl_union_map_copy(graph->edge[i].tagged_condition);
3353 uset = isl_union_map_domain(umap);
3354 source = isl_union_set_union(source, uset);
3355
3356 umap = isl_union_map_copy(graph->edge[i].tagged_condition);
3357 uset = isl_union_map_range(umap);
3358 sink = isl_union_set_union(sink, uset);
3359 }
3360
3361 for (i = 0; i < graph->n_edge; ++i) {
3362 if (update_edge(ctx, graph, &graph->edge[i]) < 0)
3363 goto error;
3364 }
3365
3366 if (any)
3367 return unconditionalize_adjacent_validity(graph, source, sink);
3368
3369 isl_union_set_free(source);
3370 isl_union_set_free(sink);
3371 return 0;
3372error:
3373 isl_union_set_free(source);
3374 isl_union_set_free(sink);
3375 return -1;
3376}
3377
3378static void next_band(struct isl_sched_graph *graph)
3379{
3380 graph->band_start = graph->n_total_row;
3381}
3382
3383/* Return the union of the universe domains of the nodes in "graph"
3384 * that satisfy "pred".
3385 */
3386static __isl_give isl_union_set *isl_sched_graph_domain(isl_ctx *ctx,
3387 struct isl_sched_graph *graph,
3388 int (*pred)(struct isl_sched_node *node, int data), int data)
3389{
3390 int i;
3391 isl_setisl_map *set;
3392 isl_union_set *dom;
3393
3394 for (i = 0; i < graph->n; ++i)
3395 if (pred(&graph->node[i], data))
3396 break;
3397
3398 if (i >= graph->n)
3399 isl_die(ctx, isl_error_internal,do { isl_handle_error(ctx, isl_error_internal, "empty component"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 3400); return ((void*)0); } while (0)
3400 "empty component", return NULL)do { isl_handle_error(ctx, isl_error_internal, "empty component"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 3400); return ((void*)0); } while (0)
;
3401
3402 set = isl_set_universe(isl_space_copy(graph->node[i].space));
3403 dom = isl_union_set_from_set(set);
3404
3405 for (i = i + 1; i < graph->n; ++i) {
3406 if (!pred(&graph->node[i], data))
3407 continue;
3408 set = isl_set_universe(isl_space_copy(graph->node[i].space));
3409 dom = isl_union_set_union(dom, isl_union_set_from_set(set));
3410 }
3411
3412 return dom;
3413}
3414
3415/* Return a list of unions of universe domains, where each element
3416 * in the list corresponds to an SCC (or WCC) indexed by node->scc.
3417 */
3418static __isl_give isl_union_set_list *extract_sccs(isl_ctx *ctx,
3419 struct isl_sched_graph *graph)
3420{
3421 int i;
3422 isl_union_set_list *filters;
3423
3424 filters = isl_union_set_list_alloc(ctx, graph->scc);
3425 for (i = 0; i < graph->scc; ++i) {
3426 isl_union_set *dom;
3427
3428 dom = isl_sched_graph_domain(ctx, graph, &node_scc_exactly, i);
3429 filters = isl_union_set_list_add(filters, dom);
3430 }
3431
3432 return filters;
3433}
3434
3435/* Return a list of two unions of universe domains, one for the SCCs up
3436 * to and including graph->src_scc and another for the other SCCs.
3437 */
3438static __isl_give isl_union_set_list *extract_split(isl_ctx *ctx,
3439 struct isl_sched_graph *graph)
3440{
3441 isl_union_set *dom;
3442 isl_union_set_list *filters;
3443
3444 filters = isl_union_set_list_alloc(ctx, 2);
3445 dom = isl_sched_graph_domain(ctx, graph,
3446 &node_scc_at_most, graph->src_scc);
3447 filters = isl_union_set_list_add(filters, dom);
3448 dom = isl_sched_graph_domain(ctx, graph,
3449 &node_scc_at_least, graph->src_scc + 1);
3450 filters = isl_union_set_list_add(filters, dom);
3451
3452 return filters;
3453}
3454
3455/* Copy nodes that satisfy node_pred from the src dependence graph
3456 * to the dst dependence graph.
3457 */
3458static isl_stat copy_nodes(struct isl_sched_graph *dst,
3459 struct isl_sched_graph *src,
3460 int (*node_pred)(struct isl_sched_node *node, int data), int data)
3461{
3462 int i;
3463
3464 dst->n = 0;
3465 for (i = 0; i < src->n; ++i) {
3466 int j;
3467
3468 if (!node_pred(&src->node[i], data))
3469 continue;
3470
3471 j = dst->n;
3472 dst->node[j].space = isl_space_copy(src->node[i].space);
3473 dst->node[j].compressed = src->node[i].compressed;
3474 dst->node[j].hull = isl_set_copy(src->node[i].hull);
3475 dst->node[j].compress =
3476 isl_multi_aff_copy(src->node[i].compress);
3477 dst->node[j].decompress =
3478 isl_multi_aff_copy(src->node[i].decompress);
3479 dst->node[j].nvar = src->node[i].nvar;
3480 dst->node[j].nparam = src->node[i].nparam;
3481 dst->node[j].sched = isl_mat_copy(src->node[i].sched);
3482 dst->node[j].sched_map = isl_map_copy(src->node[i].sched_map);
3483 dst->node[j].coincident = src->node[i].coincident;
3484 dst->node[j].sizes = isl_multi_val_copy(src->node[i].sizes);
3485 dst->node[j].bounds = isl_basic_set_copy(src->node[i].bounds);
3486 dst->node[j].max = isl_vec_copy(src->node[i].max);
3487 dst->n++;
3488
3489 if (!dst->node[j].space || !dst->node[j].sched)
3490 return isl_stat_error;
3491 if (dst->node[j].compressed &&
3492 (!dst->node[j].hull || !dst->node[j].compress ||
3493 !dst->node[j].decompress))
3494 return isl_stat_error;
3495 }
3496
3497 return isl_stat_ok;
3498}
3499
3500/* Copy non-empty edges that satisfy edge_pred from the src dependence graph
3501 * to the dst dependence graph.
3502 * If the source or destination node of the edge is not in the destination
3503 * graph, then it must be a backward proximity edge and it should simply
3504 * be ignored.
3505 */
3506static isl_stat copy_edges(isl_ctx *ctx, struct isl_sched_graph *dst,
3507 struct isl_sched_graph *src,
3508 int (*edge_pred)(struct isl_sched_edge *edge, int data), int data)
3509{
3510 int i;
3511
3512 dst->n_edge = 0;
3513 for (i = 0; i < src->n_edge; ++i) {
3514 struct isl_sched_edge *edge = &src->edge[i];
3515 isl_map *map;
3516 isl_union_map *tagged_condition;
3517 isl_union_map *tagged_validity;
3518 struct isl_sched_node *dst_src, *dst_dst;
3519
3520 if (!edge_pred(edge, data))
3521 continue;
3522
3523 if (isl_map_plain_is_empty(edge->map))
3524 continue;
3525
3526 dst_src = graph_find_node(ctx, dst, edge->src->space);
3527 dst_dst = graph_find_node(ctx, dst, edge->dst->space);
3528 if (!dst_src || !dst_dst)
3529 return isl_stat_error;
3530 if (!is_node(dst, dst_src) || !is_node(dst, dst_dst)) {
3531 if (is_validity(edge) || is_conditional_validity(edge))
3532 isl_die(ctx, isl_error_internal,do { isl_handle_error(ctx, isl_error_internal, "backward (conditional) validity edge"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 3534); return isl_stat_error; } while (0)
3533 "backward (conditional) validity edge",do { isl_handle_error(ctx, isl_error_internal, "backward (conditional) validity edge"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 3534); return isl_stat_error; } while (0)
3534 return isl_stat_error)do { isl_handle_error(ctx, isl_error_internal, "backward (conditional) validity edge"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 3534); return isl_stat_error; } while (0)
;
3535 continue;
3536 }
3537
3538 map = isl_map_copy(edge->map);
3539 tagged_condition = isl_union_map_copy(edge->tagged_condition);
3540 tagged_validity = isl_union_map_copy(edge->tagged_validity);
3541
3542 dst->edge[dst->n_edge].src = dst_src;
3543 dst->edge[dst->n_edge].dst = dst_dst;
3544 dst->edge[dst->n_edge].map = map;
3545 dst->edge[dst->n_edge].tagged_condition = tagged_condition;
3546 dst->edge[dst->n_edge].tagged_validity = tagged_validity;
3547 dst->edge[dst->n_edge].types = edge->types;
3548 dst->n_edge++;
3549
3550 if (edge->tagged_condition && !tagged_condition)
3551 return isl_stat_error;
3552 if (edge->tagged_validity && !tagged_validity)
3553 return isl_stat_error;
3554
3555 if (graph_edge_tables_add(ctx, dst,
3556 &dst->edge[dst->n_edge - 1]) < 0)
3557 return isl_stat_error;
3558 }
3559
3560 return isl_stat_ok;
3561}
3562
3563/* Compute the maximal number of variables over all nodes.
3564 * This is the maximal number of linearly independent schedule
3565 * rows that we need to compute.
3566 * Just in case we end up in a part of the dependence graph
3567 * with only lower-dimensional domains, we make sure we will
3568 * compute the required amount of extra linearly independent rows.
3569 */
3570static int compute_maxvar(struct isl_sched_graph *graph)
3571{
3572 int i;
3573
3574 graph->maxvar = 0;
3575 for (i = 0; i < graph->n; ++i) {
3576 struct isl_sched_node *node = &graph->node[i];
3577 int nvar;
3578
3579 if (node_update_vmap(node) < 0)
3580 return -1;
3581 nvar = node->nvar + graph->n_row - node->rank;
3582 if (nvar > graph->maxvar)
3583 graph->maxvar = nvar;
3584 }
3585
3586 return 0;
3587}
3588
3589/* Extract the subgraph of "graph" that consists of the nodes satisfying
3590 * "node_pred" and the edges satisfying "edge_pred" and store
3591 * the result in "sub".
3592 */
3593static isl_stat extract_sub_graph(isl_ctx *ctx, struct isl_sched_graph *graph,
3594 int (*node_pred)(struct isl_sched_node *node, int data),
3595 int (*edge_pred)(struct isl_sched_edge *edge, int data),
3596 int data, struct isl_sched_graph *sub)
3597{
3598 int i, n = 0, n_edge = 0;
3599 int t;
3600
3601 for (i = 0; i < graph->n; ++i)
3602 if (node_pred(&graph->node[i], data))
3603 ++n;
3604 for (i = 0; i < graph->n_edge; ++i)
3605 if (edge_pred(&graph->edge[i], data))
3606 ++n_edge;
3607 if (graph_alloc(ctx, sub, n, n_edge) < 0)
3608 return isl_stat_error;
3609 sub->root = graph->root;
3610 if (copy_nodes(sub, graph, node_pred, data) < 0)
3611 return isl_stat_error;
3612 if (graph_init_table(ctx, sub) < 0)
3613 return isl_stat_error;
3614 for (t = 0; t <= isl_edge_last; ++t)
3615 sub->max_edge[t] = graph->max_edge[t];
3616 if (graph_init_edge_tables(ctx, sub) < 0)
3617 return isl_stat_error;
3618 if (copy_edges(ctx, sub, graph, edge_pred, data) < 0)
3619 return isl_stat_error;
3620 sub->n_row = graph->n_row;
3621 sub->max_row = graph->max_row;
3622 sub->n_total_row = graph->n_total_row;
3623 sub->band_start = graph->band_start;
3624
3625 return isl_stat_ok;
3626}
3627
3628static __isl_give isl_schedule_node *compute_schedule(isl_schedule_node *node,
3629 struct isl_sched_graph *graph);
3630static __isl_give isl_schedule_node *compute_schedule_wcc(
3631 isl_schedule_node *node, struct isl_sched_graph *graph);
3632
3633/* Compute a schedule for a subgraph of "graph". In particular, for
3634 * the graph composed of nodes that satisfy node_pred and edges that
3635 * that satisfy edge_pred.
3636 * If the subgraph is known to consist of a single component, then wcc should
3637 * be set and then we call compute_schedule_wcc on the constructed subgraph.
3638 * Otherwise, we call compute_schedule, which will check whether the subgraph
3639 * is connected.
3640 *
3641 * The schedule is inserted at "node" and the updated schedule node
3642 * is returned.
3643 */
3644static __isl_give isl_schedule_node *compute_sub_schedule(
3645 __isl_take isl_schedule_node *node, isl_ctx *ctx,
3646 struct isl_sched_graph *graph,
3647 int (*node_pred)(struct isl_sched_node *node, int data),
3648 int (*edge_pred)(struct isl_sched_edge *edge, int data),
3649 int data, int wcc)
3650{
3651 struct isl_sched_graph split = { 0 };
3652
3653 if (extract_sub_graph(ctx, graph, node_pred, edge_pred, data,
3654 &split) < 0)
3655 goto error;
3656
3657 if (wcc)
3658 node = compute_schedule_wcc(node, &split);
3659 else
3660 node = compute_schedule(node, &split);
3661
3662 graph_free(ctx, &split);
3663 return node;
3664error:
3665 graph_free(ctx, &split);
3666 return isl_schedule_node_free(node);
3667}
3668
3669static int edge_scc_exactly(struct isl_sched_edge *edge, int scc)
3670{
3671 return edge->src->scc == scc && edge->dst->scc == scc;
3672}
3673
3674static int edge_dst_scc_at_most(struct isl_sched_edge *edge, int scc)
3675{
3676 return edge->dst->scc <= scc;
3677}
3678
3679static int edge_src_scc_at_least(struct isl_sched_edge *edge, int scc)
3680{
3681 return edge->src->scc >= scc;
3682}
3683
3684/* Reset the current band by dropping all its schedule rows.
3685 */
3686static isl_stat reset_band(struct isl_sched_graph *graph)
3687{
3688 int i;
3689 int drop;
3690
3691 drop = graph->n_total_row - graph->band_start;
3692 graph->n_total_row -= drop;
3693 graph->n_row -= drop;
3694
3695 for (i = 0; i < graph->n; ++i) {
3696 struct isl_sched_node *node = &graph->node[i];
3697
3698 isl_map_free(node->sched_map);
3699 node->sched_map = NULL((void*)0);
3700
3701 node->sched = isl_mat_drop_rows(node->sched,
3702 graph->band_start, drop);
3703
3704 if (!node->sched)
3705 return isl_stat_error;
3706 }
3707
3708 return isl_stat_ok;
3709}
3710
3711/* Split the current graph into two parts and compute a schedule for each
3712 * part individually. In particular, one part consists of all SCCs up
3713 * to and including graph->src_scc, while the other part contains the other
3714 * SCCs. The split is enforced by a sequence node inserted at position "node"
3715 * in the schedule tree. Return the updated schedule node.
3716 * If either of these two parts consists of a sequence, then it is spliced
3717 * into the sequence containing the two parts.
3718 *
3719 * The current band is reset. It would be possible to reuse
3720 * the previously computed rows as the first rows in the next
3721 * band, but recomputing them may result in better rows as we are looking
3722 * at a smaller part of the dependence graph.
3723 */
3724static __isl_give isl_schedule_node *compute_split_schedule(
3725 __isl_take isl_schedule_node *node, struct isl_sched_graph *graph)
3726{
3727 int is_seq;
3728 isl_ctx *ctx;
3729 isl_union_set_list *filters;
3730
3731 if (!node)
3732 return NULL((void*)0);
3733
3734 if (reset_band(graph) < 0)
3735 return isl_schedule_node_free(node);
3736
3737 next_band(graph);
3738
3739 ctx = isl_schedule_node_get_ctx(node);
3740 filters = extract_split(ctx, graph);
3741 node = isl_schedule_node_insert_sequence(node, filters);
3742 node = isl_schedule_node_child(node, 1);
3743 node = isl_schedule_node_child(node, 0);
3744
3745 node = compute_sub_schedule(node, ctx, graph,
3746 &node_scc_at_least, &edge_src_scc_at_least,
3747 graph->src_scc + 1, 0);
3748 is_seq = isl_schedule_node_get_type(node) == isl_schedule_node_sequence;
3749 node = isl_schedule_node_parent(node);
3750 node = isl_schedule_node_parent(node);
3751 if (is_seq)
3752 node = isl_schedule_node_sequence_splice_child(node, 1);
3753 node = isl_schedule_node_child(node, 0);
3754 node = isl_schedule_node_child(node, 0);
3755 node = compute_sub_schedule(node, ctx, graph,
3756 &node_scc_at_most, &edge_dst_scc_at_most,
3757 graph->src_scc, 0);
3758 is_seq = isl_schedule_node_get_type(node) == isl_schedule_node_sequence;
3759 node = isl_schedule_node_parent(node);
3760 node = isl_schedule_node_parent(node);
3761 if (is_seq)
3762 node = isl_schedule_node_sequence_splice_child(node, 0);
3763
3764 return node;
3765}
3766
3767/* Insert a band node at position "node" in the schedule tree corresponding
3768 * to the current band in "graph". Mark the band node permutable
3769 * if "permutable" is set.
3770 * The partial schedules and the coincidence property are extracted
3771 * from the graph nodes.
3772 * Return the updated schedule node.
3773 */
3774static __isl_give isl_schedule_node *insert_current_band(
3775 __isl_take isl_schedule_node *node, struct isl_sched_graph *graph,
3776 int permutable)
3777{
3778 int i;
3779 int start, end, n;
3780 isl_multi_aff *ma;
3781 isl_multi_pw_aff *mpa;
3782 isl_multi_union_pw_aff *mupa;
3783
3784 if (!node)
3785 return NULL((void*)0);
3786
3787 if (graph->n < 1)
3788 isl_die(isl_schedule_node_get_ctx(node), isl_error_internal,do { isl_handle_error(isl_schedule_node_get_ctx(node), isl_error_internal
, "graph should have at least one node", "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 3790); return isl_schedule_node_free(node); } while (0)
3789 "graph should have at least one node",do { isl_handle_error(isl_schedule_node_get_ctx(node), isl_error_internal
, "graph should have at least one node", "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 3790); return isl_schedule_node_free(node); } while (0)
3790 return isl_schedule_node_free(node))do { isl_handle_error(isl_schedule_node_get_ctx(node), isl_error_internal
, "graph should have at least one node", "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 3790); return isl_schedule_node_free(node); } while (0)
;
3791
3792 start = graph->band_start;
3793 end = graph->n_total_row;
3794 n = end - start;
3795
3796 ma = node_extract_partial_schedule_multi_aff(&graph->node[0], start, n);
3797 mpa = isl_multi_pw_aff_from_multi_aff(ma);
3798 mupa = isl_multi_union_pw_aff_from_multi_pw_aff(mpa);
3799
3800 for (i = 1; i < graph->n; ++i) {
3801 isl_multi_union_pw_aff *mupa_i;
3802
3803 ma = node_extract_partial_schedule_multi_aff(&graph->node[i],
3804 start, n);
3805 mpa = isl_multi_pw_aff_from_multi_aff(ma);
3806 mupa_i = isl_multi_union_pw_aff_from_multi_pw_aff(mpa);
3807 mupa = isl_multi_union_pw_aff_union_add(mupa, mupa_i);
3808 }
3809 node = isl_schedule_node_insert_partial_schedule(node, mupa);
3810
3811 for (i = 0; i < n; ++i)
3812 node = isl_schedule_node_band_member_set_coincident(node, i,
3813 graph->node[0].coincident[start + i]);
3814 node = isl_schedule_node_band_set_permutable(node, permutable);
3815
3816 return node;
3817}
3818
3819/* Update the dependence relations based on the current schedule,
3820 * add the current band to "node" and then continue with the computation
3821 * of the next band.
3822 * Return the updated schedule node.
3823 */
3824static __isl_give isl_schedule_node *compute_next_band(
3825 __isl_take isl_schedule_node *node,
3826 struct isl_sched_graph *graph, int permutable)
3827{
3828 isl_ctx *ctx;
3829
3830 if (!node)
3831 return NULL((void*)0);
3832
3833 ctx = isl_schedule_node_get_ctx(node);
3834 if (update_edges(ctx, graph) < 0)
3835 return isl_schedule_node_free(node);
3836 node = insert_current_band(node, graph, permutable);
3837 next_band(graph);
3838
3839 node = isl_schedule_node_child(node, 0);
3840 node = compute_schedule(node, graph);
3841 node = isl_schedule_node_parent(node);
3842
3843 return node;
3844}
3845
3846/* Add the constraints "coef" derived from an edge from "node" to itself
3847 * to graph->lp in order to respect the dependences and to try and carry them.
3848 * "pos" is the sequence number of the edge that needs to be carried.
3849 * "coef" represents general constraints on coefficients (c_0, c_x)
3850 * of valid constraints for (y - x) with x and y instances of the node.
3851 *
3852 * The constraints added to graph->lp need to enforce
3853 *
3854 * (c_j_0 + c_j_x y) - (c_j_0 + c_j_x x)
3855 * = c_j_x (y - x) >= e_i
3856 *
3857 * for each (x,y) in the dependence relation of the edge.
3858 * That is, (-e_i, c_j_x) needs to be plugged in for (c_0, c_x),
3859 * taking into account that each coefficient in c_j_x is represented
3860 * as a pair of non-negative coefficients.
3861 */
3862static isl_stat add_intra_constraints(struct isl_sched_graph *graph,
3863 struct isl_sched_node *node, __isl_take isl_basic_setisl_basic_map *coef, int pos)
3864{
3865 int offset;
3866 isl_ctx *ctx;
3867 isl_dim_map *dim_map;
3868
3869 if (!coef)
3870 return isl_stat_error;
3871
3872 ctx = isl_basic_set_get_ctx(coef);
3873 offset = coef_var_offset(coef);
3874 dim_map = intra_dim_map(ctx, graph, node, offset, 1);
3875 isl_dim_map_range(dim_map, 3 + pos, 0, 0, 0, 1, -1);
3876 graph->lp = add_constraints_dim_map(graph->lp, coef, dim_map);
3877
3878 return isl_stat_ok;
3879}
3880
3881/* Add the constraints "coef" derived from an edge from "src" to "dst"
3882 * to graph->lp in order to respect the dependences and to try and carry them.
3883 * "pos" is the sequence number of the edge that needs to be carried or
3884 * -1 if no attempt should be made to carry the dependences.
3885 * "coef" represents general constraints on coefficients (c_0, c_n, c_x, c_y)
3886 * of valid constraints for (x, y) with x and y instances of "src" and "dst".
3887 *
3888 * The constraints added to graph->lp need to enforce
3889 *
3890 * (c_k_0 + c_k_n n + c_k_x y) - (c_j_0 + c_j_n n + c_j_x x) >= e_i
3891 *
3892 * for each (x,y) in the dependence relation of the edge or
3893 *
3894 * (c_k_0 + c_k_n n + c_k_x y) - (c_j_0 + c_j_n n + c_j_x x) >= 0
3895 *
3896 * if pos is -1.
3897 * That is,
3898 * (-e_i + c_k_0 - c_j_0, c_k_n - c_j_n, -c_j_x, c_k_x)
3899 * or
3900 * (c_k_0 - c_j_0, c_k_n - c_j_n, -c_j_x, c_k_x)
3901 * needs to be plugged in for (c_0, c_n, c_x, c_y),
3902 * taking into account that each coefficient in c_j_x and c_k_x is represented
3903 * as a pair of non-negative coefficients.
3904 */
3905static isl_stat add_inter_constraints(struct isl_sched_graph *graph,
3906 struct isl_sched_node *src, struct isl_sched_node *dst,
3907 __isl_take isl_basic_setisl_basic_map *coef, int pos)
3908{
3909 int offset;
3910 isl_ctx *ctx;
3911 isl_dim_map *dim_map;
3912
3913 if (!coef)
3914 return isl_stat_error;
3915
3916 ctx = isl_basic_set_get_ctx(coef);
3917 offset = coef_var_offset(coef);
3918 dim_map = inter_dim_map(ctx, graph, src, dst, offset, 1);
3919 if (pos >= 0)
3920 isl_dim_map_range(dim_map, 3 + pos, 0, 0, 0, 1, -1);
3921 graph->lp = add_constraints_dim_map(graph->lp, coef, dim_map);
3922
3923 return isl_stat_ok;
3924}
3925
3926/* Data structure for keeping track of the data needed
3927 * to exploit non-trivial lineality spaces.
3928 *
3929 * "any_non_trivial" is true if there are any non-trivial lineality spaces.
3930 * If "any_non_trivial" is not true, then "equivalent" and "mask" may be NULL.
3931 * "equivalent" connects instances to other instances on the same line(s).
3932 * "mask" contains the domain spaces of "equivalent".
3933 * Any instance set not in "mask" does not have a non-trivial lineality space.
3934 */
3935struct isl_exploit_lineality_data {
3936 isl_bool any_non_trivial;
3937 isl_union_map *equivalent;
3938 isl_union_set *mask;
3939};
3940
3941/* Data structure collecting information used during the construction
3942 * of an LP for carrying dependences.
3943 *
3944 * "intra" is a sequence of coefficient constraints for intra-node edges.
3945 * "inter" is a sequence of coefficient constraints for inter-node edges.
3946 * "lineality" contains data used to exploit non-trivial lineality spaces.
3947 */
3948struct isl_carry {
3949 isl_basic_set_listisl_basic_map_list *intra;
3950 isl_basic_set_listisl_basic_map_list *inter;
3951 struct isl_exploit_lineality_data lineality;
3952};
3953
3954/* Free all the data stored in "carry".
3955 */
3956static void isl_carry_clear(struct isl_carry *carry)
3957{
3958 isl_basic_set_list_free(carry->intra);
3959 isl_basic_set_list_free(carry->inter);
3960 isl_union_map_free(carry->lineality.equivalent);
3961 isl_union_set_free(carry->lineality.mask);
3962}
3963
3964/* Return a pointer to the node in "graph" that lives in "space".
3965 * If the requested node has been compressed, then "space"
3966 * corresponds to the compressed space.
3967 * The graph is assumed to have such a node.
3968 * Return NULL in case of error.
3969 *
3970 * First try and see if "space" is the space of an uncompressed node.
3971 * If so, return that node.
3972 * Otherwise, "space" was constructed by construct_compressed_id and
3973 * contains a user pointer pointing to the node in the tuple id.
3974 * However, this node belongs to the original dependence graph.
3975 * If "graph" is a subgraph of this original dependence graph,
3976 * then the node with the same space still needs to be looked up
3977 * in the current graph.
3978 */
3979static struct isl_sched_node *graph_find_compressed_node(isl_ctx *ctx,
3980 struct isl_sched_graph *graph, __isl_keep isl_space *space)
3981{
3982 isl_id *id;
3983 struct isl_sched_node *node;
3984
3985 if (!space)
3986 return NULL((void*)0);
3987
3988 node = graph_find_node(ctx, graph, space);
3989 if (!node)
3990 return NULL((void*)0);
3991 if (is_node(graph, node))
3992 return node;
3993
3994 id = isl_space_get_tuple_id(space, isl_dim_set);
3995 node = isl_id_get_user(id);
3996 isl_id_free(id);
3997
3998 if (!node)
3999 return NULL((void*)0);
4000
4001 if (!is_node(graph->root, node))
4002 isl_die(ctx, isl_error_internal,do { isl_handle_error(ctx, isl_error_internal, "space points to invalid node"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 4003); return ((void*)0); } while (0)
4003 "space points to invalid node", return NULL)do { isl_handle_error(ctx, isl_error_internal, "space points to invalid node"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 4003); return ((void*)0); } while (0)
;
4004 if (graph != graph->root)
4005 node = graph_find_node(ctx, graph, node->space);
4006 if (!is_node(graph, node))
4007 isl_die(ctx, isl_error_internal,do { isl_handle_error(ctx, isl_error_internal, "unable to find node"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 4008); return ((void*)0); } while (0)
4008 "unable to find node", return NULL)do { isl_handle_error(ctx, isl_error_internal, "unable to find node"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 4008); return ((void*)0); } while (0)
;
4009
4010 return node;
4011}
4012
4013/* Internal data structure for add_all_constraints.
4014 *
4015 * "graph" is the schedule constraint graph for which an LP problem
4016 * is being constructed.
4017 * "carry_inter" indicates whether inter-node edges should be carried.
4018 * "pos" is the position of the next edge that needs to be carried.
4019 */
4020struct isl_add_all_constraints_data {
4021 isl_ctx *ctx;
4022 struct isl_sched_graph *graph;
4023 int carry_inter;
4024 int pos;
4025};
4026
4027/* Add the constraints "coef" derived from an edge from a node to itself
4028 * to data->graph->lp in order to respect the dependences and
4029 * to try and carry them.
4030 *
4031 * The space of "coef" is of the form
4032 *
4033 * coefficients[[c_cst] -> S[c_x]]
4034 *
4035 * with S[c_x] the (compressed) space of the node.
4036 * Extract the node from the space and call add_intra_constraints.
4037 */
4038static isl_stat lp_add_intra(__isl_take isl_basic_setisl_basic_map *coef, void *user)
4039{
4040 struct isl_add_all_constraints_data *data = user;
4041 isl_space *space;
4042 struct isl_sched_node *node;
4043
4044 space = isl_basic_set_get_space(coef);
4045 space = isl_space_range(isl_space_unwrap(space));
4046 node = graph_find_compressed_node(data->ctx, data->graph, space);
4047 isl_space_free(space);
4048 return add_intra_constraints(data->graph, node, coef, data->pos++);
4049}
4050
4051/* Add the constraints "coef" derived from an edge from a node j
4052 * to a node k to data->graph->lp in order to respect the dependences and
4053 * to try and carry them (provided data->carry_inter is set).
4054 *
4055 * The space of "coef" is of the form
4056 *
4057 * coefficients[[c_cst, c_n] -> [S_j[c_x] -> S_k[c_y]]]
4058 *
4059 * with S_j[c_x] and S_k[c_y] the (compressed) spaces of the nodes.
4060 * Extract the nodes from the space and call add_inter_constraints.
4061 */
4062static isl_stat lp_add_inter(__isl_take isl_basic_setisl_basic_map *coef, void *user)
4063{
4064 struct isl_add_all_constraints_data *data = user;
4065 isl_space *space, *dom;
4066 struct isl_sched_node *src, *dst;
4067 int pos;
4068
4069 space = isl_basic_set_get_space(coef);
4070 space = isl_space_unwrap(isl_space_range(isl_space_unwrap(space)));
4071 dom = isl_space_domain(isl_space_copy(space));
4072 src = graph_find_compressed_node(data->ctx, data->graph, dom);
4073 isl_space_free(dom);
4074 space = isl_space_range(space);
4075 dst = graph_find_compressed_node(data->ctx, data->graph, space);
4076 isl_space_free(space);
4077
4078 pos = data->carry_inter ? data->pos++ : -1;
4079 return add_inter_constraints(data->graph, src, dst, coef, pos);
4080}
4081
4082/* Add constraints to graph->lp that force all (conditional) validity
4083 * dependences to be respected and attempt to carry them.
4084 * "intra" is the sequence of coefficient constraints for intra-node edges.
4085 * "inter" is the sequence of coefficient constraints for inter-node edges.
4086 * "carry_inter" indicates whether inter-node edges should be carried or
4087 * only respected.
4088 */
4089static isl_stat add_all_constraints(isl_ctx *ctx, struct isl_sched_graph *graph,
4090 __isl_keep isl_basic_set_listisl_basic_map_list *intra,
4091 __isl_keep isl_basic_set_listisl_basic_map_list *inter, int carry_inter)
4092{
4093 struct isl_add_all_constraints_data data = { ctx, graph, carry_inter };
4094
4095 data.pos = 0;
4096 if (isl_basic_set_list_foreach(intra, &lp_add_intra, &data) < 0)
4097 return isl_stat_error;
4098 if (isl_basic_set_list_foreach(inter, &lp_add_inter, &data) < 0)
4099 return isl_stat_error;
4100 return isl_stat_ok;
4101}
4102
4103/* Internal data structure for count_all_constraints
4104 * for keeping track of the number of equality and inequality constraints.
4105 */
4106struct isl_sched_count {
4107 int n_eq;
4108 int n_ineq;
4109};
4110
4111/* Add the number of equality and inequality constraints of "bset"
4112 * to data->n_eq and data->n_ineq.
4113 */
4114static isl_stat bset_update_count(__isl_take isl_basic_setisl_basic_map *bset, void *user)
4115{
4116 struct isl_sched_count *data = user;
4117
4118 return update_count(bset, 1, &data->n_eq, &data->n_ineq);
4119}
4120
4121/* Count the number of equality and inequality constraints
4122 * that will be added to the carry_lp problem.
4123 * We count each edge exactly once.
4124 * "intra" is the sequence of coefficient constraints for intra-node edges.
4125 * "inter" is the sequence of coefficient constraints for inter-node edges.
4126 */
4127static isl_stat count_all_constraints(__isl_keep isl_basic_set_listisl_basic_map_list *intra,
4128 __isl_keep isl_basic_set_listisl_basic_map_list *inter, int *n_eq, int *n_ineq)
4129{
4130 struct isl_sched_count data;
4131
4132 data.n_eq = data.n_ineq = 0;
4133 if (isl_basic_set_list_foreach(inter, &bset_update_count, &data) < 0)
4134 return isl_stat_error;
4135 if (isl_basic_set_list_foreach(intra, &bset_update_count, &data) < 0)
4136 return isl_stat_error;
4137
4138 *n_eq = data.n_eq;
4139 *n_ineq = data.n_ineq;
4140
4141 return isl_stat_ok;
4142}
4143
4144/* Construct an LP problem for finding schedule coefficients
4145 * such that the schedule carries as many validity dependences as possible.
4146 * In particular, for each dependence i, we bound the dependence distance
4147 * from below by e_i, with 0 <= e_i <= 1 and then maximize the sum
4148 * of all e_i's. Dependences with e_i = 0 in the solution are simply
4149 * respected, while those with e_i > 0 (in practice e_i = 1) are carried.
4150 * "intra" is the sequence of coefficient constraints for intra-node edges.
4151 * "inter" is the sequence of coefficient constraints for inter-node edges.
4152 * "n_edge" is the total number of edges.
4153 * "carry_inter" indicates whether inter-node edges should be carried or
4154 * only respected. That is, if "carry_inter" is not set, then
4155 * no e_i variables are introduced for the inter-node edges.
4156 *
4157 * All variables of the LP are non-negative. The actual coefficients
4158 * may be negative, so each coefficient is represented as the difference
4159 * of two non-negative variables. The negative part always appears
4160 * immediately before the positive part.
4161 * Other than that, the variables have the following order
4162 *
4163 * - sum of (1 - e_i) over all edges
4164 * - sum of all c_n coefficients
4165 * (unconstrained when computing non-parametric schedules)
4166 * - sum of positive and negative parts of all c_x coefficients
4167 * - for each edge
4168 * - e_i
4169 * - for each node
4170 * - positive and negative parts of c_i_x, in opposite order
4171 * - c_i_n (if parametric)
4172 * - c_i_0
4173 *
4174 * The constraints are those from the (validity) edges plus three equalities
4175 * to express the sums and n_edge inequalities to express e_i <= 1.
4176 */
4177static isl_stat setup_carry_lp(isl_ctx *ctx, struct isl_sched_graph *graph,
4178 int n_edge, __isl_keep isl_basic_set_listisl_basic_map_list *intra,
4179 __isl_keep isl_basic_set_listisl_basic_map_list *inter, int carry_inter)
4180{
4181 int i;
4182 int k;
4183 isl_space *dim;
4184 unsigned total;
4185 int n_eq, n_ineq;
4186
4187 total = 3 + n_edge;
4188 for (i = 0; i < graph->n; ++i) {
4189 struct isl_sched_node *node = &graph->node[graph->sorted[i]];
4190 node->start = total;
4191 total += 1 + node->nparam + 2 * node->nvar;
4192 }
4193
4194 if (count_all_constraints(intra, inter, &n_eq, &n_ineq) < 0)
4195 return isl_stat_error;
4196
4197 dim = isl_space_set_alloc(ctx, 0, total);
4198 isl_basic_set_free(graph->lp);
4199 n_eq += 3;
4200 n_ineq += n_edge;
4201 graph->lp = isl_basic_set_alloc_space(dim, 0, n_eq, n_ineq);
4202 graph->lp = isl_basic_set_set_rational(graph->lp);
4203
4204 k = isl_basic_set_alloc_equality(graph->lp);
4205 if (k < 0)
4206 return isl_stat_error;
4207 isl_seq_clr(graph->lp->eq[k], 1 + total);
4208 isl_int_set_si(graph->lp->eq[k][0], -n_edge)isl_sioimath_set_si((graph->lp->eq[k][0]), -n_edge);
4209 isl_int_set_si(graph->lp->eq[k][1], 1)isl_sioimath_set_si((graph->lp->eq[k][1]), 1);
4210 for (i = 0; i < n_edge; ++i)
4211 isl_int_set_si(graph->lp->eq[k][4 + i], 1)isl_sioimath_set_si((graph->lp->eq[k][4 + i]), 1);
4212
4213 if (add_param_sum_constraint(graph, 1) < 0)
4214 return isl_stat_error;
4215 if (add_var_sum_constraint(graph, 2) < 0)
4216 return isl_stat_error;
4217
4218 for (i = 0; i < n_edge; ++i) {
4219 k = isl_basic_set_alloc_inequality(graph->lp);
4220 if (k < 0)
4221 return isl_stat_error;
4222 isl_seq_clr(graph->lp->ineq[k], 1 + total);
4223 isl_int_set_si(graph->lp->ineq[k][4 + i], -1)isl_sioimath_set_si((graph->lp->ineq[k][4 + i]), -1);
4224 isl_int_set_si(graph->lp->ineq[k][0], 1)isl_sioimath_set_si((graph->lp->ineq[k][0]), 1);
4225 }
4226
4227 if (add_all_constraints(ctx, graph, intra, inter, carry_inter) < 0)
4228 return isl_stat_error;
4229
4230 return isl_stat_ok;
4231}
4232
4233static __isl_give isl_schedule_node *compute_component_schedule(
4234 __isl_take isl_schedule_node *node, struct isl_sched_graph *graph,
4235 int wcc);
4236
4237/* If the schedule_split_scaled option is set and if the linear
4238 * parts of the scheduling rows for all nodes in the graphs have
4239 * a non-trivial common divisor, then remove this
4240 * common divisor from the linear part.
4241 * Otherwise, insert a band node directly and continue with
4242 * the construction of the schedule.
4243 *
4244 * If a non-trivial common divisor is found, then
4245 * the linear part is reduced and the remainder is ignored.
4246 * The pieces of the graph that are assigned different remainders
4247 * form (groups of) strongly connected components within
4248 * the scaled down band. If needed, they can therefore
4249 * be ordered along this remainder in a sequence node.
4250 * However, this ordering is not enforced here in order to allow
4251 * the scheduler to combine some of the strongly connected components.
4252 */
4253static __isl_give isl_schedule_node *split_scaled(
4254 __isl_take isl_schedule_node *node, struct isl_sched_graph *graph)
4255{
4256 int i;
4257 int row;
4258 isl_ctx *ctx;
4259 isl_int gcd, gcd_i;
4260
4261 if (!node)
4262 return NULL((void*)0);
4263
4264 ctx = isl_schedule_node_get_ctx(node);
4265 if (!ctx->opt->schedule_split_scaled)
4266 return compute_next_band(node, graph, 0);
4267 if (graph->n <= 1)
4268 return compute_next_band(node, graph, 0);
4269
4270 isl_int_init(gcd)isl_sioimath_init((gcd));
4271 isl_int_init(gcd_i)isl_sioimath_init((gcd_i));
4272
4273 isl_int_set_si(gcd, 0)isl_sioimath_set_si((gcd), 0);
4274
4275 row = isl_mat_rows(graph->node[0].sched) - 1;
4276
4277 for (i = 0; i < graph->n; ++i) {
4278 struct isl_sched_node *node = &graph->node[i];
4279 int cols = isl_mat_cols(node->sched);
4280
4281 isl_seq_gcd(node->sched->row[row] + 1, cols - 1, &gcd_i);
4282 isl_int_gcd(gcd, gcd, gcd_i)isl_sioimath_gcd((gcd), *(gcd), *(gcd_i));
4283 }
4284
4285 isl_int_clear(gcd_i)isl_sioimath_clear((gcd_i));
4286
4287 if (isl_int_cmp_si(gcd, 1)isl_sioimath_cmp_si(*(gcd), 1) <= 0) {
4288 isl_int_clear(gcd)isl_sioimath_clear((gcd));
4289 return compute_next_band(node, graph, 0);
4290 }
4291
4292 for (i = 0; i < graph->n; ++i) {
4293 struct isl_sched_node *node = &graph->node[i];
4294
4295 isl_int_fdiv_q(node->sched->row[row][0],isl_sioimath_fdiv_q((node->sched->row[row][0]), *(node->
sched->row[row][0]), *(gcd))
4296 node->sched->row[row][0], gcd)isl_sioimath_fdiv_q((node->sched->row[row][0]), *(node->
sched->row[row][0]), *(gcd))
;
4297 isl_int_mul(node->sched->row[row][0],isl_sioimath_mul((node->sched->row[row][0]), *(node->
sched->row[row][0]), *(gcd))
4298 node->sched->row[row][0], gcd)isl_sioimath_mul((node->sched->row[row][0]), *(node->
sched->row[row][0]), *(gcd))
;
4299 node->sched = isl_mat_scale_down_row(node->sched, row, gcd);
4300 if (!node->sched)
4301 goto error;
4302 }
4303
4304 isl_int_clear(gcd)isl_sioimath_clear((gcd));
4305
4306 return compute_next_band(node, graph, 0);
4307error:
4308 isl_int_clear(gcd)isl_sioimath_clear((gcd));
4309 return isl_schedule_node_free(node);
4310}
4311
4312/* Is the schedule row "sol" trivial on node "node"?
4313 * That is, is the solution zero on the dimensions linearly independent of
4314 * the previously found solutions?
4315 * Return 1 if the solution is trivial, 0 if it is not and -1 on error.
4316 *
4317 * Each coefficient is represented as the difference between
4318 * two non-negative values in "sol".
4319 * We construct the schedule row s and check if it is linearly
4320 * independent of previously computed schedule rows
4321 * by computing T s, with T the linear combinations that are zero
4322 * on linearly dependent schedule rows.
4323 * If the result consists of all zeros, then the solution is trivial.
4324 */
4325static int is_trivial(struct isl_sched_node *node, __isl_keep isl_vec *sol)
4326{
4327 int trivial;
4328 isl_vec *node_sol;
4329
4330 if (!sol)
4331 return -1;
4332 if (node->nvar == node->rank)
4333 return 0;
4334
4335 node_sol = extract_var_coef(node, sol);
4336 node_sol = isl_mat_vec_product(isl_mat_copy(node->indep), node_sol);
4337 if (!node_sol)
4338 return -1;
4339
4340 trivial = isl_seq_first_non_zero(node_sol->el,
4341 node->nvar - node->rank) == -1;
4342
4343 isl_vec_free(node_sol);
4344
4345 return trivial;
4346}
4347
4348/* Is the schedule row "sol" trivial on any node where it should
4349 * not be trivial?
4350 * Return 1 if any solution is trivial, 0 if they are not and -1 on error.
4351 */
4352static int is_any_trivial(struct isl_sched_graph *graph,
4353 __isl_keep isl_vec *sol)
4354{
4355 int i;
4356
4357 for (i = 0; i < graph->n; ++i) {
4358 struct isl_sched_node *node = &graph->node[i];
4359 int trivial;
4360
4361 if (!needs_row(graph, node))
4362 continue;
4363 trivial = is_trivial(node, sol);
4364 if (trivial < 0 || trivial)
4365 return trivial;
4366 }
4367
4368 return 0;
4369}
4370
4371/* Does the schedule represented by "sol" perform loop coalescing on "node"?
4372 * If so, return the position of the coalesced dimension.
4373 * Otherwise, return node->nvar or -1 on error.
4374 *
4375 * In particular, look for pairs of coefficients c_i and c_j such that
4376 * |c_j/c_i| > ceil(size_i/2), i.e., |c_j| > |c_i * ceil(size_i/2)|.
4377 * If any such pair is found, then return i.
4378 * If size_i is infinity, then no check on c_i needs to be performed.
4379 */
4380static int find_node_coalescing(struct isl_sched_node *node,
4381 __isl_keep isl_vec *sol)
4382{
4383 int i, j;
4384 isl_int max;
4385 isl_vec *csol;
4386
4387 if (node->nvar <= 1)
4388 return node->nvar;
4389
4390 csol = extract_var_coef(node, sol);
4391 if (!csol)
4392 return -1;
4393 isl_int_init(max)isl_sioimath_init((max));
4394 for (i = 0; i < node->nvar; ++i) {
4395 isl_val *v;
4396
4397 if (isl_int_is_zero(csol->el[i])(isl_sioimath_sgn(*(csol->el[i])) == 0))
4398 continue;
4399 v = isl_multi_val_get_val(node->sizes, i);
4400 if (!v)
4401 goto error;
4402 if (!isl_val_is_int(v)) {
4403 isl_val_free(v);
4404 continue;
4405 }
4406 v = isl_val_div_ui(v, 2);
4407 v = isl_val_ceil(v);
4408 if (!v)
4409 goto error;
4410 isl_int_mul(max, v->n, csol->el[i])isl_sioimath_mul((max), *(v->n), *(csol->el[i]));
4411 isl_val_free(v);
4412
4413 for (j = 0; j < node->nvar; ++j) {
4414 if (j == i)
4415 continue;
4416 if (isl_int_abs_gt(csol->el[j], max)(isl_sioimath_abs_cmp(*(csol->el[j]), *(max)) > 0))
4417 break;
4418 }
4419 if (j < node->nvar)
4420 break;
4421 }
4422
4423 isl_int_clear(max)isl_sioimath_clear((max));
4424 isl_vec_free(csol);
4425 return i;
4426error:
4427 isl_int_clear(max)isl_sioimath_clear((max));
4428 isl_vec_free(csol);
4429 return -1;
4430}
4431
4432/* Force the schedule coefficient at position "pos" of "node" to be zero
4433 * in "tl".
4434 * The coefficient is encoded as the difference between two non-negative
4435 * variables. Force these two variables to have the same value.
4436 */
4437static __isl_give isl_tab_lexmin *zero_out_node_coef(
4438 __isl_take isl_tab_lexmin *tl, struct isl_sched_node *node, int pos)
4439{
4440 int dim;
4441 isl_ctx *ctx;
4442 isl_vec *eq;
4443
4444 ctx = isl_space_get_ctx(node->space);
4445 dim = isl_tab_lexmin_dim(tl);
4446 if (dim < 0)
4447 return isl_tab_lexmin_free(tl);
4448 eq = isl_vec_alloc(ctx, 1 + dim);
4449 eq = isl_vec_clr(eq);
4450 if (!eq)
4451 return isl_tab_lexmin_free(tl);
4452
4453 pos = 1 + node_var_coef_pos(node, pos);
4454 isl_int_set_si(eq->el[pos], 1)isl_sioimath_set_si((eq->el[pos]), 1);
4455 isl_int_set_si(eq->el[pos + 1], -1)isl_sioimath_set_si((eq->el[pos + 1]), -1);
4456 tl = isl_tab_lexmin_add_eq(tl, eq->el);
4457 isl_vec_free(eq);
4458
4459 return tl;
4460}
4461
4462/* Return the lexicographically smallest rational point in the basic set
4463 * from which "tl" was constructed, double checking that this input set
4464 * was not empty.
4465 */
4466static __isl_give isl_vec *non_empty_solution(__isl_keep isl_tab_lexmin *tl)
4467{
4468 isl_vec *sol;
4469
4470 sol = isl_tab_lexmin_get_solution(tl);
4471 if (!sol)
4472 return NULL((void*)0);
4473 if (sol->size == 0)
4474 isl_die(isl_vec_get_ctx(sol), isl_error_internal,do { isl_handle_error(isl_vec_get_ctx(sol), isl_error_internal
, "error in schedule construction", "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 4476); return isl_vec_free(sol); } while (0)
4475 "error in schedule construction",do { isl_handle_error(isl_vec_get_ctx(sol), isl_error_internal
, "error in schedule construction", "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 4476); return isl_vec_free(sol); } while (0)
4476 return isl_vec_free(sol))do { isl_handle_error(isl_vec_get_ctx(sol), isl_error_internal
, "error in schedule construction", "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 4476); return isl_vec_free(sol); } while (0)
;
4477 return sol;
4478}
4479
4480/* Does the solution "sol" of the LP problem constructed by setup_carry_lp
4481 * carry any of the "n_edge" groups of dependences?
4482 * The value in the first position is the sum of (1 - e_i) over all "n_edge"
4483 * edges, with 0 <= e_i <= 1 equal to 1 when the dependences represented
4484 * by the edge are carried by the solution.
4485 * If the sum of the (1 - e_i) is smaller than "n_edge" then at least
4486 * one of those is carried.
4487 *
4488 * Note that despite the fact that the problem is solved using a rational
4489 * solver, the solution is guaranteed to be integral.
4490 * Specifically, the dependence distance lower bounds e_i (and therefore
4491 * also their sum) are integers. See Lemma 5 of [1].
4492 *
4493 * Any potential denominator of the sum is cleared by this function.
4494 * The denominator is not relevant for any of the other elements
4495 * in the solution.
4496 *
4497 * [1] P. Feautrier, Some Efficient Solutions to the Affine Scheduling
4498 * Problem, Part II: Multi-Dimensional Time.
4499 * In Intl. Journal of Parallel Programming, 1992.
4500 */
4501static int carries_dependences(__isl_keep isl_vec *sol, int n_edge)
4502{
4503 isl_int_divexact(sol->el[1], sol->el[1], sol->el[0])isl_sioimath_tdiv_q((sol->el[1]), *(sol->el[1]), *(sol->
el[0]))
;
4504 isl_int_set_si(sol->el[0], 1)isl_sioimath_set_si((sol->el[0]), 1);
4505 return isl_int_cmp_si(sol->el[1], n_edge)isl_sioimath_cmp_si(*(sol->el[1]), n_edge) < 0;
4506}
4507
4508/* Return the lexicographically smallest rational point in "lp",
4509 * assuming that all variables are non-negative and performing some
4510 * additional sanity checks.
4511 * If "want_integral" is set, then compute the lexicographically smallest
4512 * integer point instead.
4513 * In particular, "lp" should not be empty by construction.
4514 * Double check that this is the case.
4515 * If dependences are not carried for any of the "n_edge" edges,
4516 * then return an empty vector.
4517 *
4518 * If the schedule_treat_coalescing option is set and
4519 * if the computed schedule performs loop coalescing on a given node,
4520 * i.e., if it is of the form
4521 *
4522 * c_i i + c_j j + ...
4523 *
4524 * with |c_j/c_i| >= size_i, then force the coefficient c_i to be zero
4525 * to cut out this solution. Repeat this process until no more loop
4526 * coalescing occurs or until no more dependences can be carried.
4527 * In the latter case, revert to the previously computed solution.
4528 *
4529 * If the caller requests an integral solution and if coalescing should
4530 * be treated, then perform the coalescing treatment first as
4531 * an integral solution computed before coalescing treatment
4532 * would carry the same number of edges and would therefore probably
4533 * also be coalescing.
4534 *
4535 * To allow the coalescing treatment to be performed first,
4536 * the initial solution is allowed to be rational and it is only
4537 * cut out (if needed) in the next iteration, if no coalescing measures
4538 * were taken.
4539 */
4540static __isl_give isl_vec *non_neg_lexmin(struct isl_sched_graph *graph,
4541 __isl_take isl_basic_setisl_basic_map *lp, int n_edge, int want_integral)
4542{
4543 int i, pos, cut;
4544 isl_ctx *ctx;
4545 isl_tab_lexmin *tl;
4546 isl_vec *sol = NULL((void*)0), *prev;
4547 int treat_coalescing;
4548 int try_again;
4549
4550 if (!lp)
4551 return NULL((void*)0);
4552 ctx = isl_basic_set_get_ctx(lp);
4553 treat_coalescing = isl_options_get_schedule_treat_coalescing(ctx);
4554 tl = isl_tab_lexmin_from_basic_set(lp);
4555
4556 cut = 0;
4557 do {
4558 int integral;
4559
4560 try_again = 0;
4561 if (cut)
4562 tl = isl_tab_lexmin_cut_to_integer(tl);
4563 prev = sol;
4564 sol = non_empty_solution(tl);
4565 if (!sol)
4566 goto error;
4567
4568 integral = isl_int_is_one(sol->el[0])(isl_sioimath_cmp_si(*(sol->el[0]), 1) == 0);
4569 if (!carries_dependences(sol, n_edge)) {
4570 if (!prev)
4571 prev = isl_vec_alloc(ctx, 0);
4572 isl_vec_free(sol);
4573 sol = prev;
4574 break;
4575 }
4576 prev = isl_vec_free(prev);
4577 cut = want_integral && !integral;
4578 if (cut)
4579 try_again = 1;
4580 if (!treat_coalescing)
4581 continue;
4582 for (i = 0; i < graph->n; ++i) {
4583 struct isl_sched_node *node = &graph->node[i];
4584
4585 pos = find_node_coalescing(node, sol);
4586 if (pos < 0)
4587 goto error;
4588 if (pos < node->nvar)
4589 break;
4590 }
4591 if (i < graph->n) {
4592 try_again = 1;
4593 tl = zero_out_node_coef(tl, &graph->node[i], pos);
4594 cut = 0;
4595 }
4596 } while (try_again);
4597
4598 isl_tab_lexmin_free(tl);
4599
4600 return sol;
4601error:
4602 isl_tab_lexmin_free(tl);
4603 isl_vec_free(prev);
4604 isl_vec_free(sol);
4605 return NULL((void*)0);
4606}
4607
4608/* If "edge" is an edge from a node to itself, then add the corresponding
4609 * dependence relation to "umap".
4610 * If "node" has been compressed, then the dependence relation
4611 * is also compressed first.
4612 */
4613static __isl_give isl_union_map *add_intra(__isl_take isl_union_map *umap,
4614 struct isl_sched_edge *edge)
4615{
4616 isl_map *map;
4617 struct isl_sched_node *node = edge->src;
4618
4619 if (edge->src != edge->dst)
4620 return umap;
4621
4622 map = isl_map_copy(edge->map);
4623 if (node->compressed) {
4624 map = isl_map_preimage_domain_multi_aff(map,
4625 isl_multi_aff_copy(node->decompress));
4626 map = isl_map_preimage_range_multi_aff(map,
4627 isl_multi_aff_copy(node->decompress));
4628 }
4629 umap = isl_union_map_add_map(umap, map);
4630 return umap;
4631}
4632
4633/* If "edge" is an edge from a node to another node, then add the corresponding
4634 * dependence relation to "umap".
4635 * If the source or destination nodes of "edge" have been compressed,
4636 * then the dependence relation is also compressed first.
4637 */
4638static __isl_give isl_union_map *add_inter(__isl_take isl_union_map *umap,
4639 struct isl_sched_edge *edge)
4640{
4641 isl_map *map;
4642
4643 if (edge->src == edge->dst)
4644 return umap;
4645
4646 map = isl_map_copy(edge->map);
4647 if (edge->src->compressed)
4648 map = isl_map_preimage_domain_multi_aff(map,
4649 isl_multi_aff_copy(edge->src->decompress));
4650 if (edge->dst->compressed)
4651 map = isl_map_preimage_range_multi_aff(map,
4652 isl_multi_aff_copy(edge->dst->decompress));
4653 umap = isl_union_map_add_map(umap, map);
4654 return umap;
4655}
4656
4657/* Internal data structure used by union_drop_coalescing_constraints
4658 * to collect bounds on all relevant statements.
4659 *
4660 * "graph" is the schedule constraint graph for which an LP problem
4661 * is being constructed.
4662 * "bounds" collects the bounds.
4663 */
4664struct isl_collect_bounds_data {
4665 isl_ctx *ctx;
4666 struct isl_sched_graph *graph;
4667 isl_union_set *bounds;
4668};
4669
4670/* Add the size bounds for the node with instance deltas in "set"
4671 * to data->bounds.
4672 */
4673static isl_stat collect_bounds(__isl_take isl_setisl_map *set, void *user)
4674{
4675 struct isl_collect_bounds_data *data = user;
4676 struct isl_sched_node *node;
4677 isl_space *space;
4678 isl_setisl_map *bounds;
4679
4680 space = isl_set_get_space(set);
4681 isl_set_free(set);
4682
4683 node = graph_find_compressed_node(data->ctx, data->graph, space);
4684 isl_space_free(space);
4685
4686 bounds = isl_set_from_basic_set(get_size_bounds(node));
4687 data->bounds = isl_union_set_add_set(data->bounds, bounds);
4688
4689 return isl_stat_ok;
4690}
4691
4692/* Drop some constraints from "delta" that could be exploited
4693 * to construct loop coalescing schedules.
4694 * In particular, drop those constraint that bound the difference
4695 * to the size of the domain.
4696 * Do this for each set/node in "delta" separately.
4697 * The parameters are assumed to have been projected out by the caller.
4698 */
4699static __isl_give isl_union_set *union_drop_coalescing_constraints(isl_ctx *ctx,
4700 struct isl_sched_graph *graph, __isl_take isl_union_set *delta)
4701{
4702 struct isl_collect_bounds_data data = { ctx, graph };
4703
4704 data.bounds = isl_union_set_empty(isl_space_params_alloc(ctx, 0));
4705 if (isl_union_set_foreach_set(delta, &collect_bounds, &data) < 0)
4706 data.bounds = isl_union_set_free(data.bounds);
4707 delta = isl_union_set_plain_gist(delta, data.bounds);
4708
4709 return delta;
4710}
4711
4712/* Given a non-trivial lineality space "lineality", add the corresponding
4713 * universe set to data->mask and add a map from elements to
4714 * other elements along the lines in "lineality" to data->equivalent.
4715 * If this is the first time this function gets called
4716 * (data->any_non_trivial is still false), then set data->any_non_trivial and
4717 * initialize data->mask and data->equivalent.
4718 *
4719 * In particular, if the lineality space is defined by equality constraints
4720 *
4721 * E x = 0
4722 *
4723 * then construct an affine mapping
4724 *
4725 * f : x -> E x
4726 *
4727 * and compute the equivalence relation of having the same image under f:
4728 *
4729 * { x -> x' : E x = E x' }
4730 */
4731static isl_stat add_non_trivial_lineality(__isl_take isl_basic_setisl_basic_map *lineality,
4732 struct isl_exploit_lineality_data *data)
4733{
4734 isl_mat *eq;
4735 isl_space *space;
4736 isl_setisl_map *univ;
4737 isl_multi_aff *ma;
4738 isl_multi_pw_aff *mpa;
4739 isl_map *map;
4740 int n;
4741
4742 if (!lineality)
4743 return isl_stat_error;
4744 if (isl_basic_set_dim(lineality, isl_dim_div) != 0)
4745 isl_die(isl_basic_set_get_ctx(lineality), isl_error_internal,do { isl_handle_error(isl_basic_set_get_ctx(lineality), isl_error_internal
, "local variables not allowed", "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 4746); goto error; } while (0)
4746 "local variables not allowed", goto error)do { isl_handle_error(isl_basic_set_get_ctx(lineality), isl_error_internal
, "local variables not allowed", "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 4746); goto error; } while (0)
;
4747
4748 space = isl_basic_set_get_space(lineality);
4749 if (!data->any_non_trivial) {
4750 data->equivalent = isl_union_map_empty(isl_space_copy(space));
4751 data->mask = isl_union_set_empty(isl_space_copy(space));
4752 }
4753 data->any_non_trivial = isl_bool_true;
4754
4755 univ = isl_set_universe(isl_space_copy(space));
4756 data->mask = isl_union_set_add_set(data->mask, univ);
4757
4758 eq = isl_basic_set_extract_equalities(lineality);
4759 n = isl_mat_rows(eq);
4760 eq = isl_mat_insert_zero_rows(eq, 0, 1);
4761 eq = isl_mat_set_element_si(eq, 0, 0, 1);
4762 space = isl_space_from_domain(space);
4763 space = isl_space_add_dims(space, isl_dim_out, n);
4764 ma = isl_multi_aff_from_aff_mat(space, eq);
4765 mpa = isl_multi_pw_aff_from_multi_aff(ma);
4766 map = isl_multi_pw_aff_eq_map(mpa, isl_multi_pw_aff_copy(mpa));
4767 data->equivalent = isl_union_map_add_map(data->equivalent, map);
4768
4769 isl_basic_set_free(lineality);
4770 return isl_stat_ok;
4771error:
4772 isl_basic_set_free(lineality);
4773 return isl_stat_error;
4774}
4775
4776/* Check if the lineality space "set" is non-trivial (i.e., is not just
4777 * the origin or, in other words, satisfies a number of equality constraints
4778 * that is smaller than the dimension of the set).
4779 * If so, extend data->mask and data->equivalent accordingly.
4780 *
4781 * The input should not have any local variables already, but
4782 * isl_set_remove_divs is called to make sure it does not.
4783 */
4784static isl_stat add_lineality(__isl_take isl_setisl_map *set, void *user)
4785{
4786 struct isl_exploit_lineality_data *data = user;
4787 isl_basic_setisl_basic_map *hull;
4788 int dim, n_eq;
4789
4790 set = isl_set_remove_divs(set);
4791 hull = isl_set_unshifted_simple_hull(set);
4792 dim = isl_basic_set_dim(hull, isl_dim_set);
4793 n_eq = isl_basic_set_n_equality(hull);
4794 if (!hull)
4795 return isl_stat_error;
4796 if (dim != n_eq)
4797 return add_non_trivial_lineality(hull, data);
4798 isl_basic_set_free(hull);
4799 return isl_stat_ok;
4800}
4801
4802/* Check if the difference set on intra-node schedule constraints "intra"
4803 * has any non-trivial lineality space.
4804 * If so, then extend the difference set to a difference set
4805 * on equivalent elements. That is, if "intra" is
4806 *
4807 * { y - x : (x,y) \in V }
4808 *
4809 * and elements are equivalent if they have the same image under f,
4810 * then return
4811 *
4812 * { y' - x' : (x,y) \in V and f(x) = f(x') and f(y) = f(y') }
4813 *
4814 * or, since f is linear,
4815 *
4816 * { y' - x' : (x,y) \in V and f(y - x) = f(y' - x') }
4817 *
4818 * The results of the search for non-trivial lineality spaces is stored
4819 * in "data".
4820 */
4821static __isl_give isl_union_set *exploit_intra_lineality(
4822 __isl_take isl_union_set *intra,
4823 struct isl_exploit_lineality_data *data)
4824{
4825 isl_union_set *lineality;
4826 isl_union_set *uset;
4827
4828 data->any_non_trivial = isl_bool_false;
4829 lineality = isl_union_set_copy(intra);
4830 lineality = isl_union_set_combined_lineality_space(lineality);
4831 if (isl_union_set_foreach_set(lineality, &add_lineality, data) < 0)
4832 data->any_non_trivial = isl_bool_error;
4833 isl_union_set_free(lineality);
4834
4835 if (data->any_non_trivial < 0)
4836 return isl_union_set_free(intra);
4837 if (!data->any_non_trivial)
4838 return intra;
4839
4840 uset = isl_union_set_copy(intra);
4841 intra = isl_union_set_subtract(intra, isl_union_set_copy(data->mask));
4842 uset = isl_union_set_apply(uset, isl_union_map_copy(data->equivalent));
4843 intra = isl_union_set_union(intra, uset);
4844
4845 intra = isl_union_set_remove_divs(intra);
4846
4847 return intra;
4848}
4849
4850/* If the difference set on intra-node schedule constraints was found to have
4851 * any non-trivial lineality space by exploit_intra_lineality,
4852 * as recorded in "data", then extend the inter-node
4853 * schedule constraints "inter" to schedule constraints on equivalent elements.
4854 * That is, if "inter" is V and
4855 * elements are equivalent if they have the same image under f, then return
4856 *
4857 * { (x', y') : (x,y) \in V and f(x) = f(x') and f(y) = f(y') }
4858 */
4859static __isl_give isl_union_map *exploit_inter_lineality(
4860 __isl_take isl_union_map *inter,
4861 struct isl_exploit_lineality_data *data)
4862{
4863 isl_union_map *umap;
4864
4865 if (data->any_non_trivial < 0)
4866 return isl_union_map_free(inter);
4867 if (!data->any_non_trivial)
4868 return inter;
4869
4870 umap = isl_union_map_copy(inter);
4871 inter = isl_union_map_subtract_range(inter,
4872 isl_union_set_copy(data->mask));
4873 umap = isl_union_map_apply_range(umap,
4874 isl_union_map_copy(data->equivalent));
4875 inter = isl_union_map_union(inter, umap);
4876 umap = isl_union_map_copy(inter);
4877 inter = isl_union_map_subtract_domain(inter,
4878 isl_union_set_copy(data->mask));
4879 umap = isl_union_map_apply_range(isl_union_map_copy(data->equivalent),
4880 umap);
4881 inter = isl_union_map_union(inter, umap);
4882
4883 inter = isl_union_map_remove_divs(inter);
4884
4885 return inter;
4886}
4887
4888/* For each (conditional) validity edge in "graph",
4889 * add the corresponding dependence relation using "add"
4890 * to a collection of dependence relations and return the result.
4891 * If "coincidence" is set, then coincidence edges are considered as well.
4892 */
4893static __isl_give isl_union_map *collect_validity(struct isl_sched_graph *graph,
4894 __isl_give isl_union_map *(*add)(__isl_take isl_union_map *umap,
4895 struct isl_sched_edge *edge), int coincidence)
4896{
4897 int i;
4898 isl_space *space;
4899 isl_union_map *umap;
4900
4901 space = isl_space_copy(graph->node[0].space);
4902 umap = isl_union_map_empty(space);
4903
4904 for (i = 0; i < graph->n_edge; ++i) {
4905 struct isl_sched_edge *edge = &graph->edge[i];
4906
4907 if (!is_any_validity(edge) &&
4908 (!coincidence || !is_coincidence(edge)))
4909 continue;
4910
4911 umap = add(umap, edge);
4912 }
4913
4914 return umap;
4915}
4916
4917/* Project out all parameters from "uset" and return the result.
4918 */
4919static __isl_give isl_union_set *union_set_drop_parameters(
4920 __isl_take isl_union_set *uset)
4921{
4922 unsigned nparam;
4923
4924 nparam = isl_union_set_dim(uset, isl_dim_param);
4925 return isl_union_set_project_out(uset, isl_dim_param, 0, nparam);
4926}
4927
4928/* For each dependence relation on a (conditional) validity edge
4929 * from a node to itself,
4930 * construct the set of coefficients of valid constraints for elements
4931 * in that dependence relation and collect the results.
4932 * If "coincidence" is set, then coincidence edges are considered as well.
4933 *
4934 * In particular, for each dependence relation R, constraints
4935 * on coefficients (c_0, c_x) are constructed such that
4936 *
4937 * c_0 + c_x d >= 0 for each d in delta R = { y - x | (x,y) in R }
4938 *
4939 * If the schedule_treat_coalescing option is set, then some constraints
4940 * that could be exploited to construct coalescing schedules
4941 * are removed before the dual is computed, but after the parameters
4942 * have been projected out.
4943 * The entire computation is essentially the same as that performed
4944 * by intra_coefficients, except that it operates on multiple
4945 * edges together and that the parameters are always projected out.
4946 *
4947 * Additionally, exploit any non-trivial lineality space
4948 * in the difference set after removing coalescing constraints and
4949 * store the results of the non-trivial lineality space detection in "data".
4950 * The procedure is currently run unconditionally, but it is unlikely
4951 * to find any non-trivial lineality spaces if no coalescing constraints
4952 * have been removed.
4953 *
4954 * Note that if a dependence relation is a union of basic maps,
4955 * then each basic map needs to be treated individually as it may only
4956 * be possible to carry the dependences expressed by some of those
4957 * basic maps and not all of them.
4958 * The collected validity constraints are therefore not coalesced and
4959 * it is assumed that they are not coalesced automatically.
4960 * Duplicate basic maps can be removed, however.
4961 * In particular, if the same basic map appears as a disjunct
4962 * in multiple edges, then it only needs to be carried once.
4963 */
4964static __isl_give isl_basic_set_listisl_basic_map_list *collect_intra_validity(isl_ctx *ctx,
4965 struct isl_sched_graph *graph, int coincidence,
4966 struct isl_exploit_lineality_data *data)
4967{
4968 isl_union_map *intra;
4969 isl_union_set *delta;
4970 isl_basic_set_listisl_basic_map_list *list;
4971
4972 intra = collect_validity(graph, &add_intra, coincidence);
4973 delta = isl_union_map_deltas(intra);
4974 delta = union_set_drop_parameters(delta);
4975 delta = isl_union_set_remove_divs(delta);
4976 if (isl_options_get_schedule_treat_coalescing(ctx))
4977 delta = union_drop_coalescing_constraints(ctx, graph, delta);
4978 delta = exploit_intra_lineality(delta, data);
4979 list = isl_union_set_get_basic_set_list(delta);
4980 isl_union_set_free(delta);
4981
4982 return isl_basic_set_list_coefficients(list);
4983}
4984
4985/* For each dependence relation on a (conditional) validity edge
4986 * from a node to some other node,
4987 * construct the set of coefficients of valid constraints for elements
4988 * in that dependence relation and collect the results.
4989 * If "coincidence" is set, then coincidence edges are considered as well.
4990 *
4991 * In particular, for each dependence relation R, constraints
4992 * on coefficients (c_0, c_n, c_x, c_y) are constructed such that
4993 *
4994 * c_0 + c_n n + c_x x + c_y y >= 0 for each (x,y) in R
4995 *
4996 * This computation is essentially the same as that performed
4997 * by inter_coefficients, except that it operates on multiple
4998 * edges together.
4999 *
5000 * Additionally, exploit any non-trivial lineality space
5001 * that may have been discovered by collect_intra_validity
5002 * (as stored in "data").
5003 *
5004 * Note that if a dependence relation is a union of basic maps,
5005 * then each basic map needs to be treated individually as it may only
5006 * be possible to carry the dependences expressed by some of those
5007 * basic maps and not all of them.
5008 * The collected validity constraints are therefore not coalesced and
5009 * it is assumed that they are not coalesced automatically.
5010 * Duplicate basic maps can be removed, however.
5011 * In particular, if the same basic map appears as a disjunct
5012 * in multiple edges, then it only needs to be carried once.
5013 */
5014static __isl_give isl_basic_set_listisl_basic_map_list *collect_inter_validity(
5015 struct isl_sched_graph *graph, int coincidence,
5016 struct isl_exploit_lineality_data *data)
5017{
5018 isl_union_map *inter;
5019 isl_union_set *wrap;
5020 isl_basic_set_listisl_basic_map_list *list;
5021
5022 inter = collect_validity(graph, &add_inter, coincidence);
5023 inter = exploit_inter_lineality(inter, data);
5024 inter = isl_union_map_remove_divs(inter);
5025 wrap = isl_union_map_wrap(inter);
5026 list = isl_union_set_get_basic_set_list(wrap);
5027 isl_union_set_free(wrap);
5028 return isl_basic_set_list_coefficients(list);
5029}
5030
5031/* Construct an LP problem for finding schedule coefficients
5032 * such that the schedule carries as many of the "n_edge" groups of
5033 * dependences as possible based on the corresponding coefficient
5034 * constraints and return the lexicographically smallest non-trivial solution.
5035 * "intra" is the sequence of coefficient constraints for intra-node edges.
5036 * "inter" is the sequence of coefficient constraints for inter-node edges.
5037 * If "want_integral" is set, then compute an integral solution
5038 * for the coefficients rather than using the numerators
5039 * of a rational solution.
5040 * "carry_inter" indicates whether inter-node edges should be carried or
5041 * only respected.
5042 *
5043 * If none of the "n_edge" groups can be carried
5044 * then return an empty vector.
5045 */
5046static __isl_give isl_vec *compute_carrying_sol_coef(isl_ctx *ctx,
5047 struct isl_sched_graph *graph, int n_edge,
5048 __isl_keep isl_basic_set_listisl_basic_map_list *intra,
5049 __isl_keep isl_basic_set_listisl_basic_map_list *inter, int want_integral,
5050 int carry_inter)
5051{
5052 isl_basic_setisl_basic_map *lp;
5053
5054 if (setup_carry_lp(ctx, graph, n_edge, intra, inter, carry_inter) < 0)
5055 return NULL((void*)0);
5056
5057 lp = isl_basic_set_copy(graph->lp);
5058 return non_neg_lexmin(graph, lp, n_edge, want_integral);
5059}
5060
5061/* Construct an LP problem for finding schedule coefficients
5062 * such that the schedule carries as many of the validity dependences
5063 * as possible and
5064 * return the lexicographically smallest non-trivial solution.
5065 * If "fallback" is set, then the carrying is performed as a fallback
5066 * for the Pluto-like scheduler.
5067 * If "coincidence" is set, then try and carry coincidence edges as well.
5068 *
5069 * The variable "n_edge" stores the number of groups that should be carried.
5070 * If none of the "n_edge" groups can be carried
5071 * then return an empty vector.
5072 * If, moreover, "n_edge" is zero, then the LP problem does not even
5073 * need to be constructed.
5074 *
5075 * If a fallback solution is being computed, then compute an integral solution
5076 * for the coefficients rather than using the numerators
5077 * of a rational solution.
5078 *
5079 * If a fallback solution is being computed, if there are any intra-node
5080 * dependences, and if requested by the user, then first try
5081 * to only carry those intra-node dependences.
5082 * If this fails to carry any dependences, then try again
5083 * with the inter-node dependences included.
5084 */
5085static __isl_give isl_vec *compute_carrying_sol(isl_ctx *ctx,
5086 struct isl_sched_graph *graph, int fallback, int coincidence)
5087{
5088 int n_intra, n_inter;
5089 int n_edge;
5090 struct isl_carry carry = { 0 };
5091 isl_vec *sol;
5092
5093 carry.intra = collect_intra_validity(ctx, graph, coincidence,
5094 &carry.lineality);
5095 carry.inter = collect_inter_validity(graph, coincidence,
5096 &carry.lineality);
5097 if (!carry.intra || !carry.inter)
5098 goto error;
5099 n_intra = isl_basic_set_list_n_basic_set(carry.intra);
5100 n_inter = isl_basic_set_list_n_basic_set(carry.inter);
5101
5102 if (fallback && n_intra > 0 &&
5103 isl_options_get_schedule_carry_self_first(ctx)) {
5104 sol = compute_carrying_sol_coef(ctx, graph, n_intra,
5105 carry.intra, carry.inter, fallback, 0);
5106 if (!sol || sol->size != 0 || n_inter == 0) {
5107 isl_carry_clear(&carry);
5108 return sol;
5109 }
5110 isl_vec_free(sol);
5111 }
5112
5113 n_edge = n_intra + n_inter;
5114 if (n_edge == 0) {
5115 isl_carry_clear(&carry);
5116 return isl_vec_alloc(ctx, 0);
5117 }
5118
5119 sol = compute_carrying_sol_coef(ctx, graph, n_edge,
5120 carry.intra, carry.inter, fallback, 1);
5121 isl_carry_clear(&carry);
5122 return sol;
5123error:
5124 isl_carry_clear(&carry);
5125 return NULL((void*)0);
5126}
5127
5128/* Construct a schedule row for each node such that as many validity dependences
5129 * as possible are carried and then continue with the next band.
5130 * If "fallback" is set, then the carrying is performed as a fallback
5131 * for the Pluto-like scheduler.
5132 * If "coincidence" is set, then try and carry coincidence edges as well.
5133 *
5134 * If there are no validity dependences, then no dependence can be carried and
5135 * the procedure is guaranteed to fail. If there is more than one component,
5136 * then try computing a schedule on each component separately
5137 * to prevent or at least postpone this failure.
5138 *
5139 * If a schedule row is computed, then check that dependences are carried
5140 * for at least one of the edges.
5141 *
5142 * If the computed schedule row turns out to be trivial on one or
5143 * more nodes where it should not be trivial, then we throw it away
5144 * and try again on each component separately.
5145 *
5146 * If there is only one component, then we accept the schedule row anyway,
5147 * but we do not consider it as a complete row and therefore do not
5148 * increment graph->n_row. Note that the ranks of the nodes that
5149 * do get a non-trivial schedule part will get updated regardless and
5150 * graph->maxvar is computed based on these ranks. The test for
5151 * whether more schedule rows are required in compute_schedule_wcc
5152 * is therefore not affected.
5153 *
5154 * Insert a band corresponding to the schedule row at position "node"
5155 * of the schedule tree and continue with the construction of the schedule.
5156 * This insertion and the continued construction is performed by split_scaled
5157 * after optionally checking for non-trivial common divisors.
5158 */
5159static __isl_give isl_schedule_node *carry(__isl_take isl_schedule_node *node,
5160 struct isl_sched_graph *graph, int fallback, int coincidence)
5161{
5162 int trivial;
5163 isl_ctx *ctx;
5164 isl_vec *sol;
5165
5166 if (!node)
5167 return NULL((void*)0);
5168
5169 ctx = isl_schedule_node_get_ctx(node);
5170 sol = compute_carrying_sol(ctx, graph, fallback, coincidence);
5171 if (!sol)
5172 return isl_schedule_node_free(node);
5173 if (sol->size == 0) {
5174 isl_vec_free(sol);
5175 if (graph->scc > 1)
5176 return compute_component_schedule(node, graph, 1);
5177 isl_die(ctx, isl_error_unknown, "unable to carry dependences",do { isl_handle_error(ctx, isl_error_unknown, "unable to carry dependences"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 5178); return isl_schedule_node_free(node); } while (0)
5178 return isl_schedule_node_free(node))do { isl_handle_error(ctx, isl_error_unknown, "unable to carry dependences"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 5178); return isl_schedule_node_free(node); } while (0)
;
5179 }
5180
5181 trivial = is_any_trivial(graph, sol);
5182 if (trivial < 0) {
5183 sol = isl_vec_free(sol);
5184 } else if (trivial && graph->scc > 1) {
5185 isl_vec_free(sol);
5186 return compute_component_schedule(node, graph, 1);
5187 }
5188
5189 if (update_schedule(graph, sol, 0) < 0)
5190 return isl_schedule_node_free(node);
5191 if (trivial)
5192 graph->n_row--;
5193
5194 return split_scaled(node, graph);
5195}
5196
5197/* Construct a schedule row for each node such that as many validity dependences
5198 * as possible are carried and then continue with the next band.
5199 * Do so as a fallback for the Pluto-like scheduler.
5200 * If "coincidence" is set, then try and carry coincidence edges as well.
5201 */
5202static __isl_give isl_schedule_node *carry_fallback(
5203 __isl_take isl_schedule_node *node, struct isl_sched_graph *graph,
5204 int coincidence)
5205{
5206 return carry(node, graph, 1, coincidence);
5207}
5208
5209/* Construct a schedule row for each node such that as many validity dependences
5210 * as possible are carried and then continue with the next band.
5211 * Do so for the case where the Feautrier scheduler was selected
5212 * by the user.
5213 */
5214static __isl_give isl_schedule_node *carry_feautrier(
5215 __isl_take isl_schedule_node *node, struct isl_sched_graph *graph)
5216{
5217 return carry(node, graph, 0, 0);
5218}
5219
5220/* Construct a schedule row for each node such that as many validity dependences
5221 * as possible are carried and then continue with the next band.
5222 * Do so as a fallback for the Pluto-like scheduler.
5223 */
5224static __isl_give isl_schedule_node *carry_dependences(
5225 __isl_take isl_schedule_node *node, struct isl_sched_graph *graph)
5226{
5227 return carry_fallback(node, graph, 0);
5228}
5229
5230/* Construct a schedule row for each node such that as many validity or
5231 * coincidence dependences as possible are carried and
5232 * then continue with the next band.
5233 * Do so as a fallback for the Pluto-like scheduler.
5234 */
5235static __isl_give isl_schedule_node *carry_coincidence(
5236 __isl_take isl_schedule_node *node, struct isl_sched_graph *graph)
5237{
5238 return carry_fallback(node, graph, 1);
5239}
5240
5241/* Topologically sort statements mapped to the same schedule iteration
5242 * and add insert a sequence node in front of "node"
5243 * corresponding to this order.
5244 * If "initialized" is set, then it may be assumed that compute_maxvar
5245 * has been called on the current band. Otherwise, call
5246 * compute_maxvar if and before carry_dependences gets called.
5247 *
5248 * If it turns out to be impossible to sort the statements apart,
5249 * because different dependences impose different orderings
5250 * on the statements, then we extend the schedule such that
5251 * it carries at least one more dependence.
5252 */
5253static __isl_give isl_schedule_node *sort_statements(
5254 __isl_take isl_schedule_node *node, struct isl_sched_graph *graph,
5255 int initialized)
5256{
5257 isl_ctx *ctx;
5258 isl_union_set_list *filters;
5259
5260 if (!node)
5261 return NULL((void*)0);
5262
5263 ctx = isl_schedule_node_get_ctx(node);
5264 if (graph->n < 1)
5265 isl_die(ctx, isl_error_internal,do { isl_handle_error(ctx, isl_error_internal, "graph should have at least one node"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 5267); return isl_schedule_node_free(node); } while (0)
5266 "graph should have at least one node",do { isl_handle_error(ctx, isl_error_internal, "graph should have at least one node"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 5267); return isl_schedule_node_free(node); } while (0)
5267 return isl_schedule_node_free(node))do { isl_handle_error(ctx, isl_error_internal, "graph should have at least one node"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 5267); return isl_schedule_node_free(node); } while (0)
;
5268
5269 if (graph->n == 1)
5270 return node;
5271
5272 if (update_edges(ctx, graph) < 0)
5273 return isl_schedule_node_free(node);
5274
5275 if (graph->n_edge == 0)
5276 return node;
5277
5278 if (detect_sccs(ctx, graph) < 0)
5279 return isl_schedule_node_free(node);
5280
5281 next_band(graph);
5282 if (graph->scc < graph->n) {
5283 if (!initialized && compute_maxvar(graph) < 0)
5284 return isl_schedule_node_free(node);
5285 return carry_dependences(node, graph);
5286 }
5287
5288 filters = extract_sccs(ctx, graph);
5289 node = isl_schedule_node_insert_sequence(node, filters);
5290
5291 return node;
5292}
5293
5294/* Are there any (non-empty) (conditional) validity edges in the graph?
5295 */
5296static int has_validity_edges(struct isl_sched_graph *graph)
5297{
5298 int i;
5299
5300 for (i = 0; i < graph->n_edge; ++i) {
5301 int empty;
5302
5303 empty = isl_map_plain_is_empty(graph->edge[i].map);
5304 if (empty < 0)
5305 return -1;
5306 if (empty)
5307 continue;
5308 if (is_any_validity(&graph->edge[i]))
5309 return 1;
5310 }
5311
5312 return 0;
5313}
5314
5315/* Should we apply a Feautrier step?
5316 * That is, did the user request the Feautrier algorithm and are
5317 * there any validity dependences (left)?
5318 */
5319static int need_feautrier_step(isl_ctx *ctx, struct isl_sched_graph *graph)
5320{
5321 if (ctx->opt->schedule_algorithm != ISL_SCHEDULE_ALGORITHM_FEAUTRIER1)
5322 return 0;
5323
5324 return has_validity_edges(graph);
5325}
5326
5327/* Compute a schedule for a connected dependence graph using Feautrier's
5328 * multi-dimensional scheduling algorithm and return the updated schedule node.
5329 *
5330 * The original algorithm is described in [1].
5331 * The main idea is to minimize the number of scheduling dimensions, by
5332 * trying to satisfy as many dependences as possible per scheduling dimension.
5333 *
5334 * [1] P. Feautrier, Some Efficient Solutions to the Affine Scheduling
5335 * Problem, Part II: Multi-Dimensional Time.
5336 * In Intl. Journal of Parallel Programming, 1992.
5337 */
5338static __isl_give isl_schedule_node *compute_schedule_wcc_feautrier(
5339 isl_schedule_node *node, struct isl_sched_graph *graph)
5340{
5341 return carry_feautrier(node, graph);
5342}
5343
5344/* Turn off the "local" bit on all (condition) edges.
5345 */
5346static void clear_local_edges(struct isl_sched_graph *graph)
5347{
5348 int i;
5349
5350 for (i = 0; i < graph->n_edge; ++i)
5351 if (is_condition(&graph->edge[i]))
5352 clear_local(&graph->edge[i]);
5353}
5354
5355/* Does "graph" have both condition and conditional validity edges?
5356 */
5357static int need_condition_check(struct isl_sched_graph *graph)
5358{
5359 int i;
5360 int any_condition = 0;
5361 int any_conditional_validity = 0;
5362
5363 for (i = 0; i < graph->n_edge; ++i) {
5364 if (is_condition(&graph->edge[i]))
5365 any_condition = 1;
5366 if (is_conditional_validity(&graph->edge[i]))
5367 any_conditional_validity = 1;
5368 }
5369
5370 return any_condition && any_conditional_validity;
5371}
5372
5373/* Does "graph" contain any coincidence edge?
5374 */
5375static int has_any_coincidence(struct isl_sched_graph *graph)
5376{
5377 int i;
5378
5379 for (i = 0; i < graph->n_edge; ++i)
5380 if (is_coincidence(&graph->edge[i]))
5381 return 1;
5382
5383 return 0;
5384}
5385
5386/* Extract the final schedule row as a map with the iteration domain
5387 * of "node" as domain.
5388 */
5389static __isl_give isl_map *final_row(struct isl_sched_node *node)
5390{
5391 isl_multi_aff *ma;
5392 int row;
5393
5394 row = isl_mat_rows(node->sched) - 1;
5395 ma = node_extract_partial_schedule_multi_aff(node, row, 1);
5396 return isl_map_from_multi_aff(ma);
5397}
5398
5399/* Is the conditional validity dependence in the edge with index "edge_index"
5400 * violated by the latest (i.e., final) row of the schedule?
5401 * That is, is i scheduled after j
5402 * for any conditional validity dependence i -> j?
5403 */
5404static int is_violated(struct isl_sched_graph *graph, int edge_index)
5405{
5406 isl_map *src_sched, *dst_sched, *map;
5407 struct isl_sched_edge *edge = &graph->edge[edge_index];
5408 int empty;
5409
5410 src_sched = final_row(edge->src);
5411 dst_sched = final_row(edge->dst);
5412 map = isl_map_copy(edge->map);
5413 map = isl_map_apply_domain(map, src_sched);
5414 map = isl_map_apply_range(map, dst_sched);
5415 map = isl_map_order_gt(map, isl_dim_in, 0, isl_dim_out, 0);
5416 empty = isl_map_is_empty(map);
5417 isl_map_free(map);
5418
5419 if (empty < 0)
5420 return -1;
5421
5422 return !empty;
5423}
5424
5425/* Does "graph" have any satisfied condition edges that
5426 * are adjacent to the conditional validity constraint with
5427 * domain "conditional_source" and range "conditional_sink"?
5428 *
5429 * A satisfied condition is one that is not local.
5430 * If a condition was forced to be local already (i.e., marked as local)
5431 * then there is no need to check if it is in fact local.
5432 *
5433 * Additionally, mark all adjacent condition edges found as local.
5434 */
5435static int has_adjacent_true_conditions(struct isl_sched_graph *graph,
5436 __isl_keep isl_union_set *conditional_source,
5437 __isl_keep isl_union_set *conditional_sink)
5438{
5439 int i;
5440 int any = 0;
5441
5442 for (i = 0; i < graph->n_edge; ++i) {
5443 int adjacent, local;
5444 isl_union_map *condition;
5445
5446 if (!is_condition(&graph->edge[i]))
5447 continue;
5448 if (is_local(&graph->edge[i]))
5449 continue;
5450
5451 condition = graph->edge[i].tagged_condition;
5452 adjacent = domain_intersects(condition, conditional_sink);
5453 if (adjacent >= 0 && !adjacent)
5454 adjacent = range_intersects(condition,
5455 conditional_source);
5456 if (adjacent < 0)
5457 return -1;
5458 if (!adjacent)
5459 continue;
5460
5461 set_local(&graph->edge[i]);
5462
5463 local = is_condition_false(&graph->edge[i]);
5464 if (local < 0)
5465 return -1;
5466 if (!local)
5467 any = 1;
5468 }
5469
5470 return any;
5471}
5472
5473/* Are there any violated conditional validity dependences with
5474 * adjacent condition dependences that are not local with respect
5475 * to the current schedule?
5476 * That is, is the conditional validity constraint violated?
5477 *
5478 * Additionally, mark all those adjacent condition dependences as local.
5479 * We also mark those adjacent condition dependences that were not marked
5480 * as local before, but just happened to be local already. This ensures
5481 * that they remain local if the schedule is recomputed.
5482 *
5483 * We first collect domain and range of all violated conditional validity
5484 * dependences and then check if there are any adjacent non-local
5485 * condition dependences.
5486 */
5487static int has_violated_conditional_constraint(isl_ctx *ctx,
5488 struct isl_sched_graph *graph)
5489{
5490 int i;
5491 int any = 0;
5492 isl_union_set *source, *sink;
5493
5494 source = isl_union_set_empty(isl_space_params_alloc(ctx, 0));
5495 sink = isl_union_set_empty(isl_space_params_alloc(ctx, 0));
5496 for (i = 0; i < graph->n_edge; ++i) {
5497 isl_union_set *uset;
5498 isl_union_map *umap;
5499 int violated;
5500
5501 if (!is_conditional_validity(&graph->edge[i]))
5502 continue;
5503
5504 violated = is_violated(graph, i);
5505 if (violated < 0)
5506 goto error;
5507 if (!violated)
5508 continue;
5509
5510 any = 1;
5511
5512 umap = isl_union_map_copy(graph->edge[i].tagged_validity);
5513 uset = isl_union_map_domain(umap);
5514 source = isl_union_set_union(source, uset);
5515 source = isl_union_set_coalesce(source);
5516
5517 umap = isl_union_map_copy(graph->edge[i].tagged_validity);
5518 uset = isl_union_map_range(umap);
5519 sink = isl_union_set_union(sink, uset);
5520 sink = isl_union_set_coalesce(sink);
5521 }
5522
5523 if (any)
5524 any = has_adjacent_true_conditions(graph, source, sink);
5525
5526 isl_union_set_free(source);
5527 isl_union_set_free(sink);
5528 return any;
5529error:
5530 isl_union_set_free(source);
5531 isl_union_set_free(sink);
5532 return -1;
5533}
5534
5535/* Examine the current band (the rows between graph->band_start and
5536 * graph->n_total_row), deciding whether to drop it or add it to "node"
5537 * and then continue with the computation of the next band, if any.
5538 * If "initialized" is set, then it may be assumed that compute_maxvar
5539 * has been called on the current band. Otherwise, call
5540 * compute_maxvar if and before carry_dependences gets called.
5541 *
5542 * The caller keeps looking for a new row as long as
5543 * graph->n_row < graph->maxvar. If the latest attempt to find
5544 * such a row failed (i.e., we still have graph->n_row < graph->maxvar),
5545 * then we either
5546 * - split between SCCs and start over (assuming we found an interesting
5547 * pair of SCCs between which to split)
5548 * - continue with the next band (assuming the current band has at least
5549 * one row)
5550 * - if there is more than one SCC left, then split along all SCCs
5551 * - if outer coincidence needs to be enforced, then try to carry as many
5552 * validity or coincidence dependences as possible and
5553 * continue with the next band
5554 * - try to carry as many validity dependences as possible and
5555 * continue with the next band
5556 * In each case, we first insert a band node in the schedule tree
5557 * if any rows have been computed.
5558 *
5559 * If the caller managed to complete the schedule and the current band
5560 * is empty, then finish off by topologically
5561 * sorting the statements based on the remaining dependences.
5562 * If, on the other hand, the current band has at least one row,
5563 * then continue with the next band. Note that this next band
5564 * will necessarily be empty, but the graph may still be split up
5565 * into weakly connected components before arriving back here.
5566 */
5567static __isl_give isl_schedule_node *compute_schedule_finish_band(
5568 __isl_take isl_schedule_node *node, struct isl_sched_graph *graph,
5569 int initialized)
5570{
5571 int empty;
5572
5573 if (!node)
5574 return NULL((void*)0);
5575
5576 empty = graph->n_total_row == graph->band_start;
5577 if (graph->n_row < graph->maxvar) {
5578 isl_ctx *ctx;
5579
5580 ctx = isl_schedule_node_get_ctx(node);
5581 if (!ctx->opt->schedule_maximize_band_depth && !empty)
5582 return compute_next_band(node, graph, 1);
5583 if (graph->src_scc >= 0)
5584 return compute_split_schedule(node, graph);
5585 if (!empty)
5586 return compute_next_band(node, graph, 1);
5587 if (graph->scc > 1)
5588 return compute_component_schedule(node, graph, 1);
5589 if (!initialized && compute_maxvar(graph) < 0)
5590 return isl_schedule_node_free(node);
5591 if (isl_options_get_schedule_outer_coincidence(ctx))
5592 return carry_coincidence(node, graph);
5593 return carry_dependences(node, graph);
5594 }
5595
5596 if (!empty)
5597 return compute_next_band(node, graph, 1);
5598 return sort_statements(node, graph, initialized);
5599}
5600
5601/* Construct a band of schedule rows for a connected dependence graph.
5602 * The caller is responsible for determining the strongly connected
5603 * components and calling compute_maxvar first.
5604 *
5605 * We try to find a sequence of as many schedule rows as possible that result
5606 * in non-negative dependence distances (independent of the previous rows
5607 * in the sequence, i.e., such that the sequence is tilable), with as
5608 * many of the initial rows as possible satisfying the coincidence constraints.
5609 * The computation stops if we can't find any more rows or if we have found
5610 * all the rows we wanted to find.
5611 *
5612 * If ctx->opt->schedule_outer_coincidence is set, then we force the
5613 * outermost dimension to satisfy the coincidence constraints. If this
5614 * turns out to be impossible, we fall back on the general scheme above
5615 * and try to carry as many dependences as possible.
5616 *
5617 * If "graph" contains both condition and conditional validity dependences,
5618 * then we need to check that that the conditional schedule constraint
5619 * is satisfied, i.e., there are no violated conditional validity dependences
5620 * that are adjacent to any non-local condition dependences.
5621 * If there are, then we mark all those adjacent condition dependences
5622 * as local and recompute the current band. Those dependences that
5623 * are marked local will then be forced to be local.
5624 * The initial computation is performed with no dependences marked as local.
5625 * If we are lucky, then there will be no violated conditional validity
5626 * dependences adjacent to any non-local condition dependences.
5627 * Otherwise, we mark some additional condition dependences as local and
5628 * recompute. We continue this process until there are no violations left or
5629 * until we are no longer able to compute a schedule.
5630 * Since there are only a finite number of dependences,
5631 * there will only be a finite number of iterations.
5632 */
5633static isl_stat compute_schedule_wcc_band(isl_ctx *ctx,
5634 struct isl_sched_graph *graph)
5635{
5636 int has_coincidence;
5637 int use_coincidence;
5638 int force_coincidence = 0;
5639 int check_conditional;
5640
5641 if (sort_sccs(graph) < 0)
5642 return isl_stat_error;
5643
5644 clear_local_edges(graph);
5645 check_conditional = need_condition_check(graph);
5646 has_coincidence = has_any_coincidence(graph);
5647
5648 if (ctx->opt->schedule_outer_coincidence)
5649 force_coincidence = 1;
5650
5651 use_coincidence = has_coincidence;
5652 while (graph->n_row < graph->maxvar) {
5653 isl_vec *sol;
5654 int violated;
5655 int coincident;
5656
5657 graph->src_scc = -1;
5658 graph->dst_scc = -1;
5659
5660 if (setup_lp(ctx, graph, use_coincidence) < 0)
5661 return isl_stat_error;
5662 sol = solve_lp(ctx, graph);
5663 if (!sol)
5664 return isl_stat_error;
5665 if (sol->size == 0) {
5666 int empty = graph->n_total_row == graph->band_start;
5667
5668 isl_vec_free(sol);
5669 if (use_coincidence && (!force_coincidence || !empty)) {
5670 use_coincidence = 0;
5671 continue;
5672 }
5673 return isl_stat_ok;
5674 }
5675 coincident = !has_coincidence || use_coincidence;
5676 if (update_schedule(graph, sol, coincident) < 0)
5677 return isl_stat_error;
5678
5679 if (!check_conditional)
5680 continue;
5681 violated = has_violated_conditional_constraint(ctx, graph);
5682 if (violated < 0)
5683 return isl_stat_error;
5684 if (!violated)
5685 continue;
5686 if (reset_band(graph) < 0)
5687 return isl_stat_error;
5688 use_coincidence = has_coincidence;
5689 }
5690
5691 return isl_stat_ok;
5692}
5693
5694/* Compute a schedule for a connected dependence graph by considering
5695 * the graph as a whole and return the updated schedule node.
5696 *
5697 * The actual schedule rows of the current band are computed by
5698 * compute_schedule_wcc_band. compute_schedule_finish_band takes
5699 * care of integrating the band into "node" and continuing
5700 * the computation.
5701 */
5702static __isl_give isl_schedule_node *compute_schedule_wcc_whole(
5703 __isl_take isl_schedule_node *node, struct isl_sched_graph *graph)
5704{
5705 isl_ctx *ctx;
5706
5707 if (!node)
5708 return NULL((void*)0);
5709
5710 ctx = isl_schedule_node_get_ctx(node);
5711 if (compute_schedule_wcc_band(ctx, graph) < 0)
5712 return isl_schedule_node_free(node);
5713
5714 return compute_schedule_finish_band(node, graph, 1);
5715}
5716
5717/* Clustering information used by compute_schedule_wcc_clustering.
5718 *
5719 * "n" is the number of SCCs in the original dependence graph
5720 * "scc" is an array of "n" elements, each representing an SCC
5721 * of the original dependence graph. All entries in the same cluster
5722 * have the same number of schedule rows.
5723 * "scc_cluster" maps each SCC index to the cluster to which it belongs,
5724 * where each cluster is represented by the index of the first SCC
5725 * in the cluster. Initially, each SCC belongs to a cluster containing
5726 * only that SCC.
5727 *
5728 * "scc_in_merge" is used by merge_clusters_along_edge to keep
5729 * track of which SCCs need to be merged.
5730 *
5731 * "cluster" contains the merged clusters of SCCs after the clustering
5732 * has completed.
5733 *
5734 * "scc_node" is a temporary data structure used inside copy_partial.
5735 * For each SCC, it keeps track of the number of nodes in the SCC
5736 * that have already been copied.
5737 */
5738struct isl_clustering {
5739 int n;
5740 struct isl_sched_graph *scc;
5741 struct isl_sched_graph *cluster;
5742 int *scc_cluster;
5743 int *scc_node;
5744 int *scc_in_merge;
5745};
5746
5747/* Initialize the clustering data structure "c" from "graph".
5748 *
5749 * In particular, allocate memory, extract the SCCs from "graph"
5750 * into c->scc, initialize scc_cluster and construct
5751 * a band of schedule rows for each SCC.
5752 * Within each SCC, there is only one SCC by definition.
5753 * Each SCC initially belongs to a cluster containing only that SCC.
5754 */
5755static isl_stat clustering_init(isl_ctx *ctx, struct isl_clustering *c,
5756 struct isl_sched_graph *graph)
5757{
5758 int i;
5759
5760 c->n = graph->scc;
5761 c->scc = isl_calloc_array(ctx, struct isl_sched_graph, c->n)((struct isl_sched_graph *)isl_calloc_or_die(ctx, c->n, sizeof
(struct isl_sched_graph)))
;
5762 c->cluster = isl_calloc_array(ctx, struct isl_sched_graph, c->n)((struct isl_sched_graph *)isl_calloc_or_die(ctx, c->n, sizeof
(struct isl_sched_graph)))
;
5763 c->scc_cluster = isl_calloc_array(ctx, int, c->n)((int *)isl_calloc_or_die(ctx, c->n, sizeof(int)));
5764 c->scc_node = isl_calloc_array(ctx, int, c->n)((int *)isl_calloc_or_die(ctx, c->n, sizeof(int)));
5765 c->scc_in_merge = isl_calloc_array(ctx, int, c->n)((int *)isl_calloc_or_die(ctx, c->n, sizeof(int)));
5766 if (!c->scc || !c->cluster ||
5767 !c->scc_cluster || !c->scc_node || !c->scc_in_merge)
5768 return isl_stat_error;
5769
5770 for (i = 0; i < c->n; ++i) {
5771 if (extract_sub_graph(ctx, graph, &node_scc_exactly,
5772 &edge_scc_exactly, i, &c->scc[i]) < 0)
5773 return isl_stat_error;
5774 c->scc[i].scc = 1;
5775 if (compute_maxvar(&c->scc[i]) < 0)
5776 return isl_stat_error;
5777 if (compute_schedule_wcc_band(ctx, &c->scc[i]) < 0)
5778 return isl_stat_error;
5779 c->scc_cluster[i] = i;
5780 }
5781
5782 return isl_stat_ok;
5783}
5784
5785/* Free all memory allocated for "c".
5786 */
5787static void clustering_free(isl_ctx *ctx, struct isl_clustering *c)
5788{
5789 int i;
5790
5791 if (c->scc)
5792 for (i = 0; i < c->n; ++i)
5793 graph_free(ctx, &c->scc[i]);
5794 free(c->scc);
5795 if (c->cluster)
5796 for (i = 0; i < c->n; ++i)
5797 graph_free(ctx, &c->cluster[i]);
5798 free(c->cluster);
5799 free(c->scc_cluster);
5800 free(c->scc_node);
5801 free(c->scc_in_merge);
5802}
5803
5804/* Should we refrain from merging the cluster in "graph" with
5805 * any other cluster?
5806 * In particular, is its current schedule band empty and incomplete.
5807 */
5808static int bad_cluster(struct isl_sched_graph *graph)
5809{
5810 return graph->n_row < graph->maxvar &&
5811 graph->n_total_row == graph->band_start;
5812}
5813
5814/* Is "edge" a proximity edge with a non-empty dependence relation?
5815 */
5816static isl_bool is_non_empty_proximity(struct isl_sched_edge *edge)
5817{
5818 if (!is_proximity(edge))
5819 return isl_bool_false;
5820 return isl_bool_not(isl_map_plain_is_empty(edge->map));
5821}
5822
5823/* Return the index of an edge in "graph" that can be used to merge
5824 * two clusters in "c".
5825 * Return graph->n_edge if no such edge can be found.
5826 * Return -1 on error.
5827 *
5828 * In particular, return a proximity edge between two clusters
5829 * that is not marked "no_merge" and such that neither of the
5830 * two clusters has an incomplete, empty band.
5831 *
5832 * If there are multiple such edges, then try and find the most
5833 * appropriate edge to use for merging. In particular, pick the edge
5834 * with the greatest weight. If there are multiple of those,
5835 * then pick one with the shortest distance between
5836 * the two cluster representatives.
5837 */
5838static int find_proximity(struct isl_sched_graph *graph,
5839 struct isl_clustering *c)
5840{
5841 int i, best = graph->n_edge, best_dist, best_weight;
5842
5843 for (i = 0; i < graph->n_edge; ++i) {
5844 struct isl_sched_edge *edge = &graph->edge[i];
5845 int dist, weight;
5846 isl_bool prox;
5847
5848 prox = is_non_empty_proximity(edge);
5849 if (prox < 0)
5850 return -1;
5851 if (!prox)
5852 continue;
5853 if (edge->no_merge)
5854 continue;
5855 if (bad_cluster(&c->scc[edge->src->scc]) ||
5856 bad_cluster(&c->scc[edge->dst->scc]))
5857 continue;
5858 dist = c->scc_cluster[edge->dst->scc] -
5859 c->scc_cluster[edge->src->scc];
5860 if (dist == 0)
5861 continue;
5862 weight = edge->weight;
5863 if (best < graph->n_edge) {
5864 if (best_weight > weight)
5865 continue;
5866 if (best_weight == weight && best_dist <= dist)
5867 continue;
5868 }
5869 best = i;
5870 best_dist = dist;
5871 best_weight = weight;
5872 }
5873
5874 return best;
5875}
5876
5877/* Internal data structure used in mark_merge_sccs.
5878 *
5879 * "graph" is the dependence graph in which a strongly connected
5880 * component is constructed.
5881 * "scc_cluster" maps each SCC index to the cluster to which it belongs.
5882 * "src" and "dst" are the indices of the nodes that are being merged.
5883 */
5884struct isl_mark_merge_sccs_data {
5885 struct isl_sched_graph *graph;
5886 int *scc_cluster;
5887 int src;
5888 int dst;
5889};
5890
5891/* Check whether the cluster containing node "i" depends on the cluster
5892 * containing node "j". If "i" and "j" belong to the same cluster,
5893 * then they are taken to depend on each other to ensure that
5894 * the resulting strongly connected component consists of complete
5895 * clusters. Furthermore, if "i" and "j" are the two nodes that
5896 * are being merged, then they are taken to depend on each other as well.
5897 * Otherwise, check if there is a (conditional) validity dependence
5898 * from node[j] to node[i], forcing node[i] to follow node[j].
5899 */
5900static isl_bool cluster_follows(int i, int j, void *user)
5901{
5902 struct isl_mark_merge_sccs_data *data = user;
5903 struct isl_sched_graph *graph = data->graph;
5904 int *scc_cluster = data->scc_cluster;
5905
5906 if (data->src == i && data->dst == j)
5907 return isl_bool_true;
5908 if (data->src == j && data->dst == i)
5909 return isl_bool_true;
5910 if (scc_cluster[graph->node[i].scc] == scc_cluster[graph->node[j].scc])
5911 return isl_bool_true;
5912
5913 return graph_has_validity_edge(graph, &graph->node[j], &graph->node[i]);
5914}
5915
5916/* Mark all SCCs that belong to either of the two clusters in "c"
5917 * connected by the edge in "graph" with index "edge", or to any
5918 * of the intermediate clusters.
5919 * The marking is recorded in c->scc_in_merge.
5920 *
5921 * The given edge has been selected for merging two clusters,
5922 * meaning that there is at least a proximity edge between the two nodes.
5923 * However, there may also be (indirect) validity dependences
5924 * between the two nodes. When merging the two clusters, all clusters
5925 * containing one or more of the intermediate nodes along the
5926 * indirect validity dependences need to be merged in as well.
5927 *
5928 * First collect all such nodes by computing the strongly connected
5929 * component (SCC) containing the two nodes connected by the edge, where
5930 * the two nodes are considered to depend on each other to make
5931 * sure they end up in the same SCC. Similarly, each node is considered
5932 * to depend on every other node in the same cluster to ensure
5933 * that the SCC consists of complete clusters.
5934 *
5935 * Then the original SCCs that contain any of these nodes are marked
5936 * in c->scc_in_merge.
5937 */
5938static isl_stat mark_merge_sccs(isl_ctx *ctx, struct isl_sched_graph *graph,
5939 int edge, struct isl_clustering *c)
5940{
5941 struct isl_mark_merge_sccs_data data;
5942 struct isl_tarjan_graph *g;
5943 int i;
5944
5945 for (i = 0; i < c->n; ++i)
5946 c->scc_in_merge[i] = 0;
5947
5948 data.graph = graph;
5949 data.scc_cluster = c->scc_cluster;
5950 data.src = graph->edge[edge].src - graph->node;
5951 data.dst = graph->edge[edge].dst - graph->node;
5952
5953 g = isl_tarjan_graph_component(ctx, graph->n, data.dst,
5954 &cluster_follows, &data);
5955 if (!g)
5956 goto error;
5957
5958 i = g->op;
5959 if (i < 3)
5960 isl_die(ctx, isl_error_internal,do { isl_handle_error(ctx, isl_error_internal, "expecting at least two nodes in component"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 5962); goto error; } while (0)
5961 "expecting at least two nodes in component",do { isl_handle_error(ctx, isl_error_internal, "expecting at least two nodes in component"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 5962); goto error; } while (0)
5962 goto error)do { isl_handle_error(ctx, isl_error_internal, "expecting at least two nodes in component"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 5962); goto error; } while (0)
;
5963 if (g->order[--i] != -1)
5964 isl_die(ctx, isl_error_internal,do { isl_handle_error(ctx, isl_error_internal, "expecting end of component marker"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 5965); goto error; } while (0)
5965 "expecting end of component marker", goto error)do { isl_handle_error(ctx, isl_error_internal, "expecting end of component marker"
, "/build/llvm-toolchain-snapshot-7~svn338205/tools/polly/lib/External/isl/isl_scheduler.c"
, 5965); goto error; } while (0)
;
5966
5967 for (--i; i >= 0 && g->order[i] != -1; --i) {
5968 int scc = graph->node[g->order[i]].scc;
5969 c->scc_in_merge[scc] = 1;
5970 }
5971
5972 isl_tarjan_graph_free(g);
5973 return isl_stat_ok;
5974error:
5975 isl_tarjan_graph_free(g);
5976 return isl_stat_error;
5977}
5978
5979/* Construct the identifier "cluster_i".
5980 */
5981static __isl_give isl_id *cluster_id(isl_ctx *ctx, int i)
5982{
5983 char name[40];
5984
5985 snprintf(name, sizeof(name), "cluster_%d", i);
5986 return isl_id_alloc(ctx, name, NULL((void*)0));
5987}
5988
5989/* Construct the space of the cluster with index "i" containing
5990 * the strongly connected component "scc".
5991 *
5992 * In particular, construct a space called cluster_i with dimension equal
5993 * to the number of schedule rows in the current band of "scc".
5994 */
5995static __isl_give isl_space *cluster_space(struct isl_sched_graph *scc, int i)
5996{
5997 int nvar;
5998 isl_space *space;
5999 isl_id *id;
6000
6001 nvar = scc->n_total_row - scc->band_start;
6002 space = isl_space_copy(scc->node[0].space);
6003 space = isl_space_params(space);
6004 space = isl_space_set_from_params(space);
6005 space = isl_space_add_dims(space, isl_dim_set, nvar);
6006 id = cluster_id(isl_space_get_ctx(space), i);
6007 space = isl_space_set_tuple_id(space, isl_dim_set, id);
6008
6009 return space;
6010}
6011
6012/* Collect the domain of the graph for merging clusters.
6013 *
6014 * In particular, for each cluster with first SCC "i", construct
6015 * a set in the space called cluster_i with dimension equal
6016 * to the number of schedule rows in the current band of the cluster.
6017 */
6018static __isl_give isl_union_set *collect_domain(isl_ctx *ctx,
6019 struct isl_sched_graph *graph, struct isl_clustering *c)
6020{
6021 int i;
6022 isl_space *space;
6023 isl_union_set *domain;
6024
6025 space = isl_space_params_alloc(ctx, 0);
6026 domain = isl_union_set_empty(space);
6027
6028 for (i = 0; i < graph->scc; ++i) {
6029 isl_space *space;
6030
6031 if (!c->scc_in_merge[i])
6032 continue;
6033 if (c->scc_cluster[i] != i)
6034 continue;
6035 space = cluster_space(&c->scc[i], i);
6036 domain = isl_union_set_add_set(domain, isl_set_universe(space));
6037 }
6038
6039 return domain;
6040}
6041
6042/* Construct a map from the original instances to the corresponding
6043 * cluster instance in the current bands of the clusters in "c".
6044 */
6045static __isl_give isl_union_map *collect_cluster_map(isl_ctx *ctx,
6046 struct isl_sched_graph *graph, struct isl_clustering *c)
6047{
6048 int i, j;
6049 isl_space *space;
6050 isl_union_map *cluster_map;
6051
6052 space = isl_space_params_alloc(ctx, 0);
6053 cluster_map = isl_union_map_empty(space);
6054 for (i = 0; i < graph->scc; ++i) {
6055 int start, n;
6056 isl_id *id;
6057
6058 if (!c->scc_in_merge[i])
6059 continue;
6060
6061 id = cluster_id(ctx, c->scc_cluster[i]);
6062 start = c->scc[i].band_start;
6063 n = c->scc[i].n_total_row - start;
6064 for (j = 0; j < c->scc[i].n; ++j) {
6065 isl_multi_aff *ma;
6066 isl_map *map;
6067 struct isl_sched_node *node = &c->scc[i].node[j];
6068
6069 ma = node_extract_partial_schedule_multi_aff(node,
6070 start, n);
6071 ma = isl_multi_aff_set_tuple_id(ma, isl_dim_out,
6072 isl_id_copy(id));
6073 map = isl_map_from_multi_aff(ma);
6074 cluster_map = isl_union_map_add_map(cluster_map, map);
6075 }
6076 isl_id_free(id);
6077 }
6078
6079 return cluster_map;
6080}
6081
6082/* Add "umap" to the schedule constraints "sc" of all types of "edge"
6083 * that are not isl_edge_condition or isl_edge_conditional_validity.
6084 */
6085static __isl_give isl_schedule_constraints *add_non_conditional_constraints(
6086 struct isl_sched_edge *edge, __isl_keep isl_union_map *umap,
6087 __isl_take isl_schedule_constraints *sc)
6088{
6089 enum isl_edge_type t;
6090
6091 if (!sc)
6092 return NULL((void*)0);
6093
6094 for (t = isl_edge_first; t <= isl_edge_last; ++t) {
6095 if (t == isl_edge_condition ||
6096 t == isl_edge_conditional_validity)
6097 continue;
6098 if (!is_type(edge, t))
6099 continue;
6100 sc = isl_schedule_constraints_add(sc, t,
6101 isl_union_map_copy(umap));
6102 }
6103
6104 return sc;
6105}
6106
6107/* Add schedule constraints of types isl_edge_condition and
6108 * isl_edge_conditional_validity to "sc" by applying "umap" to
6109 * the domains of the wrapped relations in domain and range
6110 * of the corresponding tagged constraints of "edge".
6111 */
6112static __isl_give isl_schedule_constraints *add_conditional_constraints(
6113 struct isl_sched_edge *edge, __isl_keep isl_union_map *umap,
6114 __isl_take isl_schedule_constraints *sc)
6115{
6116 enum isl_edge_type t;
6117 isl_union_map *tagged;
6118
6119 for (t = isl_edge_condition; t <= isl_edge_conditional_validity; ++t) {
6120 if (!is_type(edge, t))
6121 continue;
6122 if (t == isl_edge_condition)
6123 tagged = isl_union_map_copy(edge->tagged_condition);
6124 else
6125 tagged = isl_union_map_copy(edge->tagged_validity);
6126 tagged = isl_union_map_zip(tagged);
6127 tagged = isl_union_map_apply_domain(tagged,
6128 isl_union_map_copy(umap));
6129 tagged = isl_union_map_zip(tagged);
6130 sc = isl_schedule_constraints_add(sc, t, tagged);
6131 if (!sc)
6132 return NULL((void*)0);
6133 }
6134
6135 return sc;
6136}
6137
6138/* Given a mapping "cluster_map" from the original instances to
6139 * the cluster instances, add schedule constraints on the clusters
6140 * to "sc" corresponding to the original constraints represented by "edge".
6141 *
6142 * For non-tagged dependence constraints, the cluster constraints
6143 * are obtained by applying "cluster_map" to the edge->map.
6144 *
6145 * For tagged dependence constraints, "cluster_map" needs to be applied
6146 * to the domains of the wrapped relations in domain and range
6147 * of the tagged dependence constraints. Pick out the mappings
6148 * from these domains from "cluster_map" and construct their product.
6149 * This mapping can then be applied to the pair of domains.
6150 */
6151static __isl_give isl_schedule_constraints *collect_edge_constraints(
6152 struct isl_sched_edge *edge, __isl_keep isl_union_map *cluster_map,
6153 __isl_take isl_schedule_constraints *sc)
6154{
6155 isl_union_map *umap;
6156 isl_space *space;
6157 isl_union_set *uset;
6158 isl_union_map *umap1, *umap2;
6159
6160 if (!sc)
6161 return NULL((void*)0);
6162
6163 umap = isl_union_map_from_map(isl_map_copy(edge->map));
6164 umap = isl_union_map_apply_domain(umap,
6165 isl_union_map_copy(cluster_map));
6166 umap = isl_union_map_apply_range(umap,
6167 isl_union_map_copy(cluster_map));
6168 sc = add_non_conditional_constraints(edge, umap, sc);
6169 isl_union_map_free(umap);
6170
6171 if (!sc || (!is_condition(edge) && !is_conditional_validity(edge)))
6172 return sc;
6173
6174 space = isl_space_domain(isl_map_get_space(edge->map));
6175 uset = isl_union_set_from_set(isl_set_universe(space));
6176 umap1 = isl_union_map_copy(cluster_map);
6177 umap1 = isl_union_map_intersect_domain(umap1, uset);
6178 space = isl_space_range(isl_map_get_space(edge->map));
6179 uset = isl_union_set_from_set(isl_set_universe(space));
6180 umap2 = isl_union_map_copy(cluster_map);
6181 umap2 = isl_union_map_intersect_domain(umap2, uset);
6182 umap = isl_union_map_product(umap1, umap2);
6183
6184 sc = add_conditional_constraints(edge, umap, sc);
6185
6186 isl_union_map_free(umap);
6187 return sc;
6188}
6189
6190/* Given a mapping "cluster_map" from the original instances to
6191 * the cluster instances, add schedule constraints on the clusters
6192 * to "sc" corresponding to all edges in "graph" between nodes that
6193 * belong to SCCs that are marked for merging in "scc_in_merge".
6194 */
6195static __isl_give isl_schedule_constraints *collect_constraints(
6196 struct isl_sched_graph *graph, int *scc_in_merge,
6197 __isl_keep isl_union_map *cluster_map,
6198 __isl_take isl_schedule_constraints *sc)
6199{
6200 int i;
6201
6202 for (i = 0; i < graph->n_edge; ++i) {
6203 struct isl_sched_edge *edge = &graph->edge[i];
6204
6205 if (!scc_in_merge[edge->src->scc])
6206 continue;
6207 if (!scc_in_merge[edge->dst->scc])
6208 continue;
6209 sc = collect_edge_constraints(edge, cluster_map, sc);
6210 }
6211
6212 return sc;
6213}
6214
6215/* Construct a dependence graph for scheduling clusters with respect
6216 * to each other and store the result in "merge_graph".
6217 * In particular, the nodes of the graph correspond to the schedule
6218 * dimensions of the current bands of those clusters that have been
6219 * marked for merging in "c".
6220 *
6221 * First construct an isl_schedule_constraints object for this domain
6222 * by transforming the edges in "graph" to the domain.
6223 * Then initialize a dependence graph for scheduling from these
6224 * constraints.
6225 */
6226static isl_stat init_merge_graph(isl_ctx *ctx, struct isl_sched_graph *graph,
6227 struct isl_clustering *c, struct isl_sched_graph *merge_graph)
6228{
6229 isl_union_set *domain;
6230 isl_union_map *cluster_map;
6231 isl_schedule_constraints *sc;
6232 isl_stat r;
6233
6234 domain = collect_domain(ctx, graph, c);
6235 sc = isl_schedule_constraints_on_domain(domain);
6236 if (!sc)
6237 return isl_stat_error;
6238 cluster_map = collect_cluster_map(ctx, graph, c);
6239 sc = collect_constraints(graph, c->scc_in_merge, cluster_map, sc);
6240 isl_union_map_free(cluster_map);
6241
6242 r = graph_init(merge_graph, sc);
6243
6244 isl_schedule_constraints_free(sc);
6245
6246 return r;
6247}
6248
6249/* Compute the maximal number of remaining schedule rows that still need
6250 * to be computed for the nodes that belong to clusters with the maximal
6251 * dimension for the current band (i.e., the band that is to be merged).
6252 * Only clusters that are about to be merged are considered.
6253 * "maxvar" is the maximal dimension for the current band.
6254 * "c" contains information about the clusters.
6255 *
6256 * Return the maximal number of remaining schedule rows or -1 on error.
6257 */
6258static int compute_maxvar_max_slack(int maxvar, struct isl_clustering *c)
6259{
6260 int i, j;
6261 int max_slack;
6262
6263 max_slack = 0;
6264 for (i = 0; i < c->n; ++i) {
6265 int nvar;
6266 struct isl_sched_graph *scc;
6267
6268 if (!c->scc_in_merge[i])
6269 continue;
6270 scc = &c->scc[i];
6271 nvar = scc->n_total_row - scc->band_start;
6272 if (nvar != maxvar)
6273 continue;
6274 for (j = 0; j < scc->n; ++j) {
6275 struct isl_sched_node *node = &scc->node[j];
6276 int slack;
6277
6278 if (node_update_vmap(node) < 0)
6279 return -1;
6280 slack = node->nvar - node->rank;
6281 if (slack > max_slack)
6282 max_slack = slack;
6283 }
6284 }
6285
6286 return max_slack;
6287}
6288
6289/* If there are any clusters where the dimension of the current band
6290 * (i.e., the band that is to be merged) is smaller than "maxvar" and
6291 * if there are any nodes in such a cluster where the number
6292 * of remaining schedule rows that still need to be computed
6293 * is greater than "max_slack", then return the smallest current band
6294 * dimension of all these clusters. Otherwise return the original value
6295 * of "maxvar". Return -1 in case of any error.
6296 * Only clusters that are about to be merged are considered.
6297 * "c" contains information about the clusters.
6298 */
6299static int limit_maxvar_to_slack(int maxvar, int max_slack,
6300 struct isl_clustering *c)
6301{
6302 int i, j;
6303
6304 for (i = 0; i < c->n; ++i) {
6305 int nvar;
6306 struct isl_sched_graph *scc;
6307
6308 if (!c->scc_in_merge[i])
6309 continue;
6310 scc = &c->scc[i];
6311 nvar = scc->n_total_row - scc->band_start;
6312 if (nvar >= maxvar)
6313 continue;
6314 for (j = 0; j < scc->n; ++j) {
6315 struct isl_sched_node *node = &scc->node[j];
6316 int slack;
6317
6318 if (node_update_vmap(node) < 0)
6319 return -1;
6320 slack = node->nvar - node->rank;
6321 if (slack > max_slack) {
6322 maxvar = nvar;
6323 break;
6324 }
6325 }
6326 }
6327
6328 return maxvar;
6329}
6330
6331/* Adjust merge_graph->maxvar based on the number of remaining schedule rows
6332 * that still need to be computed. In particular, if there is a node
6333 * in a cluster where the dimension of the current band is smaller
6334 * than merge_graph->maxvar, but the number of remaining schedule rows
6335 * is greater than that of any node in a cluster with the maximal
6336 * dimension for the current band (i.e., merge_graph->maxvar),
6337 * then adjust merge_graph->maxvar to the (smallest) current band dimension
6338 * of those clusters. Without this adjustment, the total number of
6339 * schedule dimensions would be increased, resulting in a skewed view
6340 * of the number of coincident dimensions.
6341 * "c" contains information about the clusters.
6342 *
6343 * If the maximize_band_depth option is set and merge_graph->maxvar is reduced,
6344 * then there is no point in attempting any merge since it will be rejected
6345 * anyway. Set merge_graph->maxvar to zero in such cases.
6346 */
6347static isl_stat adjust_maxvar_to_slack(isl_ctx *ctx,
6348 struct isl_sched_graph *merge_graph, struct isl_clustering *c)
6349{
6350 int max_slack, maxvar;
6351
6352 max_slack = compute_maxvar_max_slack(merge_graph->maxvar, c);
6353 if (max_slack < 0)
6354 return isl_stat_error;
6355 maxvar = limit_maxvar_to_slack(merge_graph->maxvar, max_slack, c);
6356 if (maxvar < 0)
6357 return isl_stat_error;
6358
6359 if (maxvar < merge_graph->maxvar) {
6360 if (isl_options_get_schedule_maximize_band_depth(ctx))
6361 merge_graph->maxvar = 0;
6362 else
6363 merge_graph->maxvar = maxvar;
6364 }
6365
6366 return isl_stat_ok;
6367}
6368
6369/* Return the number of coincident dimensions in the current band of "graph",
6370 * where the nodes of "graph" are assumed to be scheduled by a single band.
6371 */
6372static int get_n_coincident(struct isl_sched_graph *graph)
6373{
6374 int i;
6375
6376 for (i = graph->band_start; i < graph->n_total_row; ++i)
6377 if (!graph->node[0].coincident[i])
6378 break;
6379
6380 return i - graph->band_start;
6381}
6382
6383/* Should the clusters be merged based on the cluster schedule
6384 * in the current (and only) band of "merge_graph", given that
6385 * coincidence should be maximized?
6386 *
6387 * If the number of coincident schedule dimensions in the merged band
6388 * would be less than the maximal number of coincident schedule dimensions
6389 * in any of the merged clusters, then the clusters should not be merged.
6390 */
6391static isl_bool ok_to_merge_coincident(struct isl_clustering *c,
6392 struct isl_sched_graph *merge_graph)
6393{
6394 int i;
6395 int n_coincident;
6396 int max_coincident;
6397
6398 max_coincident = 0;
6399 for (i = 0; i < c->n; ++i) {
6400 if (!c->scc_in_merge[i])
6401 continue;
6402 n_coincident = get_n_coincident(&c->scc[i]);
6403 if (n_coincident > max_coincident)
6404 max_coincident = n_coincident;
6405 }
6406
6407 n_coincident = get_n_coincident(merge_graph);
6408
6409 return n_coincident >= max_coincident;
6410}
6411
6412/* Return the transformation on "node" expressed by the current (and only)
6413 * band of "merge_graph" applied to the clusters in "c".
6414 *
6415 * First find the representation of "node" in its SCC in "c" and
6416 * extract the transformation expressed by the current band.
6417 * Then extract the transformation applied by "merge_graph"
6418 * to the cluster to which this SCC belongs.
6419 * Combine the two to obtain the complete transformation on the node.
6420 *
6421 * Note that the range of the first transformation is an anonymous space,
6422 * while the domain of the second is named "cluster_X". The range
6423 * of the former therefore needs to be adjusted before the two
6424 * can be combined.
6425 */
6426static __isl_give isl_map *extract_node_transformation(isl_ctx *ctx,
6427 struct isl_sched_node *node, struct isl_clustering *c,
6428 struct isl_sched_graph *merge_graph)
6429{
6430 struct isl_sched_node *scc_node, *cluster_node;
6431