LLVM 24.0.0git
ScalarEvolution.cpp
Go to the documentation of this file.
1//===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the scalar evolution analysis
10// engine, which is used primarily to analyze expressions involving induction
11// variables in loops.
12//
13// There are several aspects to this library. First is the representation of
14// scalar expressions, which are represented as subclasses of the SCEV class.
15// These classes are used to represent certain types of subexpressions that we
16// can handle. We only create one SCEV of a particular shape, so
17// pointer-comparisons for equality are legal.
18//
19// One important aspect of the SCEV objects is that they are never cyclic, even
20// if there is a cycle in the dataflow for an expression (ie, a PHI node). If
21// the PHI node is one of the idioms that we can represent (e.g., a polynomial
22// recurrence) then we represent it directly as a recurrence node, otherwise we
23// represent it as a SCEVUnknown node.
24//
25// In addition to being able to represent expressions of various types, we also
26// have folders that are used to build the *canonical* representation for a
27// particular expression. These folders are capable of using a variety of
28// rewrite rules to simplify the expressions.
29//
30// Once the folders are defined, we can implement the more interesting
31// higher-level code, such as the code that recognizes PHI nodes of various
32// types, computes the execution count of a loop, etc.
33//
34// TODO: We should use these routines and value representations to implement
35// dependence analysis!
36//
37//===----------------------------------------------------------------------===//
38//
39// There are several good references for the techniques used in this analysis.
40//
41// Chains of recurrences -- a method to expedite the evaluation
42// of closed-form functions
43// Olaf Bachmann, Paul S. Wang, Eugene V. Zima
44//
45// On computational properties of chains of recurrences
46// Eugene V. Zima
47//
48// Symbolic Evaluation of Chains of Recurrences for Loop Optimization
49// Robert A. van Engelen
50//
51// Efficient Symbolic Analysis for Optimizing Compilers
52// Robert A. van Engelen
53//
54// Using the chains of recurrences algebra for data dependence testing and
55// induction variable substitution
56// MS Thesis, Johnie Birch
57//
58//===----------------------------------------------------------------------===//
59
61#include "AnalysisOptions.h"
62#include "llvm/ADT/APInt.h"
63#include "llvm/ADT/ArrayRef.h"
64#include "llvm/ADT/DenseMap.h"
66#include "llvm/ADT/FoldingSet.h"
67#include "llvm/ADT/STLExtras.h"
68#include "llvm/ADT/ScopeExit.h"
69#include "llvm/ADT/Sequence.h"
72#include "llvm/ADT/Statistic.h"
74#include "llvm/ADT/StringRef.h"
84#include "llvm/Config/llvm-config.h"
85#include "llvm/IR/Argument.h"
86#include "llvm/IR/BasicBlock.h"
87#include "llvm/IR/CFG.h"
88#include "llvm/IR/Constant.h"
90#include "llvm/IR/Constants.h"
91#include "llvm/IR/DataLayout.h"
93#include "llvm/IR/Dominators.h"
94#include "llvm/IR/Function.h"
95#include "llvm/IR/GlobalAlias.h"
96#include "llvm/IR/GlobalValue.h"
98#include "llvm/IR/InstrTypes.h"
99#include "llvm/IR/Instruction.h"
100#include "llvm/IR/Instructions.h"
102#include "llvm/IR/Intrinsics.h"
103#include "llvm/IR/LLVMContext.h"
104#include "llvm/IR/Operator.h"
105#include "llvm/IR/PatternMatch.h"
106#include "llvm/IR/Type.h"
107#include "llvm/IR/Use.h"
108#include "llvm/IR/User.h"
109#include "llvm/IR/Value.h"
110#include "llvm/IR/Verifier.h"
112#include "llvm/Pass.h"
113#include "llvm/Support/Casting.h"
116#include "llvm/Support/Debug.h"
122#include <algorithm>
123#include <cassert>
124#include <climits>
125#include <cstdint>
126#include <cstdlib>
127#include <map>
128#include <memory>
129#include <numeric>
130#include <optional>
131#include <tuple>
132#include <utility>
133#include <vector>
134
135using namespace llvm;
136using namespace PatternMatch;
137using namespace SCEVPatternMatch;
138
139#define DEBUG_TYPE "scalar-evolution"
140
141STATISTIC(NumExitCountsComputed,
142 "Number of loop exits with predictable exit counts");
143STATISTIC(NumExitCountsNotComputed,
144 "Number of loop exits without predictable exit counts");
145STATISTIC(NumBruteForceTripCountsComputed,
146 "Number of loops with trip counts computed by force");
147
148#ifdef EXPENSIVE_CHECKS
149bool llvm::VerifySCEV = true;
150#else
151bool llvm::VerifySCEV = false;
152#endif
153
155 "verify-scev", cl::Hidden, cl::location(VerifySCEV),
156 cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
157
158//===----------------------------------------------------------------------===//
159// SCEV class definitions
160//===----------------------------------------------------------------------===//
161
163 // Leaf nodes are always their own canonical.
164 switch (getSCEVType()) {
165 case scConstant:
166 case scVScale:
167 case scUnknown:
168 CanonicalSCEV = this;
169 return;
170 default:
171 break;
172 }
173
174 // For all other expressions, check whether any immediate operand has a
175 // different canonical. Since operands are always created before their parent,
176 // their canonical pointers are already set — no recursion needed.
177 if (all_of(operands(), [](SCEVUse Op) { return Op.isCanonical(); })) {
178 CanonicalSCEV = this;
179 return;
180 }
181
183 map_range(operands(), [](SCEVUse Op) { return Op.getCanonical(); }));
184 // Rebuild the expression from the canonical operands, stripping use flags.
185 CanonicalSCEV = SE.getWithOperands(this, CanonOps);
186}
187
188//===----------------------------------------------------------------------===//
189// Implementation of the SCEV class.
190//
191
192#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
194 print(dbgs());
195 dbgs() << '\n';
196}
197#endif
198
199void SCEV::print(raw_ostream &OS) const {
200 switch (getSCEVType()) {
201 case scConstant:
202 cast<SCEVConstant>(this)->getValue()->printAsOperand(OS, false);
203 return;
204 case scVScale:
205 OS << "vscale";
206 return;
207 case scPtrToAddr: {
208 const SCEVCastExpr *PtrCast = cast<SCEVCastExpr>(this);
209 SCEVUse Op = PtrCast->getOperand();
210 OS << "(ptrtoaddr " << *Op->getType() << " " << Op << " to "
211 << *PtrCast->getType() << ")";
212 return;
213 }
214 case scTruncate: {
215 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(this);
216 SCEVUse Op = Trunc->getOperand();
217 OS << "(trunc " << *Op->getType() << " " << Op << " to "
218 << *Trunc->getType() << ")";
219 return;
220 }
221 case scZeroExtend: {
223 SCEVUse Op = ZExt->getOperand();
224 OS << "(zext " << *Op->getType() << " " << Op << " to " << *ZExt->getType()
225 << ")";
226 return;
227 }
228 case scSignExtend: {
230 SCEVUse Op = SExt->getOperand();
231 OS << "(sext " << *Op->getType() << " " << Op << " to " << *SExt->getType()
232 << ")";
233 return;
234 }
235 case scAddRecExpr: {
236 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(this);
237 OS << "{" << AR->getOperand(0);
238 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
239 OS << ",+," << AR->getOperand(i);
240 OS << "}<";
241 if (AR->hasNoUnsignedWrap())
242 OS << "nuw><";
243 if (AR->hasNoSignedWrap())
244 OS << "nsw><";
245 if (AR->hasNoSelfWrap() && !AR->hasNoUnsignedWrap() &&
246 !AR->hasNoSignedWrap())
247 OS << "nw><";
248 AR->getLoop()->getHeader()->printAsOperand(OS, /*PrintType=*/false);
249 OS << ">";
250 return;
251 }
252 case scAddExpr:
253 case scMulExpr:
254 case scUMaxExpr:
255 case scSMaxExpr:
256 case scUMinExpr:
257 case scSMinExpr:
259 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(this);
260 const char *OpStr = nullptr;
261 switch (NAry->getSCEVType()) {
262 case scAddExpr: OpStr = " + "; break;
263 case scMulExpr: OpStr = " * "; break;
264 case scUMaxExpr: OpStr = " umax "; break;
265 case scSMaxExpr: OpStr = " smax "; break;
266 case scUMinExpr:
267 OpStr = " umin ";
268 break;
269 case scSMinExpr:
270 OpStr = " smin ";
271 break;
273 OpStr = " umin_seq ";
274 break;
275 default:
276 llvm_unreachable("There are no other nary expression types.");
277 }
278 OS << "(" << llvm::interleaved(NAry->operands(), OpStr) << ")";
279 switch (NAry->getSCEVType()) {
280 case scAddExpr:
281 case scMulExpr:
282 if (NAry->hasNoUnsignedWrap())
283 OS << "<nuw>";
284 if (NAry->hasNoSignedWrap())
285 OS << "<nsw>";
286 break;
287 default:
288 // Nothing to print for other nary expressions.
289 break;
290 }
291 return;
292 }
293 case scUDivExpr: {
294 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(this);
295 OS << "(" << UDiv->getLHS() << " /u " << UDiv->getRHS() << ")";
296 return;
297 }
298 case scUnknown:
299 cast<SCEVUnknown>(this)->getValue()->printAsOperand(OS, false);
300 return;
302 OS << "***COULDNOTCOMPUTE***";
303 return;
304 }
305 llvm_unreachable("Unknown SCEV kind!");
306}
307
309 switch (getSCEVType()) {
310 case scConstant:
311 case scVScale:
312 case scUnknown:
313 return {};
314 case scPtrToAddr:
315 case scTruncate:
316 case scZeroExtend:
317 case scSignExtend:
318 return cast<SCEVCastExpr>(this)->operands();
319 case scAddRecExpr:
320 case scAddExpr:
321 case scMulExpr:
322 case scUMaxExpr:
323 case scSMaxExpr:
324 case scUMinExpr:
325 case scSMinExpr:
327 return cast<SCEVNAryExpr>(this)->operands();
328 case scUDivExpr:
329 return cast<SCEVUDivExpr>(this)->operands();
331 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
332 }
333 llvm_unreachable("Unknown SCEV kind!");
334}
335
336bool SCEV::isZero() const { return match(this, m_scev_Zero()); }
337
338bool SCEV::isOne() const { return match(this, m_scev_One()); }
339
340bool SCEV::isAllOnesValue() const { return match(this, m_scev_AllOnes()); }
341
344 if (!Mul) return false;
345
346 // If there is a constant factor, it will be first.
347 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0));
348 if (!SC) return false;
349
350 // Return true if the value is negative, this matches things like (-42 * V).
351 return SC->getAPInt().isNegative();
352}
353
356
358 return S->getSCEVType() == scCouldNotCompute;
359}
360
362 auto &Entry = ConstantSCEVs[V];
363 if (Entry)
364 return Entry;
365
368 ID.AddPointer(V);
370 if (SCEVConstant *S =
371 static_cast<SCEVConstant *>(UniqueSCEVs.lookup(ID, Token)))
372 return Entry = S;
373 SCEVConstant *S =
374 new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V);
375 UniqueSCEVs.insert(S, Token);
376 S->computeAndSetCanonical(*this);
377 return Entry = S;
378}
379
381 return getConstant(ConstantInt::get(getContext(), Val));
382}
383
384const SCEV *
387 // TODO: Avoid implicit trunc?
388 // See https://github.com/llvm/llvm-project/issues/112510.
389 return getConstant(
390 ConstantInt::get(ITy, V, isSigned, /*ImplicitTrunc=*/true));
391}
392
396 ID.AddPointer(Ty);
398 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
399 return S;
400 SCEV *S = new (SCEVAllocator) SCEVVScale(ID.Intern(SCEVAllocator), Ty);
401 UniqueSCEVs.insert(S, Token);
402 S->computeAndSetCanonical(*this);
403 return S;
404}
405
407 SCEVFlags Flags) {
408 const SCEV *Res = getConstant(Ty, EC.getKnownMinValue());
409 if (EC.isScalable())
410 Res = getMulExpr(Res, getVScale(Ty), Flags);
411 return Res;
412}
413
415 SCEVUse op, Type *ty)
416 : SCEV(ID, SCEVTy, computeExpressionSize(op), ty), Op(op) {}
417
418SCEVPtrToAddrExpr::SCEVPtrToAddrExpr(const FoldingSetNodeIDRef ID,
419 const SCEV *Op, Type *ITy)
420 : SCEVCastExpr(ID, scPtrToAddr, Op, ITy) {
421 assert(getOperand()->getType()->isPointerTy() && getType()->isIntegerTy() &&
422 "Must be a non-bit-width-changing pointer-to-integer cast!");
423}
424
429
430SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
431 Type *ty)
433 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
434 "Cannot truncate non-integer value!");
435}
436
437SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
438 Type *ty)
440 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
441 "Cannot zero extend non-integer value!");
442}
443
444SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
445 Type *ty)
447 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
448 "Cannot sign extend non-integer value!");
449}
450
452 // Clear this SCEVUnknown from various maps.
453 SE->forgetMemoizedResults({this});
454
455 // Remove this SCEVUnknown from the uniquing map.
456 SE->UniqueSCEVs.erase(this);
457
458 // Release the value.
459 setValPtr(nullptr);
460}
461
462void SCEVUnknown::allUsesReplacedWith(Value *New) {
463 // Clear this SCEVUnknown from various maps.
464 SE->forgetMemoizedResults({this});
465
466 // Remove this SCEVUnknown from the uniquing map.
467 SE->UniqueSCEVs.erase(this);
468
469 // Replace the value pointer in case someone is still using this SCEVUnknown.
470 setValPtr(New);
471}
472
473//===----------------------------------------------------------------------===//
474// SCEV Utilities
475//===----------------------------------------------------------------------===//
476
477/// Compare the two values \p LV and \p RV in terms of their "complexity" where
478/// "complexity" is a partial (and somewhat ad-hoc) relation used to order
479/// operands in SCEV expressions.
480static int compareValueComplexity(const AnalysisOptions &Opts,
481 const LoopInfo *const LI, Value *LV,
482 Value *RV, unsigned Depth) {
483 if (Depth > Opts.scalar_evolution_max_value_compare_depth)
484 return 0;
485
486 // Order pointer values after integer values. This helps SCEVExpander form
487 // GEPs.
488 bool LIsPointer = LV->getType()->isPointerTy(),
489 RIsPointer = RV->getType()->isPointerTy();
490 if (LIsPointer != RIsPointer)
491 return (int)LIsPointer - (int)RIsPointer;
492
493 // Compare getValueID values.
494 unsigned LID = LV->getValueID(), RID = RV->getValueID();
495 if (LID != RID)
496 return (int)LID - (int)RID;
497
498 // Sort arguments by their position.
499 if (const auto *LA = dyn_cast<Argument>(LV)) {
500 const auto *RA = cast<Argument>(RV);
501 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
502 return (int)LArgNo - (int)RArgNo;
503 }
504
505 if (const auto *LGV = dyn_cast<GlobalValue>(LV)) {
506 const auto *RGV = cast<GlobalValue>(RV);
507
508 if (auto L = LGV->getLinkage() - RGV->getLinkage())
509 return L;
510
511 const auto IsGVNameSemantic = [&](const GlobalValue *GV) {
512 auto LT = GV->getLinkage();
513 return !(GlobalValue::isPrivateLinkage(LT) ||
515 };
516
517 // Use the names to distinguish the two values, but only if the
518 // names are semantically important.
519 if (IsGVNameSemantic(LGV) && IsGVNameSemantic(RGV))
520 return LGV->getName().compare(RGV->getName());
521 }
522
523 // For instructions, compare their loop depth, and their operand count. This
524 // is pretty loose.
525 if (const auto *LInst = dyn_cast<Instruction>(LV)) {
526 const auto *RInst = cast<Instruction>(RV);
527
528 // Compare loop depths.
529 const BasicBlock *LParent = LInst->getParent(),
530 *RParent = RInst->getParent();
531 if (LParent != RParent) {
532 unsigned LDepth = LI->getLoopDepth(LParent),
533 RDepth = LI->getLoopDepth(RParent);
534 if (LDepth != RDepth)
535 return (int)LDepth - (int)RDepth;
536 }
537
538 // Compare the number of operands.
539 unsigned LNumOps = LInst->getNumOperands(),
540 RNumOps = RInst->getNumOperands();
541 if (LNumOps != RNumOps)
542 return (int)LNumOps - (int)RNumOps;
543
544 for (unsigned Idx : seq(LNumOps)) {
545 int Result = compareValueComplexity(Opts, LI, LInst->getOperand(Idx),
546 RInst->getOperand(Idx), Depth + 1);
547 if (Result != 0)
548 return Result;
549 }
550 }
551
552 return 0;
553}
554
555// Return negative, zero, or positive, if LHS is less than, equal to, or greater
556// than RHS, respectively. A three-way result allows recursive comparisons to be
557// more efficient. The result is a total order.
558static int compareSCEVComplexity(const AnalysisOptions &Opts,
559 const LoopInfo *const LI, SCEVUse LHS,
561 // Fast-path: SCEVs are uniqued so we can do a quick equality check.
562 if (LHS == RHS)
563 return 0;
564
565 if (LHS.getPointer() == RHS.getPointer())
566 return (int)LHS.getUseNoWrapFlags() - (int)RHS.getUseNoWrapFlags();
567
568 // Primarily, sort the SCEVs by their getSCEVType().
569 SCEVTypes LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
570 if (LType != RType)
571 return (int)LType - (int)RType;
572
573 // Aside from the getSCEVType() ordering, the particular ordering
574 // isn't very important except that it's beneficial to be consistent,
575 // so that (a + b) and (b + a) don't end up as different expressions.
576 switch (LType) {
577 case scUnknown: {
578 const SCEVUnknown *LU = cast<SCEVUnknown>(LHS);
579 const SCEVUnknown *RU = cast<SCEVUnknown>(RHS);
580
581 if (int X = compareValueComplexity(Opts, LI, LU->getValue(), RU->getValue(),
582 /*Depth=*/0))
583 return X;
584 return LU->getOrderID() < RU->getOrderID() ? -1 : 1;
585 }
586
587 case scConstant: {
590
591 // Compare constant values.
592 const APInt &LA = LC->getAPInt();
593 const APInt &RA = RC->getAPInt();
594 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
595 if (LBitWidth != RBitWidth)
596 return (int)LBitWidth - (int)RBitWidth;
597 return LA.ult(RA) ? -1 : 1;
598 }
599
600 case scVScale: {
601 const auto *LTy = cast<IntegerType>(cast<SCEVVScale>(LHS)->getType());
602 const auto *RTy = cast<IntegerType>(cast<SCEVVScale>(RHS)->getType());
603 return LTy->getBitWidth() - RTy->getBitWidth();
604 }
605
606 case scAddRecExpr: {
609
610 // There is always a dominance between two recs that are used by one SCEV,
611 // so we can safely sort recs by loop header dominance. We require such
612 // order in getAddExpr.
613 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
614 if (LLoop != RLoop) {
615 const BasicBlock *LHead = LLoop->getHeader(), *RHead = RLoop->getHeader();
616 assert(LHead != RHead && "Two loops share the same header?");
617 if (DT.dominates(LHead, RHead))
618 return 1;
619 assert(DT.dominates(RHead, LHead) &&
620 "No dominance between recurrences used by one SCEV?");
621 return -1;
622 }
623
624 [[fallthrough]];
625 }
626
627 case scTruncate:
628 case scZeroExtend:
629 case scSignExtend:
630 case scPtrToAddr:
631 case scAddExpr:
632 case scMulExpr:
633 case scUDivExpr:
634 case scSMaxExpr:
635 case scUMaxExpr:
636 case scSMinExpr:
637 case scUMinExpr:
639 ArrayRef<SCEVUse> LOps = LHS->operands();
640 ArrayRef<SCEVUse> ROps = RHS->operands();
641
642 // Lexicographically compare n-ary-like expressions.
643 unsigned LNumOps = LOps.size(), RNumOps = ROps.size();
644 if (LNumOps != RNumOps)
645 return (int)LNumOps - (int)RNumOps;
646
647 for (unsigned i = 0; i != LNumOps; ++i) {
648 auto X = compareSCEVComplexity(Opts, LI, LOps[i], ROps[i], DT);
649 if (X != 0)
650 return X;
651 }
652
653 assert(isa<SCEVCastExpr>(LHS) && "distinct SCEVs compare equal");
654 unsigned LBitWidth = cast<IntegerType>(LHS->getType())->getBitWidth();
655 unsigned RBitWidth = cast<IntegerType>(RHS->getType())->getBitWidth();
656 assert(LBitWidth != RBitWidth && "distinct SCEVs compare equal");
657 return (int)LBitWidth - (int)RBitWidth;
658 }
659
661 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
662 }
663 llvm_unreachable("Unknown SCEV kind!");
664}
665
666/// Given a list of SCEV objects, order them by their complexity, and group
667/// objects of the same complexity together by value. When this routine is
668/// finished, we know that any duplicates in the vector are consecutive and that
669/// complexity is monotonically increasing.
670///
671/// Note that we go take special precautions to ensure that we get deterministic
672/// results from this routine. In other words, we don't want the results of
673/// this to depend on where the addresses of various SCEV objects happened to
674/// land in memory.
675static void groupByComplexity(const AnalysisOptions &Opts,
677 DominatorTree &DT) {
678 if (Ops.size() < 2) return; // Noop
679
680 // Whether LHS has provably less complexity than RHS.
681 auto IsLessComplex = [&](SCEVUse LHS, SCEVUse RHS) {
682 return compareSCEVComplexity(Opts, LI, LHS, RHS, DT) < 0;
683 };
684 if (Ops.size() == 2) {
685 // This is the common case, which also happens to be trivially simple.
686 // Special case it.
687 SCEVUse &LHS = Ops[0], &RHS = Ops[1];
688 if (IsLessComplex(RHS, LHS))
689 std::swap(LHS, RHS);
690 return;
691 }
692
693 llvm::sort(Ops, IsLessComplex);
694}
695
696/// Returns true if \p Ops contains a huge SCEV (the subtree of S contains at
697/// least -scalar-evolution-huge-expr-threshold nodes).
698static bool hasHugeExpression(const AnalysisOptions &Opts,
700 return any_of(Ops, [&](const SCEV *S) {
701 return S->getExpressionSize() >= Opts.scalar_evolution_huge_expr_threshold;
702 });
703}
704
705/// Performs a number of common optimizations on the passed \p Ops. If the
706/// whole expression reduces down to a single operand, it will be returned.
707///
708/// The following optimizations are performed:
709/// * Fold constants using the \p Fold function.
710/// * Remove identity constants satisfying \p IsIdentity.
711/// * If a constant satisfies \p IsAbsorber, return it.
712/// * Sort operands by complexity.
713template <typename FoldT, typename IsIdentityT, typename IsAbsorberT>
714static const SCEV *
715constantFoldAndGroupOps(const AnalysisOptions &Opts, ScalarEvolution &SE,
716 LoopInfo &LI, DominatorTree &DT,
717 SmallVectorImpl<SCEVUse> &Ops, FoldT Fold,
718 IsIdentityT IsIdentity, IsAbsorberT IsAbsorber) {
719 const SCEVConstant *Folded = nullptr;
720 for (unsigned Idx = 0; Idx < Ops.size();) {
721 const SCEV *Op = Ops[Idx];
722 if (const auto *C = dyn_cast<SCEVConstant>(Op)) {
723 if (!Folded)
724 Folded = C;
725 else
727 SE.getConstant(Fold(Folded->getAPInt(), C->getAPInt())));
728 Ops.erase(Ops.begin() + Idx);
729 continue;
730 }
731 ++Idx;
732 }
733
734 if (Ops.empty()) {
735 assert(Folded && "Must have folded value");
736 return Folded;
737 }
738
739 if (Folded && IsAbsorber(Folded->getAPInt()))
740 return Folded;
741
742 groupByComplexity(Opts, Ops, &LI, DT);
743 if (Folded && !IsIdentity(Folded->getAPInt()))
744 Ops.insert(Ops.begin(), Folded);
745
746 return Ops.size() == 1 ? Ops[0] : nullptr;
747}
748
749//===----------------------------------------------------------------------===//
750// Simple SCEV method implementations
751//===----------------------------------------------------------------------===//
752
753/// Compute BC(It, K). The result has width W. Assume, K > 0.
754static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
755 ScalarEvolution &SE,
756 Type *ResultTy) {
757 // Handle the simplest case efficiently.
758 if (K == 1)
759 return SE.getTruncateOrZeroExtend(It, ResultTy);
760
761 // We are using the following formula for BC(It, K):
762 //
763 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
764 //
765 // Suppose, W is the bitwidth of the return value. We must be prepared for
766 // overflow. Hence, we must assure that the result of our computation is
767 // equal to the accurate one modulo 2^W. Unfortunately, division isn't
768 // safe in modular arithmetic.
769 //
770 // However, this code doesn't use exactly that formula; the formula it uses
771 // is something like the following, where T is the number of factors of 2 in
772 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
773 // exponentiation:
774 //
775 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
776 //
777 // This formula is trivially equivalent to the previous formula. However,
778 // this formula can be implemented much more efficiently. The trick is that
779 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
780 // arithmetic. To do exact division in modular arithmetic, all we have
781 // to do is multiply by the inverse. Therefore, this step can be done at
782 // width W.
783 //
784 // The next issue is how to safely do the division by 2^T. The way this
785 // is done is by doing the multiplication step at a width of at least W + T
786 // bits. This way, the bottom W+T bits of the product are accurate. Then,
787 // when we perform the division by 2^T (which is equivalent to a right shift
788 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get
789 // truncated out after the division by 2^T.
790 //
791 // In comparison to just directly using the first formula, this technique
792 // is much more efficient; using the first formula requires W * K bits,
793 // but this formula less than W + K bits. Also, the first formula requires
794 // a division step, whereas this formula only requires multiplies and shifts.
795 //
796 // It doesn't matter whether the subtraction step is done in the calculation
797 // width or the input iteration count's width; if the subtraction overflows,
798 // the result must be zero anyway. We prefer here to do it in the width of
799 // the induction variable because it helps a lot for certain cases; CodeGen
800 // isn't smart enough to ignore the overflow, which leads to much less
801 // efficient code if the width of the subtraction is wider than the native
802 // register width.
803 //
804 // (It's possible to not widen at all by pulling out factors of 2 before
805 // the multiplication; for example, K=2 can be calculated as
806 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
807 // extra arithmetic, so it's not an obvious win, and it gets
808 // much more complicated for K > 3.)
809
810 // Protection from insane SCEVs; this bound is conservative,
811 // but it probably doesn't matter.
812 if (K > 1000)
813 return SE.getCouldNotCompute();
814
815 unsigned W = SE.getTypeSizeInBits(ResultTy);
816
817 // Calculate K! / 2^T and T; we divide out the factors of two before
818 // multiplying for calculating K! / 2^T to avoid overflow.
819 // Other overflow doesn't matter because we only care about the bottom
820 // W bits of the result.
821 APInt OddFactorial(W, 1);
822 unsigned T = 1;
823 for (unsigned i = 3; i <= K; ++i) {
824 unsigned TwoFactors = countr_zero(i);
825 T += TwoFactors;
826 OddFactorial *= (i >> TwoFactors);
827 }
828
829 // We need at least W + T bits for the multiplication step
830 unsigned CalculationBits = W + T;
831
832 // Calculate 2^T, at width T+W.
833 APInt DivFactor = APInt::getOneBitSet(CalculationBits, T);
834
835 // Calculate the multiplicative inverse of K! / 2^T;
836 // this multiplication factor will perform the exact division by
837 // K! / 2^T.
838 APInt MultiplyFactor = OddFactorial.multiplicativeInverse();
839
840 // Calculate the product, at width T+W
841 IntegerType *CalculationTy = IntegerType::get(SE.getContext(),
842 CalculationBits);
843 const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy);
844 for (unsigned i = 1; i != K; ++i) {
845 const SCEV *S = SE.getMinusSCEV(It, SE.getConstant(It->getType(), i));
846 Dividend = SE.getMulExpr(Dividend,
847 SE.getTruncateOrZeroExtend(S, CalculationTy));
848 }
849
850 // Divide by 2^T
851 const SCEV *DivResult = SE.getUDivExpr(Dividend, SE.getConstant(DivFactor));
852
853 // Truncate the result, and divide by K! / 2^T.
854
855 return SE.getMulExpr(SE.getConstant(MultiplyFactor),
856 SE.getTruncateOrZeroExtend(DivResult, ResultTy));
857}
858
859/// Return the value of this chain of recurrences at the specified iteration
860/// number. We can evaluate this recurrence by multiplying each element in the
861/// chain by the binomial coefficient corresponding to it. In other words, we
862/// can evaluate {A,+,B,+,C,+,D} as:
863///
864/// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
865///
866/// where BC(It, k) stands for binomial coefficient.
868 ScalarEvolution &SE) const {
869 return evaluateAtIteration(operands(), It, SE);
870}
871
873 const SCEV *It, ScalarEvolution &SE,
874 SCEVFlags UseFlags) {
875 assert(Operands.size() > 0);
876 assert((Operands.size() == 2 || UseFlags == SCEV::FlagNone) &&
877 "use-specific flags only supported for affine AddRecs");
878 SCEVUse Result = Operands[0].getPointer();
879 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
880 // The computation is correct in the face of overflow provided that the
881 // multiplication is performed _after_ the evaluation of the binomial
882 // coefficient.
883 const SCEV *Coeff = BinomialCoefficient(It, i, SE, Result->getType());
884 if (isa<SCEVCouldNotCompute>(Coeff))
885 return Coeff;
886
887 SCEVUse Mul = SE.getMulExpr(Operands[i].getPointer(), Coeff,
888 {SCEV::FlagNone, UseFlags});
889 Result = SE.getAddExpr(Result, Mul, {SCEV::FlagNone, UseFlags});
890 }
891 return Result;
892}
893
895 const SCEV *BTC = SE.getBackedgeTakenCount(getLoop());
897 return BTC;
898 // The loop reaches iteration BTC, so the value this recurrence computes there
899 // is the value it had, and that did not wrap.
900 return evaluateAtIteration(operands(), BTC, SE,
903}
904
905//===----------------------------------------------------------------------===//
906// SCEV Expression folder implementations
907//===----------------------------------------------------------------------===//
908
909/// The SCEVCastSinkingRewriter takes a scalar evolution expression,
910/// which computes a pointer-typed value, and rewrites the whole expression
911/// tree so that *all* the computations are done on integers, and the only
912/// pointer-typed operands in the expression are SCEVUnknown.
913/// The CreatePtrCast callback is invoked to create the actual conversion
914/// (ptrtoint or ptrtoaddr) at the SCEVUnknown leaves.
916 : public SCEVRewriteVisitor<SCEVCastSinkingRewriter> {
918 using ConversionFn = function_ref<const SCEV *(const SCEVUnknown *)>;
919 Type *TargetTy;
920 ConversionFn CreatePtrCast;
921
922public:
924 ConversionFn CreatePtrCast)
925 : Base(SE), TargetTy(TargetTy), CreatePtrCast(std::move(CreatePtrCast)) {}
926
927 static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE,
928 Type *TargetTy, ConversionFn CreatePtrCast) {
929 SCEVCastSinkingRewriter Rewriter(SE, TargetTy, std::move(CreatePtrCast));
930 return Rewriter.visit(Scev);
931 }
932
933 const SCEV *visit(const SCEV *S) {
934 Type *STy = S->getType();
935 // If the expression is not pointer-typed, just keep it as-is.
936 if (!STy->isPointerTy())
937 return S;
938 // Else, recursively sink the cast down into it.
939 return Base::visit(S);
940 }
941
942 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
943 // Preserve wrap flags on rewritten SCEVAddExpr, which the default
944 // implementation drops.
946 bool Changed = false;
947 for (SCEVUse Op : Expr->operands()) {
948 Operands.push_back(visit(Op.getPointer()));
949 Changed |= Op.getPointer() != Operands.back();
950 }
951 return !Changed ? Expr : SE.getAddExpr(Operands, Expr->getNoWrapFlags());
952 }
953
954 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
955 assert(Expr->getType()->isPointerTy() &&
956 "Should only reach pointer-typed SCEVUnknown's.");
957 // Perform some basic constant folding. If the operand of the cast is a
958 // null pointer, don't create a cast SCEV expression (that will be left
959 // as-is), but produce a zero constant.
961 return SE.getZero(TargetTy);
962 return CreatePtrCast(Expr);
963 }
964};
965
967 assert(Op->getType()->isPointerTy() && "Op must be a pointer");
968
969 // Treat pointers with unstable representation conservatively, since the
970 // address bits may change.
971 if (DL.hasUnstableRepresentation(Op->getType()))
972 return getCouldNotCompute();
973
974 Type *Ty = DL.getAddressType(Op->getType());
975
976 // Use the rewriter to sink the cast down to SCEVUnknown leaves.
977 // The rewriter handles null pointer constant folding.
979 Op, *this, Ty, [this, Ty](const SCEVUnknown *U) {
982 ID.AddPointer(U);
983 ID.AddPointer(Ty);
985 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
986 return S;
987 SCEV *S = new (SCEVAllocator)
988 SCEVPtrToAddrExpr(ID.Intern(SCEVAllocator), U, Ty);
989 UniqueSCEVs.insert(S, Token);
990 S->computeAndSetCanonical(*this);
991 registerUser(S, {U});
992 return static_cast<const SCEV *>(S);
993 });
994 assert(IntOp->getType()->isIntegerTy() &&
995 "We must have succeeded in sinking the cast, "
996 "and ending up with an integer-typed expression!");
997 return IntOp;
998}
999
1001 unsigned Depth) {
1002 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
1003 "This is not a truncating conversion!");
1004 assert(isSCEVable(Ty) &&
1005 "This is not a conversion to a SCEVable type!");
1006 assert(!Op->getType()->isPointerTy() && "Can't truncate pointer!");
1007 Ty = getEffectiveSCEVType(Ty);
1008
1011 ID.AddPointer(Op.getOpaqueValue());
1012 ID.AddPointer(Ty);
1014 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1015 return S;
1016
1017 // Fold if the operand is constant.
1018 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
1019 return getConstant(
1020 cast<ConstantInt>(ConstantExpr::getTrunc(SC->getValue(), Ty)));
1021
1022 // trunc(trunc(x)) --> trunc(x)
1024 return getTruncateExpr(ST->getOperand(), Ty, Depth + 1);
1025
1026 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
1028 return getTruncateOrSignExtend(SS->getOperand(), Ty, Depth + 1);
1029
1030 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
1032 return getTruncateOrZeroExtend(SZ->getOperand(), Ty, Depth + 1);
1033
1034 if (Depth > Opts.scalar_evolution_max_cast_depth) {
1035 SCEV *S =
1036 new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), Op, Ty);
1037 UniqueSCEVs.insert(S, Token);
1038 S->computeAndSetCanonical(*this);
1039 registerUser(S, Op);
1040 return S;
1041 }
1042
1043 // trunc(x1 + ... + xN) --> trunc(x1) + ... + trunc(xN) and
1044 // trunc(x1 * ... * xN) --> trunc(x1) * ... * trunc(xN),
1045 // if after transforming we have at most one truncate, not counting truncates
1046 // that replace other casts.
1048 auto *CommOp = cast<SCEVCommutativeExpr>(Op);
1050 unsigned numTruncs = 0;
1051 for (unsigned i = 0, e = CommOp->getNumOperands(); i != e && numTruncs < 2;
1052 ++i) {
1053 const SCEV *S = getTruncateExpr(CommOp->getOperand(i), Ty, Depth + 1);
1054 if (!isa<SCEVIntegralCastExpr>(CommOp->getOperand(i)) &&
1056 numTruncs++;
1057 Operands.push_back(S);
1058 }
1059 if (numTruncs < 2) {
1060 if (isa<SCEVAddExpr>(Op))
1061 return getAddExpr(Operands);
1062 if (isa<SCEVMulExpr>(Op))
1063 return getMulExpr(Operands);
1064 llvm_unreachable("Unexpected SCEV type for Op.");
1065 }
1066 // Although we checked in the beginning that ID is not in the cache, it is
1067 // possible that during recursion and different modification ID was inserted
1068 // into the cache. So if we find it, just return it.
1069 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1070 return S;
1071 }
1072
1073 // If the input value is a chrec scev, truncate the chrec's operands.
1074 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) {
1076 for (const SCEV *Op : AddRec->operands())
1077 Operands.push_back(getTruncateExpr(Op, Ty, Depth + 1));
1078 return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagNone);
1079 }
1080
1081 // Return zero if truncating to known zeros.
1082 uint32_t MinTrailingZeros = getMinTrailingZeros(Op);
1083 if (MinTrailingZeros >= getTypeSizeInBits(Ty))
1084 return getZero(Ty);
1085
1086 // The cast wasn't folded; create an explicit cast node. We can reuse
1087 // the existing insert position since if we get here, we won't have
1088 // made any changes which would invalidate it.
1089 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator),
1090 Op, Ty);
1091 UniqueSCEVs.insert(S, Token);
1092 S->computeAndSetCanonical(*this);
1093 registerUser(S, Op);
1094 return S;
1095}
1096
1097// Get the limit of a recurrence such that incrementing by Step cannot cause
1098// signed overflow as long as the value of the recurrence within the
1099// loop does not exceed this limit before incrementing.
1100static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step,
1102 ScalarEvolution *SE) {
1103 unsigned BitWidth = SE->getTypeSizeInBits(Step->getType());
1104 if (SE->isKnownPositive(Step)) {
1107 SE->getSignedRangeMax(Step));
1108 }
1109 if (SE->isKnownNegative(Step)) {
1112 SE->getSignedRangeMin(Step));
1113 }
1114 return nullptr;
1115}
1116
1117// Get the limit of a recurrence such that incrementing by Step cannot cause
1118// unsigned overflow as long as the value of the recurrence within the loop does
1119// not exceed this limit before incrementing.
1122 ScalarEvolution *SE) {
1123 unsigned BitWidth = SE->getTypeSizeInBits(Step->getType());
1125
1127 SE->getUnsignedRangeMax(Step));
1128}
1129
1130namespace {
1131
1132struct ExtendOpTraitsBase {
1133 typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(SCEVUse, Type *,
1134 unsigned);
1135};
1136
1137// Used to make code generic over signed and unsigned overflow.
1138template <typename ExtendOp> struct ExtendOpTraits {
1139 // Members present:
1140 //
1141 // static const SCEVFlags WrapType;
1142 //
1143 // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr;
1144 //
1145 // static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1146 // ICmpInst::Predicate *Pred,
1147 // ScalarEvolution *SE);
1148};
1149
1150template <>
1151struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase {
1152 static const SCEVFlags WrapType = SCEV::FlagNSW;
1153
1154 static const GetExtendExprTy GetExtendExpr;
1155
1156 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1157 ICmpInst::Predicate *Pred,
1158 ScalarEvolution *SE) {
1159 return getSignedOverflowLimitForStep(Step, Pred, SE);
1160 }
1161};
1162
1163const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1165
1166template <>
1167struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase {
1168 static const SCEVFlags WrapType = SCEV::FlagNUW;
1169
1170 static const GetExtendExprTy GetExtendExpr;
1171
1172 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1173 ICmpInst::Predicate *Pred,
1174 ScalarEvolution *SE) {
1175 return getUnsignedOverflowLimitForStep(Step, Pred, SE);
1176 }
1177};
1178
1179const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1181
1182} // end anonymous namespace
1183
1184// The recurrence AR has been shown to have no signed/unsigned wrap or something
1185// close to it. Typically, if we can prove NSW/NUW for AR, then we can just as
1186// easily prove NSW/NUW for its preincrement or postincrement sibling. This
1187// allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step +
1188// Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the
1189// expression "Step + sext/zext(PreIncAR)" is congruent with
1190// "sext/zext(PostIncAR)"
1191template <typename ExtendOpTy>
1193 ScalarEvolution *SE, unsigned Depth) {
1194 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1195 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1196
1197 const Loop *L = AR->getLoop();
1198 const SCEV *Start = AR->getStart();
1199 const SCEV *Step = AR->getStepRecurrence(*SE);
1200
1201 // Check for a simple looking step prior to loop entry.
1202 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Start);
1203 if (!SA)
1204 return nullptr;
1205
1206 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
1207 // subtraction is expensive. For this purpose, perform a quick and dirty
1208 // difference, by checking for Step in the operand list. Note, that
1209 // SA might have repeated ops, like %a + %a + ..., so only remove one.
1210 SmallVector<SCEVUse, 4> DiffOps(SA->operands());
1211 for (auto It = DiffOps.begin(); It != DiffOps.end(); ++It)
1212 if (*It == Step) {
1213 DiffOps.erase(It);
1214 break;
1215 }
1216
1217 if (DiffOps.size() == SA->getNumOperands())
1218 return nullptr;
1219
1220 // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` +
1221 // `Step`:
1222
1223 // 1. NSW/NUW flags on the step increment.
1224 auto PreStartFlags =
1226 const SCEV *PreStart = SE->getAddExpr(DiffOps, PreStartFlags);
1228 SE->getAddRecExpr(PreStart, Step, L, SCEV::FlagNone));
1229
1230 // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies
1231 // "S+X does not sign/unsign-overflow".
1232 //
1233
1234 const SCEV *BECount = SE->getBackedgeTakenCount(L);
1235 if (PreAR && any(PreAR->getNoWrapFlags(WrapType)) &&
1236 !isa<SCEVCouldNotCompute>(BECount) && SE->isKnownPositive(BECount))
1237 return PreStart;
1238
1239 // 2. Direct overflow check on the step operation's expression.
1240 unsigned BitWidth = SE->getTypeSizeInBits(AR->getType());
1241 Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2);
1242 const SCEV *OperandExtendedStart =
1243 SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy, Depth),
1244 (SE->*GetExtendExpr)(Step, WideTy, Depth));
1245 if ((SE->*GetExtendExpr)(Start, WideTy, Depth) == OperandExtendedStart) {
1246 if (PreAR && any(AR->getNoWrapFlags(WrapType))) {
1247 // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW
1248 // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then
1249 // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`. Cache this fact.
1250 SE->setNoWrapFlags(const_cast<SCEVAddRecExpr *>(PreAR), WrapType);
1251 }
1252 return PreStart;
1253 }
1254
1255 // 3. Loop precondition.
1257 const SCEV *OverflowLimit =
1258 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
1259
1260 if (OverflowLimit &&
1261 SE->isLoopEntryGuardedByCond(L, Pred, PreStart, OverflowLimit))
1262 return PreStart;
1263
1264 return nullptr;
1265}
1266
1267// Get the normalized zero or sign extended expression for this AddRec's Start.
1268template <typename ExtendOpTy>
1269static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty,
1270 ScalarEvolution *SE,
1271 unsigned Depth) {
1272 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1273
1274 const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, SE, Depth);
1275 if (!PreStart)
1276 return (SE->*GetExtendExpr)(AR->getStart(), Ty, Depth);
1277
1278 return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(*SE), Ty,
1279 Depth),
1280 (SE->*GetExtendExpr)(PreStart, Ty, Depth));
1281}
1282
1283// Try to prove away overflow by looking at "nearby" add recurrences. A
1284// motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it
1285// does not itself wrap then we can conclude that `{1,+,4}` is `nuw`.
1286//
1287// Formally:
1288//
1289// {S,+,X} == {S-T,+,X} + T
1290// => Ext({S,+,X}) == Ext({S-T,+,X} + T)
1291//
1292// If ({S-T,+,X} + T) does not overflow ... (1)
1293//
1294// RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T)
1295//
1296// If {S-T,+,X} does not overflow ... (2)
1297//
1298// RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T)
1299// == {Ext(S-T)+Ext(T),+,Ext(X)}
1300//
1301// If (S-T)+T does not overflow ... (3)
1302//
1303// RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)}
1304// == {Ext(S),+,Ext(X)} == LHS
1305//
1306// Thus, if (1), (2) and (3) are true for some T, then
1307// Ext({S,+,X}) == {Ext(S),+,Ext(X)}
1308//
1309// (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T)
1310// does not overflow" restricted to the 0th iteration. Therefore we only need
1311// to check for (1) and (2).
1312//
1313// In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T
1314// is `Delta` (defined below).
1315template <typename ExtendOpTy>
1316bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
1317 const SCEV *Step,
1318 const Loop *L) {
1319 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1320
1321 // We restrict `Start` to a constant to prevent SCEV from spending too much
1322 // time here. It is correct (but more expensive) to continue with a
1323 // non-constant `Start` and do a general SCEV subtraction to compute
1324 // `PreStart` below.
1325 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start);
1326 if (!StartC)
1327 return false;
1328
1329 APInt StartAI = StartC->getAPInt();
1330
1331 for (unsigned Delta : {-2, -1, 1, 2}) {
1332 const SCEV *PreStart = getConstant(StartAI - Delta);
1333 const auto *PreAR = static_cast<SCEVAddRecExpr *>(
1334 findExistingSCEVInCache(scAddRecExpr, {PreStart, Step}, L));
1335
1336 // Give up if we don't already have the add recurrence we need because
1337 // actually constructing an add recurrence is relatively expensive.
1338 if (PreAR && any(PreAR->getNoWrapFlags(WrapType))) { // proves (2)
1339 const SCEV *DeltaS = getConstant(StartC->getType(), Delta);
1341 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
1342 DeltaS, &Pred, this);
1343 if (Limit && isKnownPredicate(Pred, PreAR, Limit)) // proves (1)
1344 return true;
1345 }
1346 }
1347
1348 return false;
1349}
1350
1351// Finds an integer D for an expression (C + x + y + ...) such that the top
1352// level addition in (D + (C - D + x + y + ...)) would not wrap (signed or
1353// unsigned) and the number of trailing zeros of (C - D + x + y + ...) is
1354// maximized, where C is the \p ConstantTerm, x, y, ... are arbitrary SCEVs, and
1355// the (C + x + y + ...) expression is \p WholeAddExpr.
1357 const SCEVConstant *ConstantTerm,
1358 const SCEVAddExpr *WholeAddExpr) {
1359 const APInt &C = ConstantTerm->getAPInt();
1360 const unsigned BitWidth = C.getBitWidth();
1361 // Find number of trailing zeros of (x + y + ...) w/o the C first:
1362 uint32_t TZ = BitWidth;
1363 for (unsigned I = 1, E = WholeAddExpr->getNumOperands(); I < E && TZ; ++I)
1364 TZ = std::min(TZ, SE.getMinTrailingZeros(WholeAddExpr->getOperand(I)));
1365 if (TZ) {
1366 // Set D to be as many least significant bits of C as possible while still
1367 // guaranteeing that adding D to (C - D + x + y + ...) won't cause a wrap:
1368 return TZ < BitWidth ? C.trunc(TZ).zext(BitWidth) : C;
1369 }
1370 return APInt(BitWidth, 0);
1371}
1372
1373// Finds an integer D for an affine AddRec expression {C,+,x} such that the top
1374// level addition in (D + {C-D,+,x}) would not wrap (signed or unsigned) and the
1375// number of trailing zeros of (C - D + x * n) is maximized, where C is the \p
1376// ConstantStart, x is an arbitrary \p Step, and n is the loop trip count.
1378 const APInt &ConstantStart,
1379 const SCEV *Step) {
1380 const unsigned BitWidth = ConstantStart.getBitWidth();
1381 const uint32_t TZ = SE.getMinTrailingZeros(Step);
1382 if (TZ)
1383 return TZ < BitWidth ? ConstantStart.trunc(TZ).zext(BitWidth)
1384 : ConstantStart;
1385 return APInt(BitWidth, 0);
1386}
1387
1389 const ScalarEvolution::FoldID &ID, const SCEV *S,
1392 &FoldCacheUser) {
1393 auto I = FoldCache.insert({ID, S});
1394 if (!I.second) {
1395 // Remove FoldCacheUser entry for ID when replacing an existing FoldCache
1396 // entry.
1397 auto &UserIDs = FoldCacheUser[I.first->second];
1398 assert(count(UserIDs, ID) == 1 && "unexpected duplicates in UserIDs");
1399 for (unsigned I = 0; I != UserIDs.size(); ++I)
1400 if (UserIDs[I] == ID) {
1401 std::swap(UserIDs[I], UserIDs.back());
1402 break;
1403 }
1404 UserIDs.pop_back();
1405 I.first->second = S;
1406 }
1407 FoldCacheUser[S].push_back(ID);
1408}
1409
1411 unsigned Depth) {
1412 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1413 "This is not an extending conversion!");
1414 assert(isSCEVable(Ty) &&
1415 "This is not a conversion to a SCEVable type!");
1416 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1417 Ty = getEffectiveSCEVType(Ty);
1418
1419 FoldID ID(scZeroExtend, Op, Ty);
1420 if (const SCEV *S = FoldCache.lookup(ID))
1421 return S;
1422
1423 const SCEV *S = getZeroExtendExprImpl(Op, Ty, Depth);
1425 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1426 return S;
1427}
1428
1430 unsigned Depth) {
1431 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1432 "This is not an extending conversion!");
1433 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1434 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1435
1436 // Fold if the operand is constant.
1437 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
1438 return getConstant(SC->getAPInt().zext(getTypeSizeInBits(Ty)));
1439
1440 // zext(zext(x)) --> zext(x)
1442 return getZeroExtendExpr(SZ->getOperand(), Ty, Depth + 1);
1443
1444 // If the operand is an affine AddRec with the no-unsigned-wrap flag, the
1445 // zero-extension distributes over the recurrence.
1446 const SCEV *Start, *Step;
1447 const Loop *L;
1448 if (Depth <= Opts.scalar_evolution_max_cast_depth &&
1449 match(Op, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step), m_Loop(L)))) {
1450 const auto *AR = cast<SCEVAddRecExpr>(Op);
1451 if (AR->hasNoUnsignedWrap()) {
1452 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1);
1453 Step = getZeroExtendExpr(Step, Ty, Depth + 1);
1454 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1455 }
1456 }
1457
1458 // Before doing any expensive analysis, check to see if we've already
1459 // computed a SCEV for this Op and Ty.
1462 ID.AddPointer(Op.getOpaqueValue());
1463 ID.AddPointer(Ty);
1465 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1466 return S;
1467 if (Depth > Opts.scalar_evolution_max_cast_depth) {
1468 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
1469 Op, Ty);
1470 UniqueSCEVs.insert(S, Token);
1471 S->computeAndSetCanonical(*this);
1472 registerUser(S, Op);
1473 return S;
1474 }
1475
1476 // zext(trunc(x)) --> zext(x) or x or trunc(x)
1478 // It's possible the bits taken off by the truncate were all zero bits. If
1479 // so, we should be able to simplify this further.
1480 const SCEV *X = ST->getOperand();
1482 unsigned TruncBits = getTypeSizeInBits(ST->getType());
1483 unsigned NewBits = getTypeSizeInBits(Ty);
1484 if (CR.truncate(TruncBits).zeroExtend(NewBits).contains(
1485 CR.zextOrTrunc(NewBits)))
1486 return getTruncateOrZeroExtend(X, Ty, Depth);
1487 }
1488
1489 // If the input value is a chrec scev, and we can prove that the value
1490 // did not overflow the old, smaller, value, we can zero extend all of the
1491 // operands (often constants). This allows analysis of something like
1492 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
1493 if (match(Op, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step), m_Loop(L)))) {
1494 const auto *AR = cast<SCEVAddRecExpr>(Op);
1495 unsigned BitWidth = getTypeSizeInBits(AR->getType());
1496
1497 // The no-unsigned-wrap case is handled before the uniquing lookup above.
1498
1499 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1500 // Note that this serves two purposes: It filters out loops that are
1501 // simply not analyzable, and it covers the case where this code is
1502 // being called from within backedge-taken count analysis, such that
1503 // attempting to ask for the backedge-taken count would likely result
1504 // in infinite recursion. In the later case, the analysis code will
1505 // cope with a conservative value, and it will take care to purge
1506 // that value once it has finished.
1507 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
1508 if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
1509 // Manually compute the final value for AR, checking for overflow.
1510
1511 // Check whether the backedge-taken count can be losslessly casted to
1512 // the addrec's type. The count is always unsigned.
1513 const SCEV *CastedMaxBECount =
1514 getTruncateOrZeroExtend(MaxBECount, Start->getType(), Depth);
1515 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
1516 CastedMaxBECount, MaxBECount->getType(), Depth);
1517 if (MaxBECount == RecastedMaxBECount) {
1518 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
1519 // Check whether Start+Step*MaxBECount has no unsigned overflow.
1520 const SCEV *ZMul =
1521 getMulExpr(CastedMaxBECount, Step, SCEV::FlagNone, Depth + 1);
1522 const SCEV *ZAdd = getZeroExtendExpr(
1523 getAddExpr(Start, ZMul, SCEV::FlagNone, Depth + 1), WideTy,
1524 Depth + 1);
1525 const SCEV *WideStart = getZeroExtendExpr(Start, WideTy, Depth + 1);
1526 const SCEV *WideMaxBECount =
1527 getZeroExtendExpr(CastedMaxBECount, WideTy, Depth + 1);
1528 const SCEV *OperandExtendedAdd =
1529 getAddExpr(WideStart,
1530 getMulExpr(WideMaxBECount,
1531 getZeroExtendExpr(Step, WideTy, Depth + 1),
1532 SCEV::FlagNone, Depth + 1),
1533 SCEV::FlagNone, Depth + 1);
1534 if (ZAdd == OperandExtendedAdd) {
1535 // Cache knowledge of AR NUW, which is propagated to this AddRec.
1536 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNUW);
1537 // Return the expression with the addrec on the outside.
1538 Start =
1540 Step = getZeroExtendExpr(Step, Ty, Depth + 1);
1541 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1542 }
1543 // Similar to above, only this time treat the step value as signed.
1544 // This covers loops that count down.
1545 OperandExtendedAdd =
1546 getAddExpr(WideStart,
1547 getMulExpr(WideMaxBECount,
1548 getSignExtendExpr(Step, WideTy, Depth + 1),
1549 SCEV::FlagNone, Depth + 1),
1550 SCEV::FlagNone, Depth + 1);
1551 if (ZAdd == OperandExtendedAdd) {
1552 // Cache knowledge of AR NW, which is propagated to this AddRec.
1553 // Negative step causes unsigned wrap, but it still can't self-wrap.
1554 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW);
1555 // Return the expression with the addrec on the outside.
1556 Start =
1558 Step = getSignExtendExpr(Step, Ty, Depth + 1);
1559 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1560 }
1561 }
1562 }
1563
1564 // Normally, in the cases we can prove no-overflow via a
1565 // backedge guarding condition, we can also compute a backedge
1566 // taken count for the loop. The exceptions are assumptions and
1567 // guards present in the loop -- SCEV is not great at exploiting
1568 // these to compute max backedge taken counts, but can still use
1569 // these to prove lack of overflow. Use this fact to avoid
1570 // doing extra work that may not pay off.
1571 if (!isa<SCEVCouldNotCompute>(MaxBECount) || HasGuards ||
1572 !AC.assumptions().empty()) {
1573
1574 auto NewFlags = proveNoUnsignedWrapViaInduction(AR);
1575 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), NewFlags);
1576 if (AR->hasNoUnsignedWrap()) {
1577 // Same as nuw case above - duplicated here to avoid a compile time
1578 // issue. It's not clear that the order of checks does matter, but
1579 // it's one of two issue possible causes for a change which was
1580 // reverted. Be conservative for the moment.
1581 Start =
1583 Step = getZeroExtendExpr(Step, Ty, Depth + 1);
1584 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1585 }
1586
1587 // For a negative step, we can extend the operands iff doing so only
1588 // traverses values in the range zext([0,UINT_MAX]).
1589 if (isKnownNegative(Step)) {
1590 const SCEV *N =
1594 // Cache knowledge of AR NW, which is propagated to this
1595 // AddRec. Negative step causes unsigned wrap, but it
1596 // still can't self-wrap.
1597 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW);
1598 // Return the expression with the addrec on the outside.
1599 Start =
1601 Step = getSignExtendExpr(Step, Ty, Depth + 1);
1602 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1603 }
1604 }
1605 }
1606
1607 // zext({C,+,Step}) --> (zext(D) + zext({C-D,+,Step}))<nuw><nsw>
1608 // if D + (C - D + Step * n) could be proven to not unsigned wrap
1609 // where D maximizes the number of trailing zeros of (C - D + Step * n)
1610 if (const auto *SC = dyn_cast<SCEVConstant>(Start)) {
1611 const APInt &C = SC->getAPInt();
1612 const APInt &D = extractConstantWithoutWrapping(*this, C, Step);
1613 if (D != 0) {
1614 const SCEV *SZExtD = getZeroExtendExpr(getConstant(D), Ty, Depth);
1615 const SCEV *SResidual =
1616 getAddRecExpr(getConstant(C - D), Step, L, AR->getNoWrapFlags());
1617 const SCEV *SZExtR = getZeroExtendExpr(SResidual, Ty, Depth + 1);
1618 return getAddExpr(SZExtD, SZExtR, SCEV::FlagNSW | SCEV::FlagNUW,
1619 Depth + 1);
1620 }
1621 }
1622
1623 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
1624 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNUW);
1625 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1);
1626 Step = getZeroExtendExpr(Step, Ty, Depth + 1);
1627 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1628 }
1629 }
1630
1631 // zext(A % B) --> zext(A) % zext(B)
1632 {
1633 const SCEV *LHS;
1634 const SCEV *RHS;
1635 if (match(Op, m_scev_URem(m_SCEV(LHS), m_SCEV(RHS), *this)))
1636 return getURemExpr(getZeroExtendExpr(LHS, Ty, Depth + 1),
1637 getZeroExtendExpr(RHS, Ty, Depth + 1));
1638 }
1639
1640 // zext(A / B) --> zext(A) / zext(B).
1641 if (auto *Div = dyn_cast<SCEVUDivExpr>(Op))
1642 return getUDivExpr(getZeroExtendExpr(Div->getLHS(), Ty, Depth + 1),
1643 getZeroExtendExpr(Div->getRHS(), Ty, Depth + 1));
1644
1645 if (auto *SA = dyn_cast<SCEVAddExpr>(Op)) {
1646 // zext((A + B + ...)<nuw>) --> (zext(A) + zext(B) + ...)<nuw>
1647 if (SA->hasNoUnsignedWrap()) {
1648 // If the addition does not unsign overflow then we can, by definition,
1649 // commute the zero extension with the addition operation.
1651 for (SCEVUse Op : SA->operands())
1652 Ops.push_back(getZeroExtendExpr(Op, Ty, Depth + 1));
1653 return getAddExpr(Ops, SCEV::FlagNUW, Depth + 1);
1654 }
1655
1656 const APInt *C, *C2;
1657 // zext (C + A)<nsw> -> (sext(C) + sext(A))<nsw> if zext (C + A)<nsw> >=s 0.
1658 // Currently the non-negative check is done manually, as isKnownNonNegative
1659 // is too expensive.
1660 if (SA->hasNoSignedWrap() &&
1662 m_scev_SMax(m_scev_APInt(C2), m_SCEV()))) &&
1663 C->isNegative() && !C->isMinSignedValue() && C2->sge(C->abs())) {
1664 assert(isKnownNonNegative(SA) && "incorrectly determined non-negative");
1665 return getAddExpr(getSignExtendExpr(SA->getOperand(0), Ty, Depth + 1),
1666 getSignExtendExpr(SA->getOperand(1), Ty, Depth + 1),
1667 SCEV::FlagNSW, Depth + 1);
1668 }
1669
1670 // zext(C + x + y + ...) --> (zext(D) + zext((C - D) + x + y + ...))
1671 // if D + (C - D + x + y + ...) could be proven to not unsigned wrap
1672 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1673 //
1674 // Often address arithmetics contain expressions like
1675 // (zext (add (shl X, C1), C2)), for instance, (zext (5 + (4 * X))).
1676 // This transformation is useful while proving that such expressions are
1677 // equal or differ by a small constant amount, see LoadStoreVectorizer pass.
1678 if (const auto *SC = dyn_cast<SCEVConstant>(SA->getOperand(0))) {
1679 const APInt &D = extractConstantWithoutWrapping(*this, SC, SA);
1680 if (D != 0) {
1681 const SCEV *SZExtD = getZeroExtendExpr(getConstant(D), Ty, Depth);
1682 const SCEV *SResidual =
1684 const SCEV *SZExtR = getZeroExtendExpr(SResidual, Ty, Depth + 1);
1685 return getAddExpr(SZExtD, SZExtR, (SCEV::FlagNSW | SCEV::FlagNUW),
1686 Depth + 1);
1687 }
1688 }
1689 }
1690
1691 if (auto *SM = dyn_cast<SCEVMulExpr>(Op)) {
1692 // zext((A * B * ...)<nuw>) --> (zext(A) * zext(B) * ...)<nuw>
1693 if (SM->hasNoUnsignedWrap()) {
1694 // If the multiply does not unsign overflow then we can, by definition,
1695 // commute the zero extension with the multiply operation.
1697 for (SCEVUse Op : SM->operands())
1698 Ops.push_back(getZeroExtendExpr(Op, Ty, Depth + 1));
1699 return getMulExpr(Ops, SCEV::FlagNUW, Depth + 1);
1700 }
1701
1702 // zext(2^K * (trunc X to iN)) to iM ->
1703 // 2^K * (zext(trunc X to i{N-K}) to iM)<nuw>
1704 //
1705 // Proof:
1706 //
1707 // zext(2^K * (trunc X to iN)) to iM
1708 // = zext((trunc X to iN) << K) to iM
1709 // = zext((trunc X to i{N-K}) << K)<nuw> to iM
1710 // (because shl removes the top K bits)
1711 // = zext((2^K * (trunc X to i{N-K}))<nuw>) to iM
1712 // = (2^K * (zext(trunc X to i{N-K}) to iM))<nuw>.
1713 //
1714 const APInt *C;
1715 const SCEV *TruncRHS;
1716 if (match(SM,
1717 m_scev_Mul(m_scev_APInt(C), m_scev_Trunc(m_SCEV(TruncRHS)))) &&
1718 C->isPowerOf2()) {
1719 int NewTruncBits =
1720 getTypeSizeInBits(SM->getOperand(1)->getType()) - C->logBase2();
1721 Type *NewTruncTy = IntegerType::get(getContext(), NewTruncBits);
1722 return getMulExpr(
1723 getZeroExtendExpr(SM->getOperand(0), Ty),
1724 getZeroExtendExpr(getTruncateExpr(TruncRHS, NewTruncTy), Ty),
1725 SCEV::FlagNUW, Depth + 1);
1726 }
1727 }
1728
1729 // zext(umin(x, y)) -> umin(zext(x), zext(y))
1730 // zext(umax(x, y)) -> umax(zext(x), zext(y))
1734 for (SCEVUse Operand : MinMax->operands())
1735 Operands.push_back(getZeroExtendExpr(Operand, Ty));
1737 return getUMinExpr(Operands);
1738 return getUMaxExpr(Operands);
1739 }
1740
1741 // zext(umin_seq(x, y)) -> umin_seq(zext(x), zext(y))
1743 assert(isa<SCEVSequentialUMinExpr>(MinMax) && "Not supported!");
1745 for (SCEVUse Operand : MinMax->operands())
1746 Operands.push_back(getZeroExtendExpr(Operand, Ty));
1747 return getUMinExpr(Operands, /*Sequential*/ true);
1748 }
1749
1750 // The cast wasn't folded; create an explicit cast node.
1751 // Recompute the insert position, as it may have been invalidated.
1752 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1753 return S;
1754 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
1755 Op, Ty);
1756 UniqueSCEVs.insert(S, Token);
1757 S->computeAndSetCanonical(*this);
1758 registerUser(S, Op);
1759 return S;
1760}
1761
1763 unsigned Depth) {
1764 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1765 "This is not an extending conversion!");
1766 assert(isSCEVable(Ty) &&
1767 "This is not a conversion to a SCEVable type!");
1768 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1769 Ty = getEffectiveSCEVType(Ty);
1770
1771 FoldID ID(scSignExtend, Op, Ty);
1772 if (const SCEV *S = FoldCache.lookup(ID))
1773 return S;
1774
1775 const SCEV *S = getSignExtendExprImpl(Op, Ty, Depth);
1777 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1778 return S;
1779}
1780
1782 unsigned Depth) {
1783 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1784 "This is not an extending conversion!");
1785 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1786 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1787 Ty = getEffectiveSCEVType(Ty);
1788
1789 // Fold if the operand is constant.
1790 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
1791 return getConstant(SC->getAPInt().sext(getTypeSizeInBits(Ty)));
1792
1793 // sext(sext(x)) --> sext(x)
1795 return getSignExtendExpr(SS->getOperand(), Ty, Depth + 1);
1796
1797 // sext(zext(x)) --> zext(x)
1799 return getZeroExtendExpr(SZ->getOperand(), Ty, Depth + 1);
1800
1801 // If the operand is an affine AddRec with the no-signed-wrap flag, the
1802 // sign-extension distributes over the recurrence.
1803 const SCEV *Start, *Step;
1804 const Loop *L;
1805 if (Depth <= Opts.scalar_evolution_max_cast_depth &&
1806 match(Op, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step), m_Loop(L)))) {
1807 const auto *AR = cast<SCEVAddRecExpr>(Op);
1808 if (AR->hasNoSignedWrap()) {
1809 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1);
1810 Step = getSignExtendExpr(Step, Ty, Depth + 1);
1811 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1812 }
1813 }
1814
1815 // Before doing any expensive analysis, check to see if we've already
1816 // computed a SCEV for this Op and Ty.
1819 ID.AddPointer(Op.getOpaqueValue());
1820 ID.AddPointer(Ty);
1822 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1823 return S;
1824 // Limit recursion depth.
1825 if (Depth > Opts.scalar_evolution_max_cast_depth) {
1826 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
1827 Op, Ty);
1828 UniqueSCEVs.insert(S, Token);
1829 S->computeAndSetCanonical(*this);
1830 registerUser(S, Op);
1831 return S;
1832 }
1833
1834 // sext(trunc(x)) --> sext(x) or x or trunc(x)
1836 // It's possible the bits taken off by the truncate were all sign bits. If
1837 // so, we should be able to simplify this further.
1838 const SCEV *X = ST->getOperand();
1840 unsigned TruncBits = getTypeSizeInBits(ST->getType());
1841 unsigned NewBits = getTypeSizeInBits(Ty);
1842 if (CR.truncate(TruncBits).signExtend(NewBits).contains(
1843 CR.sextOrTrunc(NewBits)))
1844 return getTruncateOrSignExtend(X, Ty, Depth);
1845 }
1846
1847 if (auto *SA = dyn_cast<SCEVAddExpr>(Op)) {
1848 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
1849 if (SA->hasNoSignedWrap()) {
1850 // If the addition does not sign overflow then we can, by definition,
1851 // commute the sign extension with the addition operation.
1853 for (SCEVUse Op : SA->operands())
1854 Ops.push_back(getSignExtendExpr(Op, Ty, Depth + 1));
1855 return getAddExpr(Ops, SCEV::FlagNSW, Depth + 1);
1856 }
1857
1858 // sext(C + x + y + ...) --> (sext(D) + sext((C - D) + x + y + ...))
1859 // if D + (C - D + x + y + ...) could be proven to not signed wrap
1860 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1861 //
1862 // For instance, this will bring two seemingly different expressions:
1863 // 1 + sext(5 + 20 * %x + 24 * %y) and
1864 // sext(6 + 20 * %x + 24 * %y)
1865 // to the same form:
1866 // 2 + sext(4 + 20 * %x + 24 * %y)
1867 if (const auto *SC = dyn_cast<SCEVConstant>(SA->getOperand(0))) {
1868 const APInt &D = extractConstantWithoutWrapping(*this, SC, SA);
1869 if (D != 0) {
1870 const SCEV *SSExtD = getSignExtendExpr(getConstant(D), Ty, Depth);
1871 const SCEV *SResidual =
1873 const SCEV *SSExtR = getSignExtendExpr(SResidual, Ty, Depth + 1);
1874 return getAddExpr(SSExtD, SSExtR, (SCEV::FlagNSW | SCEV::FlagNUW),
1875 Depth + 1);
1876 }
1877 }
1878 }
1879 // If the input value is a chrec scev, and we can prove that the value
1880 // did not overflow the old, smaller, value, we can sign extend all of the
1881 // operands (often constants). This allows analysis of something like
1882 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; }
1883 if (match(Op, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step), m_Loop(L)))) {
1884 const auto *AR = cast<SCEVAddRecExpr>(Op);
1885 unsigned BitWidth = getTypeSizeInBits(AR->getType());
1886
1887 // The no-signed-wrap case is handled before the uniquing lookup above.
1888
1889 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1890 // Note that this serves two purposes: It filters out loops that are
1891 // simply not analyzable, and it covers the case where this code is
1892 // being called from within backedge-taken count analysis, such that
1893 // attempting to ask for the backedge-taken count would likely result
1894 // in infinite recursion. In the later case, the analysis code will
1895 // cope with a conservative value, and it will take care to purge
1896 // that value once it has finished.
1897 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
1898 if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
1899 // Manually compute the final value for AR, checking for
1900 // overflow.
1901
1902 // Check whether the backedge-taken count can be losslessly casted to
1903 // the addrec's type. The count is always unsigned.
1904 const SCEV *CastedMaxBECount =
1905 getTruncateOrZeroExtend(MaxBECount, Start->getType(), Depth);
1906 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
1907 CastedMaxBECount, MaxBECount->getType(), Depth);
1908 if (MaxBECount == RecastedMaxBECount) {
1909 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
1910 // Check whether Start+Step*MaxBECount has no signed overflow.
1911 const SCEV *SMul =
1912 getMulExpr(CastedMaxBECount, Step, SCEV::FlagNone, Depth + 1);
1913 const SCEV *SAdd = getSignExtendExpr(
1914 getAddExpr(Start, SMul, SCEV::FlagNone, Depth + 1), WideTy,
1915 Depth + 1);
1916 const SCEV *WideStart = getSignExtendExpr(Start, WideTy, Depth + 1);
1917 const SCEV *WideMaxBECount =
1918 getZeroExtendExpr(CastedMaxBECount, WideTy, Depth + 1);
1919 const SCEV *OperandExtendedAdd =
1920 getAddExpr(WideStart,
1921 getMulExpr(WideMaxBECount,
1922 getSignExtendExpr(Step, WideTy, Depth + 1),
1923 SCEV::FlagNone, Depth + 1),
1924 SCEV::FlagNone, Depth + 1);
1925 if (SAdd == OperandExtendedAdd) {
1926 // Cache knowledge of AR NSW, which is propagated to this AddRec.
1927 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNSW);
1928 // Return the expression with the addrec on the outside.
1929 Start =
1931 Step = getSignExtendExpr(Step, Ty, Depth + 1);
1932 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1933 }
1934 // Similar to above, only this time treat the step value as unsigned.
1935 // This covers loops that count up with an unsigned step.
1936 OperandExtendedAdd =
1937 getAddExpr(WideStart,
1938 getMulExpr(WideMaxBECount,
1939 getZeroExtendExpr(Step, WideTy, Depth + 1),
1940 SCEV::FlagNone, Depth + 1),
1941 SCEV::FlagNone, Depth + 1);
1942 if (SAdd == OperandExtendedAdd) {
1943 // If AR wraps around then
1944 //
1945 // abs(Step) * MaxBECount > unsigned-max(AR->getType())
1946 // => SAdd != OperandExtendedAdd
1947 //
1948 // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=>
1949 // (SAdd == OperandExtendedAdd => AR is NW)
1950
1951 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW);
1952
1953 // Return the expression with the addrec on the outside.
1954 Start =
1956 Step = getZeroExtendExpr(Step, Ty, Depth + 1);
1957 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1958 }
1959 }
1960 }
1961
1962 auto NewFlags = proveNoSignedWrapViaInduction(AR);
1963 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), NewFlags);
1964 if (AR->hasNoSignedWrap()) {
1965 // Same as nsw case above - duplicated here to avoid a compile time
1966 // issue. It's not clear that the order of checks does matter, but
1967 // it's one of two issue possible causes for a change which was
1968 // reverted. Be conservative for the moment.
1969 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1);
1970 Step = getSignExtendExpr(Step, Ty, Depth + 1);
1971 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1972 }
1973
1974 // sext({C,+,Step}) --> (sext(D) + sext({C-D,+,Step}))<nuw><nsw>
1975 // if D + (C - D + Step * n) could be proven to not signed wrap
1976 // where D maximizes the number of trailing zeros of (C - D + Step * n)
1977 if (const auto *SC = dyn_cast<SCEVConstant>(Start)) {
1978 const APInt &C = SC->getAPInt();
1979 const APInt &D = extractConstantWithoutWrapping(*this, C, Step);
1980 if (D != 0) {
1981 const SCEV *SSExtD = getSignExtendExpr(getConstant(D), Ty, Depth);
1982 const SCEV *SResidual =
1983 getAddRecExpr(getConstant(C - D), Step, L, AR->getNoWrapFlags());
1984 const SCEV *SSExtR = getSignExtendExpr(SResidual, Ty, Depth + 1);
1985 return getAddExpr(SSExtD, SSExtR, (SCEV::FlagNSW | SCEV::FlagNUW),
1986 Depth + 1);
1987 }
1988 }
1989
1990 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
1991 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNSW);
1992 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1);
1993 Step = getSignExtendExpr(Step, Ty, Depth + 1);
1994 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1995 }
1996 }
1997
1998 // If the input value is provably positive and we could not simplify
1999 // away the sext build a zext instead.
2001 return getZeroExtendExpr(Op, Ty, Depth + 1);
2002
2003 // sext(smin(x, y)) -> smin(sext(x), sext(y))
2004 // sext(smax(x, y)) -> smax(sext(x), sext(y))
2008 for (SCEVUse Operand : MinMax->operands())
2009 Operands.push_back(getSignExtendExpr(Operand, Ty));
2011 return getSMinExpr(Operands);
2012 return getSMaxExpr(Operands);
2013 }
2014
2015 // The cast wasn't folded; create an explicit cast node.
2016 // Recompute the insert position, as it may have been invalidated.
2017 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
2018 return S;
2019 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
2020 Op, Ty);
2021 UniqueSCEVs.insert(S, Token);
2022 S->computeAndSetCanonical(*this);
2023 registerUser(S, Op);
2024 return S;
2025}
2026
2028 switch (Kind) {
2029 case scTruncate:
2030 return getTruncateExpr(Op, Ty);
2031 case scZeroExtend:
2032 return getZeroExtendExpr(Op, Ty);
2033 case scSignExtend:
2034 return getSignExtendExpr(Op, Ty);
2035 case scPtrToAddr: {
2036 const SCEV *Expr = getPtrToAddrExpr(Op);
2037 assert(Expr->getType() == Ty && "requested type must match");
2038 return Expr;
2039 }
2040 default:
2041 llvm_unreachable("Not a SCEV cast expression!");
2042 }
2043}
2044
2045/// getAnyExtendExpr - Return a SCEV for the given operand extended with
2046/// unspecified bits out to the given type.
2048 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
2049 "This is not an extending conversion!");
2050 assert(isSCEVable(Ty) &&
2051 "This is not a conversion to a SCEVable type!");
2052 Ty = getEffectiveSCEVType(Ty);
2053
2054 // Sign-extend negative constants.
2055 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
2056 if (SC->getAPInt().isNegative())
2057 return getSignExtendExpr(Op, Ty);
2058
2059 // Peel off a truncate cast.
2061 const SCEV *NewOp = T->getOperand();
2062 if (getTypeSizeInBits(NewOp->getType()) < getTypeSizeInBits(Ty))
2063 return getAnyExtendExpr(NewOp, Ty);
2064 return getTruncateOrNoop(NewOp, Ty);
2065 }
2066
2067 // Next try a zext cast. If the cast is folded, use it.
2068 const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
2069 if (!isa<SCEVZeroExtendExpr>(ZExt))
2070 return ZExt;
2071
2072 // Next try a sext cast. If the cast is folded, use it.
2073 const SCEV *SExt = getSignExtendExpr(Op, Ty);
2074 if (!isa<SCEVSignExtendExpr>(SExt))
2075 return SExt;
2076
2077 // Force the cast to be folded into the operands of an addrec.
2078 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) {
2080 for (const SCEV *Op : AR->operands())
2081 Ops.push_back(getAnyExtendExpr(Op, Ty));
2082 return getAddRecExpr(Ops, AR->getLoop(), SCEV::FlagNW);
2083 }
2084
2085 // If the expression is obviously signed, use the sext cast value.
2086 if (isa<SCEVSMaxExpr>(Op))
2087 return SExt;
2088
2089 // Absent any other information, use the zext cast value.
2090 return ZExt;
2091}
2092
2093/// Process the given Ops list, which is a list of operands to be added under
2094/// the given scale, update the given map. This is a helper function for
2095/// getAddRecExpr. As an example of what it does, given a sequence of operands
2096/// that would form an add expression like this:
2097///
2098/// m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
2099///
2100/// where A and B are constants, update the map with these values:
2101///
2102/// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
2103///
2104/// and add 13 + A*B*29 to AccumulatedConstant.
2105/// This will allow getAddRecExpr to produce this:
2106///
2107/// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
2108///
2109/// This form often exposes folding opportunities that are hidden in
2110/// the original operand list.
2111///
2112/// Return true iff it appears that any interesting folding opportunities
2113/// may be exposed. This helps getAddRecExpr short-circuit extra work in
2114/// the common case where no interesting opportunities are present, and
2115/// is also used as a check to avoid infinite recursion.
2116static bool
2119 APInt &AccumulatedConstant, ArrayRef<SCEVUse> Ops,
2120 const APInt &Scale, ScalarEvolution &SE) {
2121 bool Interesting = false;
2122
2123 // Iterate over the add operands. They are sorted, with constants first.
2124 unsigned i = 0;
2125 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) {
2126 ++i;
2127 // Pull a buried constant out to the outside.
2128 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
2129 Interesting = true;
2130 AccumulatedConstant += Scale * C->getAPInt();
2131 }
2132
2133 // Next comes everything else. We're especially interested in multiplies
2134 // here, but they're in the middle, so just visit the rest with one loop.
2135 for (; i != Ops.size(); ++i) {
2137 if (Mul && isa<SCEVConstant>(Mul->getOperand(0))) {
2138 APInt NewScale =
2139 Scale * cast<SCEVConstant>(Mul->getOperand(0))->getAPInt();
2140 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) {
2141 // A multiplication of a constant with another add; recurse.
2142 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1));
2143 Interesting |= CollectAddOperandsWithScales(
2144 M, NewOps, AccumulatedConstant, Add->operands(), NewScale, SE);
2145 } else {
2146 // A multiplication of a constant with some other value. Update
2147 // the map.
2148 SmallVector<SCEVUse, 4> MulOps(drop_begin(Mul->operands()));
2149 const SCEV *Key = SE.getMulExpr(MulOps)->getCanonical();
2150 auto Pair = M.insert({Key, NewScale});
2151 if (Pair.second) {
2152 NewOps.push_back(Pair.first->first);
2153 } else {
2154 Pair.first->second += NewScale;
2155 // The map already had an entry for this value, which may indicate
2156 // a folding opportunity.
2157 Interesting = true;
2158 }
2159 }
2160 } else {
2161 // An ordinary operand. Update the map.
2162 auto Pair = M.insert({Ops[i]->getCanonical(), Scale});
2163 if (Pair.second) {
2164 NewOps.push_back(Pair.first->first);
2165 } else {
2166 Pair.first->second += Scale;
2167 // The map already had an entry for this value, which may indicate
2168 // a folding opportunity.
2169 Interesting = true;
2170 }
2171 }
2172 }
2173
2174 return Interesting;
2175}
2176
2178 const SCEV *LHS, const SCEV *RHS,
2179 const Instruction *CtxI) {
2180 auto Operation = [this, BinOp](SCEVUse L, SCEVUse R) -> const SCEV * {
2181 switch (BinOp) {
2182 default:
2183 llvm_unreachable("Unsupported binary op");
2184 case Instruction::Add:
2185 return getAddExpr(L, R);
2186 case Instruction::Sub:
2187 return getMinusSCEV(L, R);
2188 case Instruction::Mul:
2189 return getMulExpr(L, R);
2190 }
2191 };
2192
2193 const SCEV *(ScalarEvolution::*Extension)(SCEVUse, Type *, unsigned) =
2196
2197 // Check ext(LHS op RHS) == ext(LHS) op ext(RHS)
2198 auto *NarrowTy = cast<IntegerType>(LHS->getType());
2199 auto *WideTy =
2200 IntegerType::get(NarrowTy->getContext(), NarrowTy->getBitWidth() * 2);
2201
2202 const SCEV *A = (this->*Extension)(Operation(LHS, RHS), WideTy, 0);
2203 const SCEV *LHSB = (this->*Extension)(LHS, WideTy, 0);
2204 const SCEV *RHSB = (this->*Extension)(RHS, WideTy, 0);
2205 const SCEV *B = Operation(LHSB, RHSB);
2206 if (A == B)
2207 return true;
2208 // Can we use context to prove the fact we need?
2209 if (!CtxI)
2210 return false;
2211 // TODO: Support mul.
2212 if (BinOp == Instruction::Mul)
2213 return false;
2214 auto *RHSC = dyn_cast<SCEVConstant>(RHS);
2215 // TODO: Lift this limitation.
2216 if (!RHSC)
2217 return false;
2218 APInt C = RHSC->getAPInt();
2219 unsigned NumBits = C.getBitWidth();
2220 bool IsSub = (BinOp == Instruction::Sub);
2221 bool IsNegativeConst = (Signed && C.isNegative());
2222 // Compute the direction and magnitude by which we need to check overflow.
2223 bool OverflowDown = IsSub ^ IsNegativeConst;
2224 APInt Magnitude = C;
2225 if (IsNegativeConst) {
2226 if (C == APInt::getSignedMinValue(NumBits))
2227 // TODO: SINT_MIN on inversion gives the same negative value, we don't
2228 // want to deal with that.
2229 return false;
2230 Magnitude = -C;
2231 }
2232
2234 if (OverflowDown) {
2235 // To avoid overflow down, we need to make sure that MIN + Magnitude <= LHS.
2236 APInt Min = Signed ? APInt::getSignedMinValue(NumBits)
2237 : APInt::getMinValue(NumBits);
2238 APInt Limit = Min + Magnitude;
2239 return isKnownPredicateAt(Pred, getConstant(Limit), LHS, CtxI);
2240 } else {
2241 // To avoid overflow up, we need to make sure that LHS <= MAX - Magnitude.
2242 APInt Max = Signed ? APInt::getSignedMaxValue(NumBits)
2243 : APInt::getMaxValue(NumBits);
2244 APInt Limit = Max - Magnitude;
2245 return isKnownPredicateAt(Pred, LHS, getConstant(Limit), CtxI);
2246 }
2247}
2248
2250 const OverflowingBinaryOperator *OBO) {
2251 // It cannot be done any better.
2252 if (OBO->hasNoUnsignedWrap() && OBO->hasNoSignedWrap())
2253 return std::nullopt;
2254
2256
2257 if (OBO->hasNoUnsignedWrap())
2259 if (OBO->hasNoSignedWrap())
2261
2262 bool Deduced = false;
2263
2265 const SCEV *LHS = getSCEV(OBO->getOperand(0));
2266 const SCEV *RHS = getSCEV(OBO->getOperand(1));
2267
2268 bool CanUseNSW = true;
2269 const APInt *ShiftAmt;
2270 // Treat `shl %a, C` as `mul %a, 1 << C`.
2271 if (match(OBO, m_Shl(m_Value(), m_APInt(ShiftAmt)))) {
2272 unsigned BitWidth = ShiftAmt->getBitWidth();
2273 if (ShiftAmt->uge(BitWidth))
2274 return std::nullopt;
2275 // NSW only transfers if the shift amount is < BitWidth - 1, as INT_MIN * -1
2276 // overflows.
2277 CanUseNSW = ShiftAmt->ult(BitWidth - 1);
2278 Opcode = Instruction::Mul;
2280 } else if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
2281 Opcode != Instruction::Mul) {
2282 return std::nullopt;
2283 }
2284
2285 const Instruction *CtxI =
2286 Opts.scalar_evolution_use_context_for_no_wrap_flag_strenghening
2288 : nullptr;
2289 if (!OBO->hasNoUnsignedWrap() &&
2290 willNotOverflow(Opcode, /* Signed */ false, LHS, RHS, CtxI)) {
2292 Deduced = true;
2293 }
2294
2295 if (CanUseNSW && !OBO->hasNoSignedWrap() &&
2296 willNotOverflow(Opcode, /* Signed */ true, LHS, RHS, CtxI)) {
2298 Deduced = true;
2299 }
2300
2301 if (Deduced)
2302 return Flags;
2303 return std::nullopt;
2304}
2305
2306// We're trying to construct a SCEV of type `Type' with `Ops' as operands and
2307// `OldFlags' as can't-wrap behavior. Infer a more aggressive set of
2308// can't-overflow flags for the operation if possible.
2311 using namespace std::placeholders;
2312
2313 using OBO = OverflowingBinaryOperator;
2314
2315 bool CanAnalyze =
2317 (void)CanAnalyze;
2318 assert(CanAnalyze && "don't call from other places!");
2319
2320 SCEVFlags SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
2321 SCEVFlags SignOrUnsignWrap =
2322 ScalarEvolution::maskFlags(Flags, SignOrUnsignMask);
2323
2324 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
2325 auto IsKnownNonNegative = [&](SCEVUse U) {
2326 return SE->isKnownNonNegative(U);
2327 };
2328
2329 if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Ops, IsKnownNonNegative))
2330 Flags = ScalarEvolution::setFlags(Flags, SignOrUnsignMask);
2331
2332 SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, SignOrUnsignMask);
2333
2334 if (SignOrUnsignWrap != SignOrUnsignMask &&
2335 (Type == scAddExpr || Type == scMulExpr) && Ops.size() == 2 &&
2336 isa<SCEVConstant>(Ops[0])) {
2337
2338 auto Opcode = [&] {
2339 switch (Type) {
2340 case scAddExpr:
2341 return Instruction::Add;
2342 case scMulExpr:
2343 return Instruction::Mul;
2344 default:
2345 llvm_unreachable("Unexpected SCEV op.");
2346 }
2347 }();
2348
2349 const APInt &C = cast<SCEVConstant>(Ops[0])->getAPInt();
2350
2351 // (A <opcode> C) --> (A <opcode> C)<nsw> if the op doesn't sign overflow.
2352 if (!(SignOrUnsignWrap & SCEV::FlagNSW)) {
2353 auto NSWRegion =
2354 ConstantRange::makeExactNoWrapRegion(Opcode, C, OBO::NoSignedWrap);
2355 if (NSWRegion.contains(SE->getSignedRange(Ops[1])))
2357 }
2358
2359 // (A <opcode> C) --> (A <opcode> C)<nuw> if the op doesn't unsign overflow.
2360 if (!(SignOrUnsignWrap & SCEV::FlagNUW)) {
2361 auto NUWRegion =
2362 ConstantRange::makeExactNoWrapRegion(Opcode, C, OBO::NoUnsignedWrap);
2363 if (NUWRegion.contains(SE->getUnsignedRange(Ops[1])))
2365 }
2366 }
2367
2368 // <0,+,nonnegative><nw> is also nuw
2369 // TODO: Add corresponding nsw case
2371 !ScalarEvolution::hasFlags(Flags, SCEV::FlagNUW) && Ops.size() == 2 &&
2372 Ops[0]->isZero() && IsKnownNonNegative(Ops[1]))
2374
2375 // both (udiv X, Y) * Y and Y * (udiv X, Y) are always NUW
2377 Ops.size() == 2) {
2378 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Ops[0]))
2379 if (UDiv->getOperand(1) == Ops[1])
2381 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Ops[1]))
2382 if (UDiv->getOperand(1) == Ops[0])
2384 }
2385
2386 return Flags;
2387}
2388
2390 return isLoopInvariant(S, L) && properlyDominates(S, L->getHeader());
2391}
2392
2393/// Get a canonical add expression, or something simpler if possible.
2395 SCEVFlagsPair Flags, unsigned Depth) {
2396 SCEVFlags ExprFlags = Flags.ExprFlags;
2397 SCEVFlags UseFlags = Flags.UseFlags;
2398 assert(!(ExprFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
2399 "only nuw or nsw allowed");
2400 assert(!(UseFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
2401 "only nuw or nsw allowed");
2402 assert(!Ops.empty() && "Cannot get empty add!");
2403 if (Ops.size() == 1) return Ops[0];
2404#ifndef NDEBUG
2405 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
2406 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
2407 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
2408 "SCEVAddExpr operand types don't match!");
2409 unsigned NumPtrs = count_if(
2410 Ops, [](const SCEV *Op) { return Op->getType()->isPointerTy(); });
2411 assert(NumPtrs <= 1 && "add has at most one pointer operand");
2412#endif
2413
2415 Opts, *this, LI, DT, Ops,
2416 [](const APInt &C1, const APInt &C2) { return C1 + C2; },
2417 [](const APInt &C) { return C.isZero(); }, // identity
2418 [](const APInt &C) { return false; }); // absorber
2419 if (Folded)
2420 return Folded;
2421
2422#ifndef NDEBUG
2423 // Keep track of operands after constant folding, for verification when adding
2424 // use-specific flags.
2425 const SmallVector<SCEVUse, 8> OrigOps(Ops.begin(), Ops.end());
2426#endif
2427
2428 unsigned Idx = isa<SCEVConstant>(Ops[0]) ? 1 : 0;
2429
2430 // Delay expensive flag strengthening until necessary.
2431 auto ComputeFlags = [this, ExprFlags](ArrayRef<SCEVUse> Ops) {
2432 return StrengthenNoWrapFlags(this, scAddExpr, Ops, ExprFlags);
2433 };
2434
2435 // Limit recursion calls depth.
2436 if (Depth > Opts.scalar_evolution_max_arith_depth ||
2437 hasHugeExpression(Opts, Ops))
2438 return {getOrCreateAddExpr(Ops, ComputeFlags(Ops)), UseFlags};
2439
2440 if (SCEV *S = findExistingSCEVInCache(scAddExpr, Ops)) {
2441 // Don't strengthen flags if we have no new information.
2442 SCEVAddExpr *Add = static_cast<SCEVAddExpr *>(S);
2443 if (Add->getNoWrapFlags(ExprFlags) != ExprFlags)
2444 Add->setNoWrapFlags(ComputeFlags(Ops));
2445 return {S, UseFlags};
2446 }
2447
2448 // Okay, check to see if the same value occurs in the operand list more than
2449 // once. If so, merge them together into an multiply expression. Since we
2450 // sorted the list, these values are required to be adjacent.
2451 Type *Ty = Ops[0]->getType();
2452 bool FoundMatch = false;
2453 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
2454 if (SCEVUse Op = SCEVUse::getCommon(Ops[i], Ops[i + 1])) {
2455 // X + Y + Y --> X + Y*2
2456 // Scan ahead to count how many equal operands there are.
2457 unsigned Count = 2;
2458 for (; i + Count != e; ++Count) {
2459 SCEVUse Common = SCEVUse::getCommon(Op, Ops[i + Count]);
2460 if (!Common)
2461 break;
2462 Op = Common;
2463 }
2464 // Merge the values into a multiply.
2465 SCEVUse Scale = getConstant(Ty, Count);
2466 SCEVUse Mul = getMulExpr(Scale, Op, SCEV::FlagNone, Depth + 1);
2467 if (Ops.size() == Count)
2468 return Mul;
2469 Ops[i] = Mul;
2470 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+Count);
2471 --i; e -= Count - 1;
2472 FoundMatch = true;
2473 }
2474 if (FoundMatch)
2475 return getAddExpr(Ops, ExprFlags, Depth + 1);
2476
2477 // Check for truncates. If all the operands are truncated from the same
2478 // type, see if factoring out the truncate would permit the result to be
2479 // folded. eg., n*trunc(x) + m*trunc(y) --> trunc(trunc(m)*x + trunc(n)*y)
2480 // if the contents of the resulting outer trunc fold to something simple.
2481 auto FindTruncSrcType = [&]() -> Type * {
2482 // We're ultimately looking to fold an addrec of truncs and muls of only
2483 // constants and truncs, so if we find any other types of SCEV
2484 // as operands of the addrec then we bail and return nullptr here.
2485 // Otherwise, we return the type of the operand of a trunc that we find.
2486 if (auto *T = dyn_cast<SCEVTruncateExpr>(Ops[Idx]))
2487 return T->getOperand()->getType();
2488 if (const auto *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
2489 SCEVUse LastOp = Mul->getOperand(Mul->getNumOperands() - 1);
2490 if (const auto *T = dyn_cast<SCEVTruncateExpr>(LastOp))
2491 return T->getOperand()->getType();
2492 }
2493 return nullptr;
2494 };
2495 if (auto *SrcType = FindTruncSrcType()) {
2496 SmallVector<SCEVUse, 8> LargeOps;
2497 bool Ok = true;
2498 // Check all the operands to see if they can be represented in the
2499 // source type of the truncate.
2500 for (const SCEV *Op : Ops) {
2502 if (T->getOperand()->getType() != SrcType) {
2503 Ok = false;
2504 break;
2505 }
2506 LargeOps.push_back(T->getOperand());
2507 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Op)) {
2508 LargeOps.push_back(getAnyExtendExpr(C, SrcType));
2509 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Op)) {
2510 SmallVector<SCEVUse, 8> LargeMulOps;
2511 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
2512 if (const SCEVTruncateExpr *T =
2513 dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) {
2514 if (T->getOperand()->getType() != SrcType) {
2515 Ok = false;
2516 break;
2517 }
2518 LargeMulOps.push_back(T->getOperand());
2519 } else if (const auto *C = dyn_cast<SCEVConstant>(M->getOperand(j))) {
2520 LargeMulOps.push_back(getAnyExtendExpr(C, SrcType));
2521 } else {
2522 Ok = false;
2523 break;
2524 }
2525 }
2526 if (Ok)
2527 LargeOps.push_back(
2528 getMulExpr(LargeMulOps, SCEV::FlagNone, Depth + 1));
2529 } else {
2530 Ok = false;
2531 break;
2532 }
2533 }
2534 if (Ok) {
2535 // Evaluate the expression in the larger type.
2536 const SCEV *Fold = getAddExpr(LargeOps, SCEV::FlagNone, Depth + 1);
2537 // If it folds to something simple, use it. Otherwise, don't.
2538 if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold))
2539 return getTruncateExpr(Fold, Ty);
2540 }
2541 }
2542
2543 if (Ops.size() == 2) {
2544 // Check if we have an expression of the form ((X + C1) - C2), where C1 and
2545 // C2 can be folded in a way that allows retaining wrapping flags of (X +
2546 // C1).
2547 const SCEV *A = Ops[0];
2548 const SCEV *B = Ops[1];
2549 auto *AddExpr = dyn_cast<SCEVAddExpr>(B);
2550 auto *C = dyn_cast<SCEVConstant>(A);
2551 if (AddExpr && C && isa<SCEVConstant>(AddExpr->getOperand(0))) {
2552 auto C1 = cast<SCEVConstant>(AddExpr->getOperand(0))->getAPInt();
2553 auto C2 = C->getAPInt();
2554 SCEVFlags PreservedFlags = SCEV::FlagNone;
2555
2556 APInt ConstAdd = C1 + C2;
2557 auto AddFlags = AddExpr->getNoWrapFlags();
2558 // Adding a smaller constant is NUW if the original AddExpr was NUW.
2560 ConstAdd.ule(C1)) {
2561 PreservedFlags =
2563 }
2564
2565 // Adding a constant with the same sign and small magnitude is NSW, if the
2566 // original AddExpr was NSW.
2568 C1.isSignBitSet() == ConstAdd.isSignBitSet() &&
2569 ConstAdd.abs().ule(C1.abs())) {
2570 PreservedFlags =
2572 }
2573
2574 if (PreservedFlags != SCEV::FlagNone) {
2575 SmallVector<SCEVUse, 4> NewOps(AddExpr->operands());
2576 NewOps[0] = getConstant(ConstAdd);
2577 return getAddExpr(NewOps, PreservedFlags);
2578 }
2579 }
2580
2581 // Try to push the constant operand into a ZExt: A + zext (-A + B) -> zext
2582 // (B), if trunc (A) + -A + B does not unsigned-wrap.
2583 const SCEVAddExpr *InnerAdd;
2584 if (match(B, m_scev_ZExt(m_scev_Add(InnerAdd)))) {
2585 const SCEV *NarrowA = getTruncateExpr(A, InnerAdd->getType());
2586 if (NarrowA == getNegativeSCEV(InnerAdd->getOperand(0)) &&
2587 getZeroExtendExpr(NarrowA, B->getType()) == A &&
2588 hasFlags(StrengthenNoWrapFlags(this, scAddExpr, {NarrowA, InnerAdd},
2590 SCEV::FlagNUW)) {
2591 return getZeroExtendExpr(getAddExpr(NarrowA, InnerAdd), B->getType());
2592 }
2593 }
2594 }
2595
2596 // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y)
2597 const SCEV *Y;
2598 if (Ops.size() == 2 &&
2599 match(Ops[0],
2601 m_scev_URem(m_scev_Specific(Ops[1]), m_SCEV(Y), *this))))
2602 return getMulExpr(Y, getUDivExpr(Ops[1], Y));
2603
2604 // Skip past any other cast SCEVs.
2605 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
2606 ++Idx;
2607
2608 // If there are add operands they would be next.
2609 if (Idx < Ops.size()) {
2610 bool DeletedAdd = false;
2611 // If the original flags and all inlined SCEVAddExprs are NUW, use the
2612 // common NUW flag for expression after inlining. Other flags cannot be
2613 // preserved, because they may depend on the original order of operations.
2614 SCEVFlags CommonFlags = maskFlags(ExprFlags, SCEV::FlagNUW);
2615 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) {
2616 if (Ops.size() > Opts.scev_addops_inline_threshold ||
2617 Add->getNumOperands() > Opts.scev_addops_inline_threshold)
2618 break;
2619 // If we have an add, expand the add operands onto the end of the operands
2620 // list.
2621 Ops.erase(Ops.begin()+Idx);
2622 append_range(Ops, Add->operands());
2623 DeletedAdd = true;
2624 CommonFlags = maskFlags(CommonFlags, Add->getNoWrapFlags());
2625 }
2626
2627 // If we deleted at least one add, we added operands to the end of the list,
2628 // and they are not necessarily sorted. Recurse to resort and resimplify
2629 // any operands we just acquired.
2630 if (DeletedAdd)
2631 return getAddExpr(Ops, CommonFlags, Depth + 1);
2632 }
2633
2634 // Skip over the add expression until we get to a multiply.
2635 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
2636 ++Idx;
2637
2638 // Check to see if there are any folding opportunities present with
2639 // operands multiplied by constant values.
2640 if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) {
2641 uint64_t BitWidth = getTypeSizeInBits(Ty);
2644 APInt AccumulatedConstant(BitWidth, 0);
2645 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
2646 Ops, APInt(BitWidth, 1), *this)) {
2647 struct APIntCompare {
2648 bool operator()(const APInt &LHS, const APInt &RHS) const {
2649 return LHS.ult(RHS);
2650 }
2651 };
2652
2653 // Some interesting folding opportunity is present, so its worthwhile to
2654 // re-generate the operands list. Group the operands by constant scale,
2655 // to avoid multiplying by the same constant scale multiple times.
2656 std::map<APInt, SmallVector<SCEVUse, 4>, APIntCompare> MulOpLists;
2657 for (SCEVUse NewOp : NewOps)
2658 MulOpLists[M.find(NewOp)->second].push_back(NewOp);
2659 // Re-generate the operands list.
2660 Ops.clear();
2661 if (AccumulatedConstant != 0)
2662 Ops.push_back(getConstant(AccumulatedConstant));
2663 for (auto &MulOp : MulOpLists) {
2664 if (MulOp.first == 1) {
2665 Ops.push_back(getAddExpr(MulOp.second, SCEV::FlagNone, Depth + 1));
2666 } else if (MulOp.first != 0) {
2667 Ops.push_back(
2668 getMulExpr(getConstant(MulOp.first),
2669 getAddExpr(MulOp.second, SCEV::FlagNone, Depth + 1),
2670 SCEV::FlagNone, Depth + 1));
2671 }
2672 }
2673 if (Ops.empty())
2674 return getZero(Ty);
2675 if (Ops.size() == 1)
2676 return Ops[0];
2677 return getAddExpr(Ops, SCEV::FlagNone, Depth + 1);
2678 }
2679 }
2680
2681 // Given a SCEVMulExpr and an operand index, return the product of all
2682 // operands except the one at OpIdx.
2683 auto StripFactor = [&](const SCEVMulExpr *M, unsigned OpIdx) -> SCEVUse {
2684 if (M->getNumOperands() == 2)
2685 return M->getOperand(OpIdx == 0);
2686 SmallVector<SCEVUse, 4> Remaining(M->operands().take_front(OpIdx));
2687 append_range(Remaining, M->operands().drop_front(OpIdx + 1));
2688 return getMulExpr(Remaining, SCEV::FlagNone, Depth + 1);
2689 };
2690
2691 // If we are adding something to a multiply expression, make sure the
2692 // something is not already an operand of the multiply. If so, merge it into
2693 // the multiply.
2694 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) {
2695 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]);
2696 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
2697 // Scan all terms to find every occurrence of common factor MulOpSCEV
2698 // and fold them in one shot:
2699 // A1*X + A2*X + ... + An*X --> X * (A1 + A2 + ... + An)
2700 SCEVUse MulOpSCEV = Mul->getOperand(MulOp);
2701 if (isa<SCEVConstant>(MulOpSCEV))
2702 continue;
2703
2704 // Cofactors: 1 for bare addends matching MulOpSCEV, or the
2705 // remaining product for multiply terms containing MulOpSCEV.
2706 SmallVector<SCEVUse, 4> Cofactors;
2707 SmallVector<unsigned, 4> DeadIndices;
2708 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp) {
2709 if (SCEVUse Common = SCEVUse::getCommon(MulOpSCEV, Ops[AddOp])) {
2710 // W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1))
2711 MulOpSCEV = Common;
2712 Cofactors.push_back(getOne(Ty));
2713 DeadIndices.push_back(AddOp);
2714 continue;
2715 }
2716
2717 if (AddOp <= Idx || !isa<SCEVMulExpr>(Ops[AddOp]))
2718 continue;
2719
2720 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[AddOp]);
2721 for (unsigned OMulOp = 0, OE = OtherMul->getNumOperands(); OMulOp != OE;
2722 ++OMulOp) {
2723 if (SCEVUse Common =
2724 SCEVUse::getCommon(MulOpSCEV, OtherMul->getOperand(OMulOp))) {
2725 // (A*B*C) + (A*D*E) --> A * (B*C + D*E)
2726 MulOpSCEV = Common;
2727 Cofactors.push_back(StripFactor(OtherMul, OMulOp));
2728 DeadIndices.push_back(AddOp);
2729 break;
2730 }
2731 }
2732 }
2733
2734 // Fold all collected cofactors with the anchor multiply's cofactor:
2735 // MulOpSCEV * (Cofactor_1 + ... + Cofactor_n + AnchorCofactor)
2736 if (!Cofactors.empty()) {
2737 Cofactors.push_back(StripFactor(Mul, MulOp));
2738
2739 SCEVUse InnerSum = getAddExpr(Cofactors, SCEV::FlagNone, Depth + 1);
2740 SCEVUse OuterMul =
2741 getMulExpr(MulOpSCEV, InnerSum, SCEV::FlagNone, Depth + 1);
2742
2743 // DeadIndices does not include Idx (the anchor), hence +1.
2744 if (Ops.size() == DeadIndices.size() + 1)
2745 return OuterMul;
2746
2747 // Erase Ops[Idx] first, then erase DeadIndices in reverse order.
2748 // The -1 adjustment accounts for the shift from removing Idx;
2749 // reverse order means each erasure only shifts later positions,
2750 // which have already been processed.
2751 Ops.erase(Ops.begin() + Idx);
2752 for (unsigned Dead : reverse(DeadIndices))
2753 Ops.erase(Ops.begin() + (Dead > Idx ? Dead - 1 : Dead));
2754
2755 Ops.push_back(OuterMul);
2756 return getAddExpr(Ops, SCEV::FlagNone, Depth + 1);
2757 }
2758 }
2759 }
2760
2761 // If there are any add recurrences in the operands list, see if any other
2762 // added values are loop invariant. If so, we can fold them into the
2763 // recurrence.
2764 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
2765 ++Idx;
2766
2767 // Scan over all recurrences, trying to fold loop invariants into them.
2768 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
2769 // Scan all of the other operands to this add and add them to the vector if
2770 // they are loop invariant w.r.t. the recurrence.
2772 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
2773 const Loop *AddRecLoop = AddRec->getLoop();
2774 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
2775 if (isAvailableAtLoopEntry(Ops[i], AddRecLoop)) {
2776 LIOps.push_back(Ops[i]);
2777 Ops.erase(Ops.begin()+i);
2778 --i; --e;
2779 }
2780
2781 // If we found some loop invariants, fold them into the recurrence.
2782 if (!LIOps.empty()) {
2783 // Compute nowrap flags for the addition of the loop-invariant ops and
2784 // the addrec. Temporarily push it as an operand for that purpose. These
2785 // flags are valid in the scope of the addrec only.
2786 LIOps.push_back(AddRec);
2787 SCEVFlags Flags = ComputeFlags(LIOps);
2788 LIOps.pop_back();
2789
2790 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step}
2791 LIOps.push_back(AddRec->getStart());
2792
2793 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
2794
2795 // It is not in general safe to propagate flags valid on an add within
2796 // the addrec scope to one outside it. We must prove that the inner
2797 // scope is guaranteed to execute if the outer one does to be able to
2798 // safely propagate. We know the program is undefined if poison is
2799 // produced on the inner scoped addrec. We also know that *for this use*
2800 // the outer scoped add can't overflow (because of the flags we just
2801 // computed for the inner scoped add) without the program being undefined.
2802 // Proving that entry to the outer scope neccesitates entry to the inner
2803 // scope, thus proves the program undefined if the flags would be violated
2804 // in the outer scope.
2805 SCEVFlags AddFlags = Flags;
2806 if (AddFlags != SCEV::FlagNone) {
2807 auto *DefI = getDefiningScopeBound(LIOps);
2808 auto *ReachI = &*AddRecLoop->getHeader()->begin();
2809 if (!isGuaranteedToTransferExecutionTo(DefI, ReachI))
2810 AddFlags = SCEV::FlagNone;
2811 }
2812 AddRecOps[0] = getAddExpr(LIOps, AddFlags, Depth + 1);
2813
2814 // Build the new addrec. Propagate the NUW and NSW flags if both the
2815 // outer add and the inner addrec are guaranteed to have no overflow.
2816 // Always propagate NW.
2817 Flags = AddRec->getNoWrapFlags(setFlags(Flags, SCEV::FlagNW));
2818 const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, Flags);
2819
2820 // If all of the other operands were loop invariant, we are done.
2821 if (Ops.size() == 1) return NewRec;
2822
2823 // Otherwise, add the folded AddRec by the non-invariant parts.
2824 for (unsigned i = 0;; ++i)
2825 if (Ops[i] == AddRec) {
2826 Ops[i] = NewRec;
2827 break;
2828 }
2829 return getAddExpr(Ops, SCEV::FlagNone, Depth + 1);
2830 }
2831
2832 // Okay, if there weren't any loop invariants to be folded, check to see if
2833 // there are multiple AddRec's with the same loop induction variable being
2834 // added together. If so, we can fold them.
2835 for (unsigned OtherIdx = Idx+1;
2836 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
2837 ++OtherIdx) {
2838 // We expect the AddRecExpr's to be sorted in reverse dominance order,
2839 // so that the 1st found AddRecExpr is dominated by all others.
2840 assert(DT.dominates(
2841 cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(),
2842 AddRec->getLoop()->getHeader()) &&
2843 "AddRecExprs are not sorted in reverse dominance order?");
2844 if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) {
2845 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L>
2846 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
2847 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
2848 ++OtherIdx) {
2849 const auto *OtherAddRec = cast<SCEVAddRecExpr>(Ops[OtherIdx]);
2850 if (OtherAddRec->getLoop() == AddRecLoop) {
2851 for (unsigned i = 0, e = OtherAddRec->getNumOperands();
2852 i != e; ++i) {
2853 if (i >= AddRecOps.size()) {
2854 append_range(AddRecOps, OtherAddRec->operands().drop_front(i));
2855 break;
2856 }
2857 AddRecOps[i] =
2858 getAddExpr(AddRecOps[i], OtherAddRec->getOperand(i),
2859 SCEV::FlagNone, Depth + 1);
2860 }
2861 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
2862 }
2863 }
2864 // Step size has changed, so we cannot guarantee no self-wraparound.
2865 Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagNone);
2866 return getAddExpr(Ops, SCEV::FlagNone, Depth + 1);
2867 }
2868 }
2869
2870 // Otherwise couldn't fold anything into this recurrence. Move onto the
2871 // next one.
2872 }
2873
2874 // Okay, it looks like we really DO need an add expr. Check to see if we
2875 // already have one, otherwise create a new one.
2876 assert((UseFlags == SCEV::FlagNone || equal(OrigOps, Ops)) &&
2877 "Tried to add SCEVUse flags after operands changed");
2878 return {getOrCreateAddExpr(Ops, ComputeFlags(Ops)), UseFlags};
2879}
2880
2881const SCEV *ScalarEvolution::getOrCreateAddExpr(ArrayRef<SCEVUse> Ops,
2882 SCEVFlags Flags) {
2885 for (SCEVUse Op : Ops)
2886 ID.AddPointer(Op.getOpaqueValue());
2888 SCEVAddExpr *S = static_cast<SCEVAddExpr *>(UniqueSCEVs.lookup(ID, Token));
2889 if (!S) {
2890 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
2892 S = new (SCEVAllocator)
2893 SCEVAddExpr(ID.Intern(SCEVAllocator), O, Ops.size());
2894 UniqueSCEVs.insert(S, Token);
2895 S->computeAndSetCanonical(*this);
2896 registerUser(S, Ops);
2897 }
2898 S->setNoWrapFlags(Flags);
2899 return S;
2900}
2901
2902const SCEV *ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<SCEVUse> Ops,
2903 const Loop *L,
2904 SCEVFlags Flags) {
2905 FoldingSetNodeID ID;
2906 ID.AddInteger(scAddRecExpr);
2907 for (SCEVUse Op : Ops)
2908 ID.AddPointer(Op.getOpaqueValue());
2909 ID.AddPointer(L);
2910 FoldingSetInsertToken Token;
2911 SCEVAddRecExpr *S =
2912 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.lookup(ID, Token));
2913 if (!S) {
2914 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
2916 S = new (SCEVAllocator)
2917 SCEVAddRecExpr(ID.Intern(SCEVAllocator), O, Ops.size(), L);
2918 UniqueSCEVs.insert(S, Token);
2919 S->computeAndSetCanonical(*this);
2920 LoopUsers[L].push_back(S);
2921 registerUser(S, Ops);
2922 }
2923 setNoWrapFlags(S, Flags);
2924 return S;
2925}
2926
2927const SCEV *ScalarEvolution::getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
2928 SCEVFlags Flags) {
2929 FoldingSetNodeID ID;
2930 ID.AddInteger(scMulExpr);
2931 for (SCEVUse Op : Ops)
2932 ID.AddPointer(Op.getOpaqueValue());
2933 FoldingSetInsertToken Token;
2934 SCEVMulExpr *S = static_cast<SCEVMulExpr *>(UniqueSCEVs.lookup(ID, Token));
2935 if (!S) {
2936 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
2938 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator),
2939 O, Ops.size());
2940 UniqueSCEVs.insert(S, Token);
2941 S->computeAndSetCanonical(*this);
2942 registerUser(S, Ops);
2943 }
2944 S->setNoWrapFlags(Flags);
2945 return S;
2946}
2947
2948const SCEV *ScalarEvolution::getOrCreateUDivExpr(SCEVUse LHS, SCEVUse RHS) {
2949 FoldingSetNodeID ID;
2950 ID.AddInteger(scUDivExpr);
2951 ID.AddPointer(LHS.getOpaqueValue());
2952 ID.AddPointer(RHS.getOpaqueValue());
2953 FoldingSetInsertToken Token;
2954 SCEV *S = UniqueSCEVs.lookup(ID, Token);
2955 if (!S) {
2956 S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator), LHS, RHS);
2957 UniqueSCEVs.insert(S, Token);
2958 S->computeAndSetCanonical(*this);
2959 registerUser(S, {LHS, RHS});
2960 }
2961 return S;
2962}
2963
2964static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
2965 uint64_t k = i*j;
2966 if (j > 1 && k / j != i) Overflow = true;
2967 return k;
2968}
2969
2970/// Compute the result of "n choose k", the binomial coefficient. If an
2971/// intermediate computation overflows, Overflow will be set and the return will
2972/// be garbage. Overflow is not cleared on absence of overflow.
2973static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
2974 // We use the multiplicative formula:
2975 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
2976 // At each iteration, we take the n-th term of the numeral and divide by the
2977 // (k-n)th term of the denominator. This division will always produce an
2978 // integral result, and helps reduce the chance of overflow in the
2979 // intermediate computations. However, we can still overflow even when the
2980 // final result would fit.
2981
2982 if (n == 0 || n == k) return 1;
2983 if (k > n) return 0;
2984
2985 if (k > n/2)
2986 k = n-k;
2987
2988 uint64_t r = 1;
2989 for (uint64_t i = 1; i <= k; ++i) {
2990 r = umul_ov(r, n-(i-1), Overflow);
2991 r /= i;
2992 }
2993 return r;
2994}
2995
2996/// Determine if any of the operands in this SCEV are a constant or if
2997/// any of the add or multiply expressions in this SCEV contain a constant.
2998static bool containsConstantInAddMulChain(const SCEV *StartExpr) {
2999 struct FindConstantInAddMulChain {
3000 bool FoundConstant = false;
3001
3002 bool follow(const SCEV *S) {
3003 FoundConstant |= isa<SCEVConstant>(S);
3004 return isa<SCEVAddExpr>(S) || isa<SCEVMulExpr>(S);
3005 }
3006
3007 bool isDone() const {
3008 return FoundConstant;
3009 }
3010 };
3011
3012 FindConstantInAddMulChain F;
3014 ST.visitAll(StartExpr);
3015 return F.FoundConstant;
3016}
3017
3018/// Get a canonical multiply expression, or something simpler if possible.
3020 SCEVFlagsPair Flags, unsigned Depth) {
3021 SCEVFlags ExprFlags = Flags.ExprFlags;
3022 SCEVFlags UseFlags = Flags.UseFlags;
3023 assert(ExprFlags == maskFlags(ExprFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
3024 "only nuw or nsw allowed");
3025 assert(UseFlags == maskFlags(UseFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
3026 "only nuw or nsw allowed");
3027 assert(!Ops.empty() && "Cannot get empty mul!");
3028 if (Ops.size() == 1) return Ops[0];
3029#ifndef NDEBUG
3030 Type *ETy = Ops[0]->getType();
3031 assert(!ETy->isPointerTy());
3032 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
3033 assert(Ops[i]->getType() == ETy &&
3034 "SCEVMulExpr operand types don't match!");
3035#endif
3036
3038 Opts, *this, LI, DT, Ops,
3039 [](const APInt &C1, const APInt &C2) { return C1 * C2; },
3040 [](const APInt &C) { return C.isOne(); }, // identity
3041 [](const APInt &C) { return C.isZero(); }); // absorber
3042 if (Folded)
3043 return Folded;
3044
3045#ifndef NDEBUG
3046 // Keep track of operands after constant folding, for verification when adding
3047 // use-specific flags.
3048 const SmallVector<SCEVUse, 8> OrigOps(Ops.begin(), Ops.end());
3049#endif
3050
3051 // Delay expensive flag strengthening until necessary.
3052 auto ComputeFlags = [this, ExprFlags](const ArrayRef<SCEVUse> Ops) {
3053 return StrengthenNoWrapFlags(this, scMulExpr, Ops, ExprFlags);
3054 };
3055
3056 // Limit recursion calls depth.
3057 if (Depth > Opts.scalar_evolution_max_arith_depth ||
3058 hasHugeExpression(Opts, Ops))
3059 return {getOrCreateMulExpr(Ops, ComputeFlags(Ops)), UseFlags};
3060
3061 if (SCEV *S = findExistingSCEVInCache(scMulExpr, Ops)) {
3062 // Don't strengthen flags if we have no new information.
3063 SCEVMulExpr *Mul = static_cast<SCEVMulExpr *>(S);
3064 if (Mul->getNoWrapFlags(ExprFlags) != ExprFlags)
3065 Mul->setNoWrapFlags(ComputeFlags(Ops));
3066 return {S, UseFlags};
3067 }
3068
3069 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
3070 if (Ops.size() == 2) {
3071 // C1*(C2+V) -> C1*C2 + C1*V
3072 // If any of Add's ops are Adds or Muls with a constant, apply this
3073 // transformation as well.
3074 //
3075 // TODO: There are some cases where this transformation is not
3076 // profitable; for example, Add = (C0 + X) * Y + Z. Maybe the scope of
3077 // this transformation should be narrowed down.
3078 const SCEV *Op0, *Op1;
3079 if (match(Ops[1], m_scev_Add(m_SCEV(Op0), m_SCEV(Op1))) &&
3081 const SCEV *LHS = getMulExpr(LHSC, Op0, SCEV::FlagNone, Depth + 1);
3082 const SCEV *RHS = getMulExpr(LHSC, Op1, SCEV::FlagNone, Depth + 1);
3083 return getAddExpr(LHS, RHS, SCEV::FlagNone, Depth + 1);
3084 }
3085
3086 if (Ops[0]->isAllOnesValue()) {
3087 // If we have a mul by -1 of an add, try distributing the -1 among the
3088 // add operands.
3089 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) {
3091 bool AnyFolded = false;
3092 for (const SCEV *AddOp : Add->operands()) {
3093 const SCEV *Mul =
3094 getMulExpr(Ops[0], SCEVUse(AddOp), SCEV::FlagNone, Depth + 1);
3095 if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true;
3096 NewOps.push_back(Mul);
3097 }
3098 if (AnyFolded)
3099 return getAddExpr(NewOps, SCEV::FlagNone, Depth + 1);
3100 } else if (const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Ops[1])) {
3101 // Negation preserves a recurrence's no self-wrap property.
3103 for (const SCEV *AddRecOp : AddRec->operands())
3104 Operands.push_back(getMulExpr(Ops[0], SCEVUse(AddRecOp),
3105 SCEV::FlagNone, Depth + 1));
3106 // Let M be the minimum representable signed value. AddRec with nsw
3107 // multiplied by -1 can have signed overflow if and only if it takes a
3108 // value of M: M * (-1) would stay M and (M + 1) * (-1) would be the
3109 // maximum signed value. In all other cases signed overflow is
3110 // impossible.
3111 auto FlagsMask = SCEV::FlagNW;
3112 if (AddRec->hasNoSignedWrap()) {
3113 auto MinInt =
3114 APInt::getSignedMinValue(getTypeSizeInBits(AddRec->getType()));
3115 if (getSignedRangeMin(AddRec) != MinInt)
3117 }
3118 return getAddRecExpr(Operands, AddRec->getLoop(),
3119 AddRec->getNoWrapFlags(FlagsMask));
3120 }
3121 }
3122
3123 // Try to push the constant operand into a ZExt: C * zext (A + B) ->
3124 // zext (C*A + C*B) if trunc (C) * (A + B) does not unsigned-wrap.
3125 const SCEVAddExpr *InnerAdd;
3126 if (match(Ops[1], m_scev_ZExt(m_scev_Add(InnerAdd)))) {
3127 const SCEV *NarrowC = getTruncateExpr(LHSC, InnerAdd->getType());
3128 if (isa<SCEVConstant>(InnerAdd->getOperand(0)) &&
3129 getZeroExtendExpr(NarrowC, Ops[1]->getType()) == LHSC &&
3130 hasFlags(StrengthenNoWrapFlags(this, scMulExpr, {NarrowC, InnerAdd},
3132 SCEV::FlagNUW)) {
3133 const SCEV *Res =
3134 getMulExpr(NarrowC, InnerAdd, SCEV::FlagNUW, Depth + 1);
3135 return getZeroExtendExpr(Res, Ops[1]->getType(), Depth + 1);
3136 };
3137 }
3138
3139 // Try to fold (C1 * D /u C2) -> C1/C2 * D, if C1 and C2 are powers-of-2,
3140 // D is a multiple of C2, and C1 is a multiple of C2. If C2 is a multiple
3141 // of C1, fold to (D /u (C2 /u C1)).
3142 const SCEV *D;
3143 APInt C1V = LHSC->getAPInt();
3144 // (C1 * D /u C2) == -1 * -C1 * D /u C2 when C1 != INT_MIN. Don't treat -1
3145 // as -1 * 1, as it won't enable additional folds.
3146 if (C1V.isNegative() && !C1V.isMinSignedValue() && !C1V.isAllOnes())
3147 C1V = C1V.abs();
3148 const SCEVConstant *C2;
3149 if (C1V.isPowerOf2() &&
3151 C2->getAPInt().isPowerOf2() &&
3152 C1V.logBase2() <= getMinTrailingZeros(D)) {
3153 const SCEV *NewMul = nullptr;
3154 if (C1V.uge(C2->getAPInt())) {
3155 NewMul = getMulExpr(getUDivExpr(getConstant(C1V), C2), D);
3156 } else if (C2->getAPInt().logBase2() <= getMinTrailingZeros(D)) {
3157 assert(C1V.ugt(1) && "C1 <= 1 should have been folded earlier");
3158 NewMul = getUDivExpr(D, getUDivExpr(C2, getConstant(C1V)));
3159 }
3160 if (NewMul)
3161 return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(NewMul);
3162 }
3163 }
3164 }
3165
3166 // Skip over the add expression until we get to a multiply.
3167 unsigned Idx = 0;
3168 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
3169 ++Idx;
3170
3171 // If there are mul operands inline them all into this expression.
3172 if (Idx < Ops.size()) {
3173 bool DeletedMul = false;
3174 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
3175 if (Ops.size() > Opts.scev_mulops_inline_threshold)
3176 break;
3177 // If we have an mul, expand the mul operands onto the end of the
3178 // operands list.
3179 Ops.erase(Ops.begin()+Idx);
3180 append_range(Ops, Mul->operands());
3181 DeletedMul = true;
3182 }
3183
3184 // If we deleted at least one mul, we added operands to the end of the
3185 // list, and they are not necessarily sorted. Recurse to resort and
3186 // resimplify any operands we just acquired.
3187 if (DeletedMul)
3188 return getMulExpr(Ops, SCEV::FlagNone, Depth + 1);
3189 }
3190
3191 // If there are any add recurrences in the operands list, see if any other
3192 // added values are loop invariant. If so, we can fold them into the
3193 // recurrence.
3194 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
3195 ++Idx;
3196
3197 // Scan over all recurrences, trying to fold loop invariants into them.
3198 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
3199 // Scan all of the other operands to this mul and add them to the vector
3200 // if they are loop invariant w.r.t. the recurrence.
3202 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
3203 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
3204 if (isAvailableAtLoopEntry(Ops[i], AddRec->getLoop())) {
3205 LIOps.push_back(Ops[i]);
3206 Ops.erase(Ops.begin()+i);
3207 --i; --e;
3208 }
3209
3210 // If we found some loop invariants, fold them into the recurrence.
3211 if (!LIOps.empty()) {
3212 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step}
3214 NewOps.reserve(AddRec->getNumOperands());
3215 const SCEV *Scale = getMulExpr(LIOps, SCEV::FlagNone, Depth + 1);
3216
3217 // If both the mul and addrec are nuw, we can preserve nuw.
3218 // If both the mul and addrec are nsw, we can only preserve nsw if either
3219 // a) they are also nuw, or
3220 // b) all multiplications of addrec operands with scale are nsw.
3221 SCEVFlags Flags = AddRec->getNoWrapFlags(ComputeFlags({Scale, AddRec}));
3222
3223 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
3224 NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i),
3225 SCEV::FlagNone, Depth + 1));
3226
3227 if (hasFlags(Flags, SCEV::FlagNSW) && !hasFlags(Flags, SCEV::FlagNUW)) {
3229 Instruction::Mul, getSignedRange(Scale),
3231 if (!NSWRegion.contains(getSignedRange(AddRec->getOperand(i))))
3232 Flags = clearFlags(Flags, SCEV::FlagNSW);
3233 }
3234 }
3235
3236 const SCEV *NewRec = getAddRecExpr(NewOps, AddRec->getLoop(), Flags);
3237
3238 // If all of the other operands were loop invariant, we are done.
3239 if (Ops.size() == 1) return NewRec;
3240
3241 // Otherwise, multiply the folded AddRec by the non-invariant parts.
3242 for (unsigned i = 0;; ++i)
3243 if (Ops[i] == AddRec) {
3244 Ops[i] = NewRec;
3245 break;
3246 }
3247 return getMulExpr(Ops, SCEV::FlagNone, Depth + 1);
3248 }
3249
3250 // Okay, if there weren't any loop invariants to be folded, check to see
3251 // if there are multiple AddRec's with the same loop induction variable
3252 // being multiplied together. If so, we can fold them.
3253
3254 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
3255 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
3256 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
3257 // ]]],+,...up to x=2n}.
3258 // Note that the arguments to choose() are always integers with values
3259 // known at compile time, never SCEV objects.
3260 //
3261 // The implementation avoids pointless extra computations when the two
3262 // addrec's are of different length (mathematically, it's equivalent to
3263 // an infinite stream of zeros on the right).
3264 bool OpsModified = false;
3265 for (unsigned OtherIdx = Idx+1;
3266 OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
3267 ++OtherIdx) {
3268 const SCEVAddRecExpr *OtherAddRec =
3269 dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]);
3270 if (!OtherAddRec || OtherAddRec->getLoop() != AddRec->getLoop())
3271 continue;
3272
3273 // Limit max number of arguments to avoid creation of unreasonably big
3274 // SCEVAddRecs with very complex operands.
3275 if (AddRec->getNumOperands() + OtherAddRec->getNumOperands() - 1 >
3276 Opts.scalar_evolution_max_add_rec_size ||
3277 hasHugeExpression(Opts, {AddRec, OtherAddRec}))
3278 continue;
3279
3280 bool Overflow = false;
3281 Type *Ty = AddRec->getType();
3282 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
3283 SmallVector<SCEVUse, 7> AddRecOps;
3284 for (int x = 0, xe = AddRec->getNumOperands() +
3285 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
3287 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
3288 uint64_t Coeff1 = Choose(x, 2*x - y, Overflow);
3289 for (int z = std::max(y-x, y-(int)AddRec->getNumOperands()+1),
3290 ze = std::min(x+1, (int)OtherAddRec->getNumOperands());
3291 z < ze && !Overflow; ++z) {
3292 uint64_t Coeff2 = Choose(2*x - y, x-z, Overflow);
3293 uint64_t Coeff;
3294 if (LargerThan64Bits)
3295 Coeff = umul_ov(Coeff1, Coeff2, Overflow);
3296 else
3297 Coeff = Coeff1*Coeff2;
3298 const SCEV *CoeffTerm = getConstant(Ty, Coeff);
3299 const SCEV *Term1 = AddRec->getOperand(y-z);
3300 const SCEV *Term2 = OtherAddRec->getOperand(z);
3301 SumOps.push_back(
3302 getMulExpr(CoeffTerm, Term1, Term2, SCEV::FlagNone, Depth + 1));
3303 }
3304 }
3305 if (SumOps.empty())
3306 SumOps.push_back(getZero(Ty));
3307 AddRecOps.push_back(getAddExpr(SumOps, SCEV::FlagNone, Depth + 1));
3308 }
3309 if (!Overflow) {
3310 const SCEV *NewAddRec =
3311 getAddRecExpr(AddRecOps, AddRec->getLoop(), SCEV::FlagNone);
3312 if (Ops.size() == 2) return NewAddRec;
3313 Ops[Idx] = NewAddRec;
3314 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
3315 OpsModified = true;
3316 AddRec = dyn_cast<SCEVAddRecExpr>(NewAddRec);
3317 if (!AddRec)
3318 break;
3319 }
3320 }
3321 if (OpsModified)
3322 return getMulExpr(Ops, SCEV::FlagNone, Depth + 1);
3323
3324 // Otherwise couldn't fold anything into this recurrence. Move onto the
3325 // next one.
3326 }
3327
3328 // Okay, it looks like we really DO need an mul expr. Check to see if we
3329 // already have one, otherwise create a new one.
3330 assert((UseFlags == SCEV::FlagNone || equal(OrigOps, Ops)) &&
3331 "Tried to add SCEVUse flags after operands changed");
3332 return {getOrCreateMulExpr(Ops, ComputeFlags(Ops)), UseFlags};
3333}
3334
3335/// Represents an unsigned remainder expression based on unsigned division.
3337 assert(getEffectiveSCEVType(LHS->getType()) ==
3338 getEffectiveSCEVType(RHS->getType()) &&
3339 "SCEVURemExpr operand types don't match!");
3340
3341 // Short-circuit easy cases
3342 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
3343 // If constant is one, the result is trivial
3344 if (RHSC->getValue()->isOne())
3345 return getZero(LHS->getType()); // X urem 1 --> 0
3346
3347 // If constant is a power of two, fold into a zext(trunc(LHS)).
3348 if (RHSC->getAPInt().isPowerOf2()) {
3349 Type *FullTy = LHS->getType();
3350 Type *TruncTy =
3351 IntegerType::get(getContext(), RHSC->getAPInt().logBase2());
3352 return getZeroExtendExpr(getTruncateExpr(LHS, TruncTy), FullTy);
3353 }
3354 }
3355
3356 // Fallback to %a == %x urem %y == %x -<nuw> ((%x udiv %y) *<nuw> %y)
3357 const SCEV *UDiv = getUDivExpr(LHS, RHS);
3358 const SCEV *Mult = getMulExpr(UDiv, RHS, SCEV::FlagNUW);
3359 return getMinusSCEV(LHS, Mult, SCEV::FlagNUW);
3360}
3361
3362/// Get a canonical unsigned division expression, or something simpler if
3363/// possible.
3365 assert(!LHS->getType()->isPointerTy() &&
3366 "SCEVUDivExpr operand can't be pointer!");
3367 assert(LHS->getType() == RHS->getType() &&
3368 "SCEVUDivExpr operand types don't match!");
3369
3370 if (SCEV *S = findExistingSCEVInCache(scUDivExpr, {LHS, RHS}))
3371 return S;
3372
3373 // 0 udiv Y == 0
3374 if (match(LHS, m_scev_Zero()))
3375 return LHS;
3376
3377 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
3378 if (RHSC->getValue()->isOne())
3379 return LHS; // X udiv 1 --> x
3380 // If the denominator is zero, the result of the udiv is undefined. Don't
3381 // try to analyze it, because the resolution chosen here may differ from
3382 // the resolution chosen in other parts of the compiler.
3383 if (!RHSC->getValue()->isZero()) {
3384 // Determine if the division can be folded into the operands of
3385 // its operands.
3386 // TODO: Generalize this to non-constants by using known-bits information.
3387 Type *Ty = LHS->getType();
3388 unsigned LZ = RHSC->getAPInt().countl_zero();
3389 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
3390 // For non-power-of-two values, effectively round the value up to the
3391 // nearest power of two.
3392 if (!RHSC->getAPInt().isPowerOf2())
3393 ++MaxShiftAmt;
3394 IntegerType *ExtTy =
3395 IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt);
3396 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS))
3397 if (const SCEVConstant *Step =
3398 dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) {
3399 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
3400 const APInt &StepInt = Step->getAPInt();
3401 const APInt &DivInt = RHSC->getAPInt();
3402 if (!StepInt.urem(DivInt) &&
3403 getZeroExtendExpr(AR, ExtTy) ==
3404 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
3405 getZeroExtendExpr(Step, ExtTy), AR->getLoop(),
3406 SCEV::FlagNone)) {
3408 for (const SCEV *Op : AR->operands())
3409 Operands.push_back(getUDivExpr(Op, RHS));
3410 return getAddRecExpr(Operands, AR->getLoop(), SCEV::FlagNW);
3411 }
3412 /// Get a canonical UDivExpr for a recurrence.
3413 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
3414 const APInt *StartRem;
3415 if (!DivInt.urem(StepInt) && match(getURemExpr(AR->getStart(), Step),
3416 m_scev_APInt(StartRem))) {
3417 bool NoWrap =
3418 getZeroExtendExpr(AR, ExtTy) ==
3419 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
3420 getZeroExtendExpr(Step, ExtTy), AR->getLoop(),
3422
3423 // With N <= C and both N, C as powers-of-2, the transformation
3424 // {X,+,N}/C => {(X - X%N),+,N}/C preserves division results even
3425 // if wrapping occurs, as the division results remain equivalent for
3426 // all offsets in [[(X - X%N), X).
3427 bool CanFoldWithWrap = StepInt.ule(DivInt) && // N <= C
3428 StepInt.isPowerOf2() && DivInt.isPowerOf2();
3429 // Only fold if the subtraction can be folded in the start
3430 // expression.
3431 const SCEV *NewStart =
3432 getMinusSCEV(AR->getStart(), getConstant(*StartRem));
3433 if (*StartRem != 0 && (NoWrap || CanFoldWithWrap) &&
3434 !isa<SCEVAddExpr>(NewStart)) {
3435 const SCEV *NewLHS =
3436 getAddRecExpr(NewStart, Step, AR->getLoop(),
3437 NoWrap ? SCEV::FlagNW : SCEV::FlagNone);
3438 if (LHS != NewLHS)
3439 return getUDivExpr(NewLHS, RHS);
3440 }
3441 }
3442 }
3443 // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
3444 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) {
3445 if (M->hasNoUnsignedWrap()) {
3446 // Find an operand that's safely divisible.
3447 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
3448 const SCEV *Op = M->getOperand(i);
3449 const SCEV *Div = getUDivExpr(Op, RHSC);
3450 if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) {
3451 SmallVector<SCEVUse, 4> Operands(M->operands());
3452 Operands[i] = Div;
3453 return getMulExpr(Operands);
3454 }
3455 }
3456
3457 // Even if it's not divisible, try to remove a common factor.
3458 if (const auto *LHSC = dyn_cast<SCEVConstant>(M->getOperand(0))) {
3459 APInt Factor = APIntOps::GreatestCommonDivisor(LHSC->getAPInt(),
3460 RHSC->getAPInt());
3461 if (!Factor.isIntN(1)) {
3462 SmallVector<SCEVUse, 2> NewOperands;
3463 NewOperands.push_back(getConstant(LHSC->getAPInt().udiv(Factor)));
3464 append_range(NewOperands, M->operands().drop_front());
3465 const SCEV *NewMul = getMulExpr(NewOperands);
3466 return getUDivExpr(NewMul,
3467 getConstant(RHSC->getAPInt().udiv(Factor)));
3468 }
3469 }
3470 }
3471 }
3472
3473 // (A/B)/C --> A/(B*C) if safe and B*C can be folded.
3474 if (const SCEVUDivExpr *OtherDiv = dyn_cast<SCEVUDivExpr>(LHS)) {
3475 if (auto *DivisorConstant =
3476 dyn_cast<SCEVConstant>(OtherDiv->getRHS())) {
3477 bool Overflow = false;
3478 APInt NewRHS =
3479 DivisorConstant->getAPInt().umul_ov(RHSC->getAPInt(), Overflow);
3480 if (Overflow) {
3481 return getConstant(RHSC->getType(), 0, false);
3482 }
3483 return getUDivExpr(OtherDiv->getLHS(), getConstant(NewRHS));
3484 }
3485 }
3486
3487 // (A+B)/C --> (A/C + B/C) if the add does not unsigned wrap and A/C and
3488 // B/C can be folded.
3489 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) {
3490 if (A->hasNoUnsignedWrap()) {
3492 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
3493 const SCEV *Op = getUDivExpr(A->getOperand(i), RHS);
3494 if (isa<SCEVUDivExpr>(Op) ||
3495 getMulExpr(Op, RHS) != A->getOperand(i))
3496 break;
3497 Operands.push_back(Op);
3498 }
3499 if (Operands.size() == A->getNumOperands())
3500 return getAddExpr(Operands);
3501 }
3502 }
3503
3504 // ((N - M) + (M * A)) / N --> ((N - 1) + (M * A)) / N
3505 // This is an idiom for rounding A up to the next multiple of N, where A
3506 // is aready known to be a multiple of M. In this case, instcombine can
3507 // see that some low bits of the added constant are unused, so can clear
3508 // them, but we want to canonicalise to set the low bits. This makes the
3509 // pattern easier to match, without needing to check for known bits in
3510 // A*M.
3511 const APInt &N = RHSC->getAPInt();
3512 const APInt *NMinusM, *M;
3513 const SCEV *A;
3514 if (match(LHS, m_scev_Add(m_scev_APInt(NMinusM),
3515 m_scev_Mul(m_scev_APInt(M), m_SCEV(A))))) {
3516 if (N.isPowerOf2() && M->isPowerOf2() && M->ult(N) &&
3517 *NMinusM == N - *M) {
3518 return getUDivExpr(
3520 RHS);
3521 }
3522 }
3523
3524 // Fold if both operands are constant.
3525 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS))
3526 return getConstant(LHSC->getAPInt().udiv(RHSC->getAPInt()));
3527 }
3528 }
3529
3530 // ((-C + (C smax %x)) /u %x) evaluates to zero, for any positive constant C.
3531 const APInt *NegC, *C;
3532 if (match(LHS,
3535 NegC->isNegative() && !NegC->isMinSignedValue() && *C == -*NegC)
3536 return getZero(LHS->getType());
3537
3538 // (%a * %b)<nuw> / %b -> %a
3539 const auto *Mul = dyn_cast<SCEVMulExpr>(LHS);
3540 if (Mul && Mul->hasNoUnsignedWrap()) {
3541 for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
3542 if (Mul->getOperand(i) == RHS) {
3544 append_range(Operands, Mul->operands().take_front(i));
3545 append_range(Operands, Mul->operands().drop_front(i + 1));
3546 return getMulExpr(Operands);
3547 }
3548 }
3549 }
3550
3551 // TODO: Generalize to handle any common factors.
3552 // udiv (mul nuw a, vscale), (mul nuw b, vscale) --> udiv a, b
3553 const SCEV *NewLHS, *NewRHS;
3554 if (match(LHS, m_scev_c_NUWMul(m_SCEV(NewLHS), m_SCEVVScale())) &&
3555 match(RHS, m_scev_c_NUWMul(m_SCEV(NewRHS), m_SCEVVScale())))
3556 return getUDivExpr(NewLHS, NewRHS);
3557
3558 return getOrCreateUDivExpr(LHS, RHS);
3559}
3560
3561/// Get a canonical unsigned division expression, or something simpler if
3562/// possible. There is no representation for an exact udiv in SCEV IR, but we
3563/// can attempt to optimize it prior to construction.
3565 // Currently there is no exact specific logic.
3566
3567 return getUDivExpr(LHS, RHS);
3568}
3569
3570/// Get an add recurrence expression for the specified loop. Simplify the
3571/// expression as much as possible.
3573 const Loop *L, SCEVFlagsPair Flags) {
3575 Operands.push_back(Start);
3576 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step))
3577 if (StepChrec->getLoop() == L) {
3578 append_range(Operands, StepChrec->operands());
3579 // The use flags describe the two-operand recurrence, not the flattened
3580 // one built here, so drop them just like the expression's NUW/NSW.
3581 return getAddRecExpr(Operands, L,
3582 maskFlags(Flags.ExprFlags, SCEV::FlagNW));
3583 }
3584
3585 Operands.push_back(Step);
3586 return getAddRecExpr(Operands, L, Flags);
3587}
3588
3589/// Get an add recurrence expression for the specified loop. Simplify the
3590/// expression as much as possible.
3592 const Loop *L, SCEVFlagsPair NWFlags) {
3593 SCEVFlags ExprFlags = NWFlags.ExprFlags;
3594 SCEVFlags UseFlags = NWFlags.UseFlags;
3595 assert(!(UseFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
3596 "only nuw or nsw allowed");
3597 if (Operands.size() == 1) return Operands[0];
3598#ifndef NDEBUG
3600 for (const SCEV *Op : llvm::drop_begin(Operands)) {
3601 assert(getEffectiveSCEVType(Op->getType()) == ETy &&
3602 "SCEVAddRecExpr operand types don't match!");
3603 assert(!Op->getType()->isPointerTy() && "Step must be integer");
3604 }
3605 for (const SCEV *Op : Operands)
3607 "SCEVAddRecExpr operand is not available at loop entry!");
3608#endif
3609
3610 if (Operands.back()->isZero()) {
3611 Operands.pop_back();
3612 return getAddRecExpr(Operands, L, SCEV::FlagNone); // {X,+,0} --> X
3613 }
3614
3615 // It's tempting to want to call getConstantMaxBackedgeTakenCount count here and
3616 // use that information to infer NUW and NSW flags. However, computing a
3617 // BE count requires calling getAddRecExpr, so we may not yet have a
3618 // meaningful BE count at this point (and if we don't, we'd be stuck
3619 // with a SCEVCouldNotCompute as the cached BE count).
3620
3621 ExprFlags = StrengthenNoWrapFlags(this, scAddRecExpr, Operands, ExprFlags);
3622
3623 return {getOrCreateAddRecExpr(Operands, L, ExprFlags), UseFlags};
3624}
3625
3627 ArrayRef<SCEVUse> IndexExprs) {
3628 const SCEV *BaseExpr = getSCEV(GEP->getPointerOperand());
3629 // getSCEV(Base)->getType() has the same address space as Base->getType()
3630 // because SCEV::getType() preserves the address space.
3631 GEPNoWrapFlags BaseNW = GEP->getNoWrapFlags();
3632 GEPNoWrapFlags OffsetNW = GEP->getNoWrapFlags();
3633 if (BaseNW != GEPNoWrapFlags::none()) {
3634 // We'd like to propagate flags from the IR to the corresponding SCEV nodes,
3635 // but to do that, we have to ensure that said flag is valid in the entire
3636 // defined scope of the SCEV.
3637 // TODO: non-instructions have global scope. We might be able to prove
3638 // some global scope cases
3639 auto *GEPI = dyn_cast<Instruction>(GEP);
3640 if (!GEPI || !isSCEVExprNeverPoison(GEPI)) {
3641 BaseNW = OffsetNW = GEPNoWrapFlags::none();
3642 } else if (any_of(IndexExprs, [&](const SCEVUse &IndexExpr) {
3643 const SCEV *Idx = IndexExpr;
3644 return !isa<SCEVConstant>(Idx) &&
3645 !isGuaranteedToTransferExecutionTo(
3646 getDefiningScopeBound({Idx}), GEPI);
3647 })) {
3648 // The offset flags should not take into account the base pointer
3649 // defining scope, as the offset only depends on the indices. Thus,
3650 // derive the scope bound for the offset flags from each index operand
3651 // individually.
3652 OffsetNW = GEPNoWrapFlags::none();
3653 }
3654 }
3655
3656 return getGEPExpr(BaseExpr, IndexExprs, GEP->getSourceElementType(), BaseNW,
3657 OffsetNW);
3658}
3659
3661 ArrayRef<SCEVUse> IndexExprs,
3662 Type *SrcElementTy,
3663 GEPNoWrapFlags BaseNW,
3664 GEPNoWrapFlags OffsetNW) {
3665 SCEVFlags OffsetWrap = SCEV::FlagNone;
3666 if (OffsetNW.hasNoUnsignedSignedWrap())
3667 OffsetWrap = setFlags(OffsetWrap, SCEV::FlagNSW);
3668 if (OffsetNW.hasNoUnsignedWrap())
3669 OffsetWrap = setFlags(OffsetWrap, SCEV::FlagNUW);
3670
3671 Type *CurTy = BaseExpr->getType();
3672 Type *IntIdxTy = getEffectiveSCEVType(BaseExpr->getType());
3673 bool FirstIter = true;
3675 for (SCEVUse IndexExpr : IndexExprs) {
3676 // Compute the (potentially symbolic) offset in bytes for this index.
3677 if (StructType *STy = dyn_cast<StructType>(CurTy)) {
3678 // For a struct, add the member offset.
3679 ConstantInt *Index = cast<SCEVConstant>(IndexExpr)->getValue();
3680 unsigned FieldNo = Index->getZExtValue();
3681 const SCEV *FieldOffset = getOffsetOfExpr(IntIdxTy, STy, FieldNo);
3682 Offsets.push_back(FieldOffset);
3683
3684 // Update CurTy to the type of the field at Index.
3685 CurTy = STy->getTypeAtIndex(Index);
3686 } else {
3687 // Update CurTy to its element type.
3688 if (FirstIter) {
3689 assert(isa<PointerType>(CurTy) &&
3690 "The first index of a GEP indexes a pointer");
3691 CurTy = SrcElementTy;
3692 FirstIter = false;
3693 } else {
3694 CurTy = GetElementPtrInst::getTypeAtIndex(CurTy, (uint64_t)0);
3695 }
3696 // For an array, add the element offset, explicitly scaled.
3697 const SCEV *ElementSize = getSizeOfExpr(IntIdxTy, CurTy);
3698 // Getelementptr indices are signed.
3699 IndexExpr = getTruncateOrSignExtend(IndexExpr, IntIdxTy);
3700
3701 // Multiply the index by the element size to compute the element offset.
3702 const SCEV *LocalOffset = getMulExpr(IndexExpr, ElementSize, OffsetWrap);
3703 Offsets.push_back(LocalOffset);
3704 }
3705 }
3706
3707 // Handle degenerate case of GEP without offsets.
3708 if (Offsets.empty())
3709 return BaseExpr;
3710
3711 // Add the offsets together, assuming nsw if inbounds.
3712 const SCEV *Offset = getAddExpr(Offsets, OffsetWrap);
3713 // Add the base address and the offset. We cannot use the nsw flag, as the
3714 // base address is unsigned. However, if we know that the offset is
3715 // non-negative, we can use nuw.
3716 bool NUW = BaseNW.hasNoUnsignedWrap() ||
3718 SCEVFlags BaseWrap = NUW ? SCEV::FlagNUW : SCEV::FlagNone;
3719 const SCEV *GEPExpr = getAddExpr(BaseExpr, Offset, BaseWrap);
3720 assert(BaseExpr->getType() == GEPExpr->getType() &&
3721 "GEP should not change type mid-flight.");
3722 return GEPExpr;
3723}
3724
3725SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
3727 const Loop *L) {
3728 assert((SCEVType != scAddRecExpr || L) &&
3729 "L must be passed to find existing AddRecs");
3731 ID.AddInteger(SCEVType);
3732 for (SCEVUse Op : Ops)
3733 ID.AddPointer(Op.getOpaqueValue());
3734 if (L)
3735 ID.AddPointer(L);
3737 return UniqueSCEVs.lookup(ID, Token);
3738}
3739
3740const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) {
3741 SCEVFlags Flags = IsNSW ? SCEV::FlagNSW : SCEV::FlagNone;
3742 return getSMaxExpr(Op, getNegativeSCEV(Op, Flags));
3743}
3744
3747 assert(SCEVMinMaxExpr::isMinMaxType(Kind) && "Not a SCEVMinMaxExpr!");
3748 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
3749 if (Ops.size() == 1) return Ops[0];
3750#ifndef NDEBUG
3751 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
3752 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
3753 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
3754 "Operand types don't match!");
3755 assert(Ops[0]->getType()->isPointerTy() ==
3756 Ops[i]->getType()->isPointerTy() &&
3757 "min/max should be consistently pointerish");
3758 }
3759#endif
3760
3761 bool IsSigned = Kind == scSMaxExpr || Kind == scSMinExpr;
3762 bool IsMax = Kind == scSMaxExpr || Kind == scUMaxExpr;
3763
3765 Opts, *this, LI, DT, Ops,
3766 [&](const APInt &C1, const APInt &C2) {
3767 switch (Kind) {
3768 case scSMaxExpr:
3769 return APIntOps::smax(C1, C2);
3770 case scSMinExpr:
3771 return APIntOps::smin(C1, C2);
3772 case scUMaxExpr:
3773 return APIntOps::umax(C1, C2);
3774 case scUMinExpr:
3775 return APIntOps::umin(C1, C2);
3776 default:
3777 llvm_unreachable("Unknown SCEV min/max opcode");
3778 }
3779 },
3780 [&](const APInt &C) {
3781 // identity
3782 if (IsMax)
3783 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3784 else
3785 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3786 },
3787 [&](const APInt &C) {
3788 // absorber
3789 if (IsMax)
3790 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3791 else
3792 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3793 });
3794 if (Folded)
3795 return Folded;
3796
3797 // Check if we have created the same expression before.
3798 if (const SCEV *S = findExistingSCEVInCache(Kind, Ops)) {
3799 return S;
3800 }
3801
3802 // Find the first operation of the same kind
3803 unsigned Idx = 0;
3804 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < Kind)
3805 ++Idx;
3806
3807 // Check to see if one of the operands is of the same kind. If so, expand its
3808 // operands onto our operand list, and recurse to simplify.
3809 if (Idx < Ops.size()) {
3810 bool DeletedAny = false;
3811 while (Ops[Idx]->getSCEVType() == Kind) {
3812 const SCEVMinMaxExpr *SMME = cast<SCEVMinMaxExpr>(Ops[Idx]);
3813 Ops.erase(Ops.begin()+Idx);
3814 append_range(Ops, SMME->operands());
3815 DeletedAny = true;
3816 }
3817
3818 if (DeletedAny)
3819 return getMinMaxExpr(Kind, Ops);
3820 }
3821
3822 // Okay, check to see if the same value occurs in the operand list twice. If
3823 // so, delete one. Since we sorted the list, these values are required to
3824 // be adjacent.
3829 llvm::CmpInst::Predicate FirstPred = IsMax ? GEPred : LEPred;
3830 llvm::CmpInst::Predicate SecondPred = IsMax ? LEPred : GEPred;
3831 for (unsigned i = 0, e = Ops.size() - 1; i != e; ++i) {
3832 if (SCEVUse Common = SCEVUse::getCommon(Ops[i], Ops[i + 1]))
3833 Ops[i] = Ops[i + 1] = Common;
3834 if (Ops[i] == Ops[i + 1] ||
3835 isKnownViaNonRecursiveReasoning(FirstPred, Ops[i], Ops[i + 1])) {
3836 // X op Y op Y --> X op Y
3837 // X op Y --> X, if we know X, Y are ordered appropriately
3838 Ops.erase(Ops.begin() + i + 1, Ops.begin() + i + 2);
3839 --i;
3840 --e;
3841 } else if (isKnownViaNonRecursiveReasoning(SecondPred, Ops[i],
3842 Ops[i + 1])) {
3843 // X op Y --> Y, if we know X, Y are ordered appropriately
3844 Ops.erase(Ops.begin() + i, Ops.begin() + i + 1);
3845 --i;
3846 --e;
3847 }
3848 }
3849
3850 if (Ops.size() == 1) return Ops[0];
3851
3852 assert(!Ops.empty() && "Reduced smax down to nothing!");
3853
3854 // Okay, it looks like we really DO need an expr. Check to see if we
3855 // already have one, otherwise create a new one.
3857 ID.AddInteger(Kind);
3858 for (SCEVUse Op : Ops)
3859 ID.AddPointer(Op.getOpaqueValue());
3861 const SCEV *ExistingSCEV = UniqueSCEVs.lookup(ID, Token);
3862 if (ExistingSCEV)
3863 return ExistingSCEV;
3864 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
3866 SCEV *S = new (SCEVAllocator)
3867 SCEVMinMaxExpr(ID.Intern(SCEVAllocator), Kind, O, Ops.size());
3868
3869 UniqueSCEVs.insert(S, Token);
3870 S->computeAndSetCanonical(*this);
3871 registerUser(S, Ops);
3872 return S;
3873}
3874
3875namespace {
3876
3877class SCEVSequentialMinMaxDeduplicatingVisitor final
3878 : public SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor,
3879 std::optional<const SCEV *>> {
3880 using RetVal = std::optional<const SCEV *>;
3881
3882 ScalarEvolution &SE;
3883 const SCEVTypes RootKind; // Must be a sequential min/max expression.
3884 const SCEVTypes NonSequentialRootKind; // Non-sequential variant of RootKind.
3886
3887 bool canRecurseInto(SCEVTypes Kind) const {
3888 // We can only recurse into the SCEV expression of the same effective type
3889 // as the type of our root SCEV expression.
3890 return RootKind == Kind || NonSequentialRootKind == Kind;
3891 };
3892
3893 RetVal visit(const SCEV *S) {
3894 // Has the whole operand been seen already?
3895 if (!SeenOps.insert(S).second)
3896 return std::nullopt;
3898 SCEVTypes Kind = S->getSCEVType();
3899
3900 if (!canRecurseInto(Kind))
3901 return S;
3902
3903 auto *NAry = cast<SCEVNAryExpr>(S);
3904 SmallVector<SCEVUse> NewOps;
3905 bool Changed = visit(Kind, NAry->operands(), NewOps);
3906
3907 if (!Changed)
3908 return S;
3909 if (NewOps.empty())
3910 return std::nullopt;
3911
3913 ? SE.getSequentialMinMaxExpr(Kind, NewOps)
3914 : SE.getMinMaxExpr(Kind, NewOps);
3915 }
3916 return S;
3917 }
3918
3919public:
3920 SCEVSequentialMinMaxDeduplicatingVisitor(ScalarEvolution &SE,
3921 SCEVTypes RootKind)
3922 : SE(SE), RootKind(RootKind),
3923 NonSequentialRootKind(
3924 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
3925 RootKind)) {}
3926
3927 bool /*Changed*/ visit(SCEVTypes Kind, ArrayRef<SCEVUse> OrigOps,
3928 SmallVectorImpl<SCEVUse> &NewOps) {
3929 bool Changed = false;
3931 Ops.reserve(OrigOps.size());
3932
3933 for (const SCEV *Op : OrigOps) {
3934 RetVal NewOp = visit(Op);
3935 if (NewOp != Op)
3936 Changed = true;
3937 if (NewOp)
3938 Ops.emplace_back(*NewOp);
3939 }
3940
3941 if (Changed)
3942 NewOps = std::move(Ops);
3943 return Changed;
3944 }
3945};
3946
3947} // namespace
3948
3950 switch (Kind) {
3951 case scConstant:
3952 case scVScale:
3953 case scTruncate:
3954 case scZeroExtend:
3955 case scSignExtend:
3956 case scPtrToAddr:
3957 case scAddExpr:
3958 case scMulExpr:
3959 case scUDivExpr:
3960 case scAddRecExpr:
3961 case scUMaxExpr:
3962 case scSMaxExpr:
3963 case scUMinExpr:
3964 case scSMinExpr:
3965 case scUnknown:
3966 // If any operand is poison, the whole expression is poison.
3967 return true;
3969 // FIXME: if the *first* operand is poison, the whole expression is poison.
3970 return false; // Pessimistically, say that it does not propagate poison.
3971 case scCouldNotCompute:
3972 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
3973 }
3974 llvm_unreachable("Unknown SCEV kind!");
3975}
3976
3977namespace {
3978// The only way poison may be introduced in a SCEV expression is from a
3979// poison SCEVUnknown (ConstantExprs are also represented as SCEVUnknown,
3980// not SCEVConstant). Notably, SCEVFlags on SCEV nodes can *not*
3981// introduce poison -- they encode guaranteed, non-speculated knowledge.
3982//
3983// Additionally, all SCEV nodes propagate poison from inputs to outputs,
3984// with the notable exception of umin_seq, where only poison from the first
3985// operand is (unconditionally) propagated.
3986struct SCEVPoisonCollector {
3987 bool LookThroughMaybePoisonBlocking;
3988 SmallPtrSet<const SCEVUnknown *, 4> MaybePoison;
3989 SCEVPoisonCollector(bool LookThroughMaybePoisonBlocking)
3990 : LookThroughMaybePoisonBlocking(LookThroughMaybePoisonBlocking) {}
3991
3992 bool follow(const SCEV *S) {
3993 if (!LookThroughMaybePoisonBlocking &&
3995 return false;
3996
3997 if (auto *SU = dyn_cast<SCEVUnknown>(S)) {
3998 if (!isGuaranteedNotToBePoison(SU->getValue()))
3999 MaybePoison.insert(SU);
4000 }
4001 return true;
4002 }
4003 bool isDone() const { return false; }
4004};
4005} // namespace
4006
4007/// Return true if V is poison given that AssumedPoison is already poison.
4008static bool impliesPoison(const SCEV *AssumedPoison, const SCEV *S) {
4009 // First collect all SCEVs that might result in AssumedPoison to be poison.
4010 // We need to look through potentially poison-blocking operations here,
4011 // because we want to find all SCEVs that *might* result in poison, not only
4012 // those that are *required* to.
4013 SCEVPoisonCollector PC1(/* LookThroughMaybePoisonBlocking */ true);
4014 visitAll(AssumedPoison, PC1);
4015
4016 // AssumedPoison is never poison. As the assumption is false, the implication
4017 // is true. Don't bother walking the other SCEV in this case.
4018 if (PC1.MaybePoison.empty())
4019 return true;
4020
4021 // Collect all SCEVs in S that, if poison, *will* result in S being poison
4022 // as well. We cannot look through potentially poison-blocking operations
4023 // here, as their arguments only *may* make the result poison.
4024 SCEVPoisonCollector PC2(/* LookThroughMaybePoisonBlocking */ false);
4025 visitAll(S, PC2);
4026
4027 // Make sure that no matter which SCEV in PC1.MaybePoison is actually poison,
4028 // it will also make S poison by being part of PC2.MaybePoison.
4029 return llvm::set_is_subset(PC1.MaybePoison, PC2.MaybePoison);
4030}
4031
4033 SmallPtrSetImpl<const Value *> &Result, const SCEV *S) {
4034 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ false);
4035 visitAll(S, PC);
4036 for (const SCEVUnknown *SU : PC.MaybePoison)
4037 Result.insert(SU->getValue());
4038}
4039
4041 const SCEV *S, Instruction *I,
4042 SmallVectorImpl<Instruction *> &DropPoisonGeneratingInsts) {
4043 // If the instruction cannot be poison, it's always safe to reuse.
4045 return true;
4046
4047 // Otherwise, it is possible that I is more poisonous that S. Collect the
4048 // poison-contributors of S, and then check whether I has any additional
4049 // poison-contributors. Poison that is contributed through poison-generating
4050 // flags is handled by dropping those flags instead.
4052 getPoisonGeneratingValues(PoisonVals, S);
4053
4054 SmallVector<Value *> Worklist;
4056 Worklist.push_back(I);
4057 unsigned NumVisitedInsts = 0;
4058 const unsigned InstLimit = std::max<unsigned>(16, S->getExpressionSize());
4059 while (!Worklist.empty()) {
4060 Value *V = Worklist.pop_back_val();
4061 if (!Visited.insert(V).second)
4062 continue;
4063
4064 // Either the value can't be poison, or the S would also be poison if it
4065 // is.
4066 if (PoisonVals.contains(V) || ::isGuaranteedNotToBePoison(V))
4067 continue;
4068
4069 // Avoid walking large instruction graphs.
4070 if (++NumVisitedInsts > InstLimit)
4071 return false;
4072
4073 auto *I = dyn_cast<Instruction>(V);
4074 if (!I)
4075 return false;
4076
4077 // Disjoint or instructions are interpreted as adds by SCEV. However, we
4078 // can't replace an arbitrary add with disjoint or, even if we drop the
4079 // flag. We would need to convert the or into an add.
4080 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(I))
4081 if (PDI->isDisjoint())
4082 return false;
4083
4084 // FIXME: Ignore vscale, even though it technically could be poison. Do this
4085 // because SCEV currently assumes it can't be poison. Remove this special
4086 // case once we proper model when vscale can be poison.
4087 if (auto *II = dyn_cast<IntrinsicInst>(I);
4088 II && II->getIntrinsicID() == Intrinsic::vscale)
4089 continue;
4090
4091 if (canCreatePoison(cast<Operator>(I), /*ConsiderFlagsAndMetadata*/ false))
4092 return false;
4093
4094 // If the instruction can't create poison, we can recurse to its operands.
4095 if (I->hasPoisonGeneratingAnnotations())
4096 DropPoisonGeneratingInsts.push_back(I);
4097
4098 llvm::append_range(Worklist, I->operands());
4099 }
4100 return true;
4101}
4102
4103const SCEV *
4106 assert(SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) &&
4107 "Not a SCEVSequentialMinMaxExpr!");
4108 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
4109 if (Ops.size() == 1)
4110 return Ops[0];
4111#ifndef NDEBUG
4112 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
4113 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4114 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
4115 "Operand types don't match!");
4116 assert(Ops[0]->getType()->isPointerTy() ==
4117 Ops[i]->getType()->isPointerTy() &&
4118 "min/max should be consistently pointerish");
4119 }
4120#endif
4121
4122 // Note that SCEVSequentialMinMaxExpr is *NOT* commutative,
4123 // so we can *NOT* do any kind of sorting of the expressions!
4124
4125 // Check if we have created the same expression before.
4126 if (const SCEV *S = findExistingSCEVInCache(Kind, Ops))
4127 return S;
4128
4129 // FIXME: there are *some* simplifications that we can do here.
4130
4131 // Keep only the first instance of an operand.
4132 {
4133 SCEVSequentialMinMaxDeduplicatingVisitor Deduplicator(*this, Kind);
4134 bool Changed = Deduplicator.visit(Kind, Ops, Ops);
4135 if (Changed)
4136 return getSequentialMinMaxExpr(Kind, Ops);
4137 }
4138
4139 // Check to see if one of the operands is of the same kind. If so, expand its
4140 // operands onto our operand list, and recurse to simplify.
4141 {
4142 unsigned Idx = 0;
4143 bool DeletedAny = false;
4144 while (Idx < Ops.size()) {
4145 if (Ops[Idx]->getSCEVType() != Kind) {
4146 ++Idx;
4147 continue;
4148 }
4149 const auto *SMME = cast<SCEVSequentialMinMaxExpr>(Ops[Idx]);
4150 Ops.erase(Ops.begin() + Idx);
4151 Ops.insert(Ops.begin() + Idx, SMME->operands().begin(),
4152 SMME->operands().end());
4153 DeletedAny = true;
4154 }
4155
4156 if (DeletedAny)
4157 return getSequentialMinMaxExpr(Kind, Ops);
4158 }
4159
4160 const SCEV *SaturationPoint;
4162 switch (Kind) {
4164 SaturationPoint = getZero(Ops[0]->getType());
4165 Pred = ICmpInst::ICMP_ULE;
4166 break;
4167 default:
4168 llvm_unreachable("Not a sequential min/max type.");
4169 }
4170
4171 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4173 continue;
4174 // We can replace %x umin_seq %y with %x umin %y if either:
4175 // * %y being poison implies %x is also poison.
4176 // * %x cannot be the saturating value (e.g. zero for umin).
4177 if (::impliesPoison(Ops[i], Ops[i - 1]) ||
4178 isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_NE, Ops[i - 1],
4179 SaturationPoint)) {
4180 SmallVector<SCEVUse, 2> SeqOps = {Ops[i - 1], Ops[i]};
4181 Ops[i - 1] = getMinMaxExpr(
4183 SeqOps);
4184 Ops.erase(Ops.begin() + i);
4185 return getSequentialMinMaxExpr(Kind, Ops);
4186 }
4187 // Fold %x umin_seq %y to %x if %x ule %y.
4188 // TODO: We might be able to prove the predicate for a later operand.
4189 if (isKnownViaNonRecursiveReasoning(Pred, Ops[i - 1], Ops[i])) {
4190 Ops.erase(Ops.begin() + i);
4191 return getSequentialMinMaxExpr(Kind, Ops);
4192 }
4193 }
4194
4195 // Okay, it looks like we really DO need an expr. Check to see if we
4196 // already have one, otherwise create a new one.
4198 ID.AddInteger(Kind);
4199 for (SCEVUse Op : Ops)
4200 ID.AddPointer(Op.getOpaqueValue());
4202 const SCEV *ExistingSCEV = UniqueSCEVs.lookup(ID, Token);
4203 if (ExistingSCEV)
4204 return ExistingSCEV;
4205
4206 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
4208 SCEV *S = new (SCEVAllocator)
4209 SCEVSequentialMinMaxExpr(ID.Intern(SCEVAllocator), Kind, O, Ops.size());
4210
4211 UniqueSCEVs.insert(S, Token);
4212 S->computeAndSetCanonical(*this);
4213 registerUser(S, Ops);
4214 return S;
4215}
4216
4221
4225
4230
4234
4239
4243
4245 bool Sequential) {
4246 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4247 return getUMinExpr(Ops, Sequential);
4248}
4249
4255
4256const SCEV *
4258 const SCEV *Res = getConstant(IntTy, Size.getKnownMinValue());
4259 if (Size.isScalable())
4260 Res = getMulExpr(Res, getVScale(IntTy));
4261 return Res;
4262}
4263
4265 return getSizeOfExpr(IntTy, getDataLayout().getTypeAllocSize(AllocTy));
4266}
4267
4269 return getSizeOfExpr(IntTy, getDataLayout().getTypeStoreSize(StoreTy));
4270}
4271
4273 StructType *STy,
4274 unsigned FieldNo) {
4275 // We can bypass creating a target-independent constant expression and then
4276 // folding it back into a ConstantInt. This is just a compile-time
4277 // optimization.
4278 const StructLayout *SL = getDataLayout().getStructLayout(STy);
4279 assert(!SL->getSizeInBits().isScalable() &&
4280 "Cannot get offset for structure containing scalable vector types");
4281 return getConstant(IntTy, SL->getElementOffset(FieldNo));
4282}
4283
4285 // Don't attempt to do anything other than create a SCEVUnknown object
4286 // here. createSCEV only calls getUnknown after checking for all other
4287 // interesting possibilities, and any other code that calls getUnknown
4288 // is doing so in order to hide a value from SCEV canonicalization.
4289
4292 ID.AddPointer(V);
4294 if (SCEV *S = UniqueSCEVs.lookup(ID, Token)) {
4295 assert(cast<SCEVUnknown>(S)->getValue() == V &&
4296 "Stale SCEVUnknown in uniquing map!");
4297 return S;
4298 }
4299 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(SCEVAllocator), V, this,
4300 FirstUnknown, NextUnknownOrderID++);
4301 FirstUnknown = cast<SCEVUnknown>(S);
4302 UniqueSCEVs.insert(S, Token);
4303 S->computeAndSetCanonical(*this);
4304 return S;
4305}
4306
4307//===----------------------------------------------------------------------===//
4308// Basic SCEV Analysis and PHI Idiom Recognition Code
4309//
4310
4311/// Test if values of the given type are analyzable within the SCEV
4312/// framework. This primarily includes integer types, and it can optionally
4313/// include pointer types if the ScalarEvolution class has access to
4314/// target-specific information.
4316 // Integers and pointers are always SCEVable.
4317 return Ty->isIntOrPtrTy();
4318}
4319
4320/// Return the size in bits of the specified type, for which isSCEVable must
4321/// return true.
4323 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4324 if (Ty->isPointerTy())
4326 return getDataLayout().getTypeSizeInBits(Ty);
4327}
4328
4329/// Return a type with the same bitwidth as the given type and which represents
4330/// how SCEV will treat the given type, for which isSCEVable must return
4331/// true. For pointer types, this is the pointer index sized integer type.
4333 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4334
4335 if (Ty->isIntegerTy())
4336 return Ty;
4337
4338 // The only other support type is pointer.
4339 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
4340 return getDataLayout().getIndexType(Ty);
4341}
4342
4344 return getTypeSizeInBits(T1) >= getTypeSizeInBits(T2) ? T1 : T2;
4345}
4346
4348 const SCEV *B) {
4349 /// For a valid use point to exist, the defining scope of one operand
4350 /// must dominate the other.
4351 bool PreciseA, PreciseB;
4352 auto *ScopeA = getDefiningScopeBound({A}, PreciseA);
4353 auto *ScopeB = getDefiningScopeBound({B}, PreciseB);
4354 if (!PreciseA || !PreciseB)
4355 // Can't tell.
4356 return false;
4357 return (ScopeA == ScopeB) || DT.dominates(ScopeA, ScopeB) ||
4358 DT.dominates(ScopeB, ScopeA);
4359}
4360
4362 return CouldNotCompute.get();
4363}
4364
4365bool ScalarEvolution::checkValidity(const SCEV *S) const {
4366 bool ContainsNulls = SCEVExprContains(S, [](const SCEV *S) {
4367 auto *SU = dyn_cast<SCEVUnknown>(S);
4368 return SU && SU->getValue() == nullptr;
4369 });
4370
4371 return !ContainsNulls;
4372}
4373
4375 auto [I, Inserted] = HasRecMap.try_emplace(S);
4376 if (Inserted)
4377 I->second = SCEVExprContains(
4378 S, [](const SCEV *S) { return isa<SCEVAddRecExpr>(S); });
4379 return I->second;
4380}
4381
4382/// Return the ValueOffsetPair set for \p S. \p S can be represented
4383/// by the value and offset from any ValueOffsetPair in the set.
4384ArrayRef<Value *> ScalarEvolution::getSCEVValues(const SCEV *S) {
4385 ExprValueMapType::iterator SI = ExprValueMap.find_as(S);
4386 if (SI == ExprValueMap.end())
4387 return {};
4388 return SI->second;
4389}
4390
4391/// Erase Value from ValueExprMap and ExprValueMap. ValueExprMap.erase(V)
4392/// cannot be used separately. eraseValueFromMap should be used to remove
4393/// V from ValueExprMap and ExprValueMap at the same time.
4394void ScalarEvolution::eraseValueFromMap(Value *V) {
4395 ValueExprMapType::iterator I = ValueExprMap.find_as(V);
4396 if (I != ValueExprMap.end()) {
4397 auto EVIt = ExprValueMap.find(I->second);
4398 assert(is_contained(EVIt->second, V) && "Value not in ExprValueMap?");
4399 llvm::erase(EVIt->second, V);
4400 ValueExprMap.erase(I);
4401 }
4402}
4403
4404void ScalarEvolution::insertValueToMap(Value *V, const SCEV *S) {
4405 // A recursive query may have already computed the SCEV. It should be
4406 // equivalent, but may not necessarily be exactly the same, e.g. due to lazily
4407 // inferred nowrap flags.
4408 if (ValueExprMap.insert({SCEVCallbackVH(V, this), S}).second)
4409 ExprValueMap[S].push_back(V);
4410}
4411
4412/// Return an existing SCEV if it exists, otherwise analyze the expression and
4413/// create a new one.
4415 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4416
4417 if (const SCEV *S = getExistingSCEV(V))
4418 return S;
4419 return createSCEVIter(V);
4420}
4421
4423 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4424
4425 ValueExprMapType::iterator I = ValueExprMap.find_as(V);
4426 if (I != ValueExprMap.end()) {
4427 const SCEV *S = I->second;
4428 assert(checkValidity(S) &&
4429 "existing SCEV has not been properly invalidated");
4430 return S;
4431 }
4432 return nullptr;
4433}
4434
4435/// Return a SCEV corresponding to -V = -1*V
4437 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
4438 return getConstant(
4439 cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue())));
4440
4441 Type *Ty = V->getType();
4442 Ty = getEffectiveSCEVType(Ty);
4443 return getMulExpr(V, getMinusOne(Ty), Flags);
4444}
4445
4446/// If Expr computes ~A, return A else return nullptr
4447static const SCEV *MatchNotExpr(const SCEV *Expr) {
4448 const SCEV *MulOp;
4449 if (match(Expr, m_scev_Add(m_scev_AllOnes(),
4450 m_scev_Mul(m_scev_AllOnes(), m_SCEV(MulOp)))))
4451 return MulOp;
4452 return nullptr;
4453}
4454
4455/// Return a SCEV corresponding to ~V = -1-V
4457 assert(!V->getType()->isPointerTy() && "Can't negate pointer");
4458
4459 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
4460 return getConstant(
4461 cast<ConstantInt>(ConstantExpr::getNot(VC->getValue())));
4462
4463 // Fold ~(u|s)(min|max)(~x, ~y) to (u|s)(max|min)(x, y)
4464 if (const SCEVMinMaxExpr *MME = dyn_cast<SCEVMinMaxExpr>(V)) {
4465 auto MatchMinMaxNegation = [&](const SCEVMinMaxExpr *MME) {
4466 SmallVector<SCEVUse, 2> MatchedOperands;
4467 for (const SCEV *Operand : MME->operands()) {
4468 const SCEV *Matched = MatchNotExpr(Operand);
4469 if (!Matched)
4470 return (const SCEV *)nullptr;
4471 MatchedOperands.push_back(Matched);
4472 }
4473 return getMinMaxExpr(SCEVMinMaxExpr::negate(MME->getSCEVType()),
4474 MatchedOperands);
4475 };
4476 if (const SCEV *Replaced = MatchMinMaxNegation(MME))
4477 return Replaced;
4478 }
4479
4480 Type *Ty = V->getType();
4481 Ty = getEffectiveSCEVType(Ty);
4482 return getMinusSCEV(getMinusOne(Ty), V);
4483}
4484
4486 assert(P->getType()->isPointerTy());
4487
4488 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(P)) {
4489 // The base of an AddRec is the first operand.
4490 SmallVector<SCEVUse> Ops{AddRec->operands()};
4491 Ops[0] = removePointerBase(Ops[0]);
4492 // Don't try to transfer nowrap flags for now. We could in some cases
4493 // (for example, if pointer operand of the AddRec is a SCEVUnknown).
4494 return getAddRecExpr(Ops, AddRec->getLoop(), SCEV::FlagNone);
4495 }
4496 if (auto *Add = dyn_cast<SCEVAddExpr>(P)) {
4497 // The base of an Add is the pointer operand.
4498 SmallVector<SCEVUse> Ops{Add->operands()};
4499 SCEVUse *PtrOp = nullptr;
4500 for (SCEVUse &AddOp : Ops) {
4501 if (AddOp->getType()->isPointerTy()) {
4502 assert(!PtrOp && "Cannot have multiple pointer ops");
4503 PtrOp = &AddOp;
4504 }
4505 }
4506 *PtrOp = removePointerBase(*PtrOp);
4507 // Don't try to transfer nowrap flags for now. We could in some cases
4508 // (for example, if the pointer operand of the Add is a SCEVUnknown).
4509 return getAddExpr(Ops);
4510 }
4511 // Any other expression must be a pointer base.
4512 return getZero(P->getType());
4513}
4514
4516 SCEVFlags Flags, unsigned Depth) {
4517 // Fast path: X - X --> 0.
4518 if (SCEVUse::getCommon(LHS, RHS))
4519 return getZero(LHS->getType());
4520
4521 // If we subtract two pointers with different pointer bases, bail.
4522 // Eventually, we're going to add an assertion to getMulExpr that we
4523 // can't multiply by a pointer.
4524 if (RHS->getType()->isPointerTy()) {
4525 if (!LHS->getType()->isPointerTy() ||
4526 getPointerBase(LHS) != getPointerBase(RHS))
4527 return getCouldNotCompute();
4528 LHS = removePointerBase(LHS);
4529 RHS = removePointerBase(RHS);
4530 }
4531
4532 // We represent LHS - RHS as LHS + (-1)*RHS. This transformation
4533 // makes it so that we cannot make much use of NUW.
4534 auto AddFlags = SCEV::FlagNone;
4535 const bool RHSIsNotMinSigned =
4537 if (hasFlags(Flags, SCEV::FlagNSW)) {
4538 // Let M be the minimum representable signed value. Then (-1)*RHS
4539 // signed-wraps if and only if RHS is M. That can happen even for
4540 // a NSW subtraction because e.g. (-1)*M signed-wraps even though
4541 // -1 - M does not. So to transfer NSW from LHS - RHS to LHS +
4542 // (-1)*RHS, we need to prove that RHS != M.
4543 //
4544 // If LHS is non-negative and we know that LHS - RHS does not
4545 // signed-wrap, then RHS cannot be M. So we can rule out signed-wrap
4546 // either by proving that RHS > M or that LHS >= 0.
4547 if (RHSIsNotMinSigned || isKnownNonNegative(LHS)) {
4548 AddFlags = SCEV::FlagNSW;
4549 }
4550 }
4551
4552 // FIXME: Find a correct way to transfer NSW to (-1)*M when LHS -
4553 // RHS is NSW and LHS >= 0.
4554 //
4555 // The difficulty here is that the NSW flag may have been proven
4556 // relative to a loop that is to be found in a recurrence in LHS and
4557 // not in RHS. Applying NSW to (-1)*M may then let the NSW have a
4558 // larger scope than intended.
4559 auto NegFlags = RHSIsNotMinSigned ? SCEV::FlagNSW : SCEV::FlagNone;
4560
4561 return getAddExpr(LHS, getNegativeSCEV(RHS, NegFlags), AddFlags, Depth);
4562}
4563
4565 unsigned Depth) {
4566 Type *SrcTy = V->getType();
4567 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4568 "Cannot truncate or zero extend with non-integer arguments!");
4569 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4570 return V; // No conversion
4571 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
4572 return getTruncateExpr(V, Ty, Depth);
4573 return getZeroExtendExpr(V, Ty, Depth);
4574}
4575
4577 unsigned Depth) {
4578 Type *SrcTy = V->getType();
4579 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4580 "Cannot truncate or zero extend with non-integer arguments!");
4581 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4582 return V; // No conversion
4583 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
4584 return getTruncateExpr(V, Ty, Depth);
4585 return getSignExtendExpr(V, Ty, Depth);
4586}
4587
4589 Type *SrcTy = V->getType();
4590 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4591 "Cannot noop or zero extend with non-integer arguments!");
4593 "getNoopOrZeroExtend cannot truncate!");
4594 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4595 return V; // No conversion
4596 return getZeroExtendExpr(V, Ty);
4597}
4598
4600 Type *SrcTy = V->getType();
4601 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4602 "Cannot noop or sign extend with non-integer arguments!");
4604 "getNoopOrSignExtend cannot truncate!");
4605 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4606 return V; // No conversion
4607 return getSignExtendExpr(V, Ty);
4608}
4609
4611 Type *SrcTy = V->getType();
4612 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4613 "Cannot noop or any extend with non-integer arguments!");
4615 "getNoopOrAnyExtend cannot truncate!");
4616 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4617 return V; // No conversion
4618 return getAnyExtendExpr(V, Ty);
4619}
4620
4622 Type *SrcTy = V->getType();
4623 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4624 "Cannot truncate or noop with non-integer arguments!");
4626 "getTruncateOrNoop cannot extend!");
4627 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4628 return V; // No conversion
4629 return getTruncateExpr(V, Ty);
4630}
4631
4633 const SCEV *RHS) {
4634 const SCEV *PromotedLHS = LHS;
4635 const SCEV *PromotedRHS = RHS;
4636
4637 if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType()))
4638 PromotedRHS = getZeroExtendExpr(RHS, LHS->getType());
4639 else
4640 PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType());
4641
4642 return getUMaxExpr(PromotedLHS, PromotedRHS);
4643}
4644
4646 const SCEV *RHS,
4647 bool Sequential) {
4648 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4649 return getUMinFromMismatchedTypes(Ops, Sequential);
4650}
4651
4652const SCEV *
4654 bool Sequential) {
4655 assert(!Ops.empty() && "At least one operand must be!");
4656 // Trivial case.
4657 if (Ops.size() == 1)
4658 return Ops[0];
4659
4660 // Find the max type first.
4661 Type *MaxType = nullptr;
4662 for (SCEVUse S : Ops)
4663 if (MaxType)
4664 MaxType = getWiderType(MaxType, S->getType());
4665 else
4666 MaxType = S->getType();
4667 assert(MaxType && "Failed to find maximum type!");
4668
4669 // Extend all ops to max type.
4670 SmallVector<SCEVUse, 2> PromotedOps;
4671 for (SCEVUse S : Ops)
4672 PromotedOps.push_back(getNoopOrZeroExtend(S, MaxType));
4673
4674 // Generate umin.
4675 return getUMinExpr(PromotedOps, Sequential);
4676}
4677
4679 // A pointer operand may evaluate to a nonpointer expression, such as null.
4680 if (!V->getType()->isPointerTy())
4681 return V;
4682
4683 while (true) {
4684 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(V)) {
4685 V = AddRec->getStart();
4686 } else if (auto *Add = dyn_cast<SCEVAddExpr>(V)) {
4687 const SCEV *PtrOp = nullptr;
4688 for (const SCEV *AddOp : Add->operands()) {
4689 if (AddOp->getType()->isPointerTy()) {
4690 assert(!PtrOp && "Cannot have multiple pointer ops");
4691 PtrOp = AddOp;
4692 }
4693 }
4694 assert(PtrOp && "Must have pointer op");
4695 V = PtrOp;
4696 } else // Not something we can look further into.
4697 return V;
4698 }
4699}
4700
4701/// Push users of the given Instruction onto the given Worklist.
4705 // Push the def-use children onto the Worklist stack.
4706 for (User *U : I->users()) {
4707 auto *UserInsn = cast<Instruction>(U);
4708 if (Visited.insert(UserInsn).second)
4709 Worklist.push_back(UserInsn);
4710 }
4711}
4712
4713namespace {
4714
4715/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its start
4716/// expression in case its Loop is L. If it is not L then
4717/// if IgnoreOtherLoops is true then use AddRec itself
4718/// otherwise rewrite cannot be done.
4719/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4720class SCEVInitRewriter : public SCEVRewriteVisitor<SCEVInitRewriter> {
4721public:
4722 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
4723 bool IgnoreOtherLoops = true) {
4724 SCEVInitRewriter Rewriter(L, SE);
4725 const SCEV *Result = Rewriter.visit(S);
4726 if (Rewriter.hasSeenLoopVariantSCEVUnknown())
4727 return SE.getCouldNotCompute();
4728 return Rewriter.hasSeenOtherLoops() && !IgnoreOtherLoops
4729 ? SE.getCouldNotCompute()
4730 : Result;
4731 }
4732
4733 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4734 if (!SE.isLoopInvariant(Expr, L))
4735 SeenLoopVariantSCEVUnknown = true;
4736 return Expr;
4737 }
4738
4739 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4740 // Only re-write AddRecExprs for this loop.
4741 if (Expr->getLoop() == L)
4742 return Expr->getStart();
4743 SeenOtherLoops = true;
4744 return Expr;
4745 }
4746
4747 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4748
4749 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4750
4751private:
4752 explicit SCEVInitRewriter(const Loop *L, ScalarEvolution &SE)
4753 : SCEVRewriteVisitor(SE), L(L) {}
4754
4755 const Loop *L;
4756 bool SeenLoopVariantSCEVUnknown = false;
4757 bool SeenOtherLoops = false;
4758};
4759
4760/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its post
4761/// increment expression in case its Loop is L. If it is not L then
4762/// use AddRec itself.
4763/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4764class SCEVPostIncRewriter : public SCEVRewriteVisitor<SCEVPostIncRewriter> {
4765public:
4766 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE) {
4767 SCEVPostIncRewriter Rewriter(L, SE);
4768 const SCEV *Result = Rewriter.visit(S);
4769 return Rewriter.hasSeenLoopVariantSCEVUnknown()
4770 ? SE.getCouldNotCompute()
4771 : Result;
4772 }
4773
4774 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4775 if (!SE.isLoopInvariant(Expr, L))
4776 SeenLoopVariantSCEVUnknown = true;
4777 return Expr;
4778 }
4779
4780 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4781 // Only re-write AddRecExprs for this loop.
4782 if (Expr->getLoop() == L)
4783 return Expr->getPostIncExpr(SE);
4784 SeenOtherLoops = true;
4785 return Expr;
4786 }
4787
4788 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4789
4790 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4791
4792private:
4793 explicit SCEVPostIncRewriter(const Loop *L, ScalarEvolution &SE)
4794 : SCEVRewriteVisitor(SE), L(L) {}
4795
4796 const Loop *L;
4797 bool SeenLoopVariantSCEVUnknown = false;
4798 bool SeenOtherLoops = false;
4799};
4800
4801/// This class evaluates the compare condition by matching it against the
4802/// condition of loop latch. If there is a match we assume a true value
4803/// for the condition while building SCEV nodes.
4804class SCEVBackedgeConditionFolder
4805 : public SCEVRewriteVisitor<SCEVBackedgeConditionFolder> {
4806public:
4807 static const SCEV *rewrite(const SCEV *S, const Loop *L,
4808 ScalarEvolution &SE) {
4809 bool IsPosBECond = false;
4810 Value *BECond = nullptr;
4811 if (BasicBlock *Latch = L->getLoopLatch()) {
4812 if (CondBrInst *BI = dyn_cast<CondBrInst>(Latch->getTerminator())) {
4813 assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
4814 "Both outgoing branches should not target same header!");
4815 BECond = BI->getCondition();
4816 IsPosBECond = BI->getSuccessor(0) == L->getHeader();
4817 } else {
4818 return S;
4819 }
4820 }
4821 SCEVBackedgeConditionFolder Rewriter(L, BECond, IsPosBECond, SE);
4822 return Rewriter.visit(S);
4823 }
4824
4825 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4826 const SCEV *Result = Expr;
4827 bool InvariantF = SE.isLoopInvariant(Expr, L);
4828
4829 if (!InvariantF) {
4831 switch (I->getOpcode()) {
4832 case Instruction::Select: {
4833 SelectInst *SI = cast<SelectInst>(I);
4834 std::optional<const SCEV *> Res =
4835 compareWithBackedgeCondition(SI->getCondition());
4836 if (Res) {
4837 bool IsOne = cast<SCEVConstant>(*Res)->getValue()->isOne();
4838 Result = SE.getSCEV(IsOne ? SI->getTrueValue() : SI->getFalseValue());
4839 }
4840 break;
4841 }
4842 default: {
4843 std::optional<const SCEV *> Res = compareWithBackedgeCondition(I);
4844 if (Res)
4845 Result = *Res;
4846 break;
4847 }
4848 }
4849 }
4850 return Result;
4851 }
4852
4853private:
4854 explicit SCEVBackedgeConditionFolder(const Loop *L, Value *BECond,
4855 bool IsPosBECond, ScalarEvolution &SE)
4856 : SCEVRewriteVisitor(SE), L(L), BackedgeCond(BECond),
4857 IsPositiveBECond(IsPosBECond) {}
4858
4859 std::optional<const SCEV *> compareWithBackedgeCondition(Value *IC);
4860
4861 const Loop *L;
4862 /// Loop back condition.
4863 Value *BackedgeCond = nullptr;
4864 /// Set to true if loop back is on positive branch condition.
4865 bool IsPositiveBECond;
4866};
4867
4868std::optional<const SCEV *>
4869SCEVBackedgeConditionFolder::compareWithBackedgeCondition(Value *IC) {
4870
4871 // If value matches the backedge condition for loop latch,
4872 // then return a constant evolution node based on loopback
4873 // branch taken.
4874 if (BackedgeCond == IC)
4875 return IsPositiveBECond ? SE.getOne(Type::getInt1Ty(SE.getContext()))
4877 return std::nullopt;
4878}
4879
4880class SCEVShiftRewriter : public SCEVRewriteVisitor<SCEVShiftRewriter> {
4881public:
4882 static const SCEV *rewrite(const SCEV *S, const Loop *L,
4883 ScalarEvolution &SE) {
4884 SCEVShiftRewriter Rewriter(L, SE);
4885 const SCEV *Result = Rewriter.visit(S);
4886 return Rewriter.isValid() ? Result : SE.getCouldNotCompute();
4887 }
4888
4889 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4890 // Only allow AddRecExprs for this loop.
4891 if (!SE.isLoopInvariant(Expr, L))
4892 Valid = false;
4893 return Expr;
4894 }
4895
4896 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4897 if (Expr->getLoop() == L && Expr->isAffine())
4898 return SE.getMinusSCEV(Expr, Expr->getStepRecurrence(SE));
4899 Valid = false;
4900 return Expr;
4901 }
4902
4903 bool isValid() { return Valid; }
4904
4905private:
4906 explicit SCEVShiftRewriter(const Loop *L, ScalarEvolution &SE)
4907 : SCEVRewriteVisitor(SE), L(L) {}
4908
4909 const Loop *L;
4910 bool Valid = true;
4911};
4912
4913} // end anonymous namespace
4914
4915void ScalarEvolution::inferNoWrapViaConstantRanges(const SCEVAddRecExpr *AR) {
4916 if (!AR->isAffine())
4917 return;
4918
4919 // Force computation of ranges, which will also perform range-based flag
4920 // inference.
4921 if (!AR->hasNoSignedWrap())
4922 (void)getSignedRange(AR);
4923
4924 if (!AR->hasNoUnsignedWrap())
4925 (void)getUnsignedRange(AR);
4926
4927 if (!AR->hasNoSelfWrap()) {
4928 const SCEV *BECount = getConstantMaxBackedgeTakenCount(AR->getLoop());
4929 if (const SCEVConstant *BECountMax = dyn_cast<SCEVConstant>(BECount)) {
4930 ConstantRange StepCR = getSignedRange(AR->getStepRecurrence(*this));
4931 const APInt &BECountAP = BECountMax->getAPInt();
4932 unsigned NoOverflowBitWidth =
4933 BECountAP.getActiveBits() + StepCR.getMinSignedBits();
4934 if (NoOverflowBitWidth <= getTypeSizeInBits(AR->getType()))
4935 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
4936 }
4937 }
4938}
4939
4941ScalarEvolution::proveNoSignedWrapViaInduction(const SCEVAddRecExpr *AR) {
4943
4944 if (AR->hasNoSignedWrap())
4945 return Result;
4946
4947 if (!AR->isAffine())
4948 return Result;
4949
4950 // This function can be expensive, only try to prove NSW once per AddRec.
4951 if (!SignedWrapViaInductionTried.insert(AR).second)
4952 return Result;
4953
4954 const SCEV *Step = AR->getStepRecurrence(*this);
4955 const Loop *L = AR->getLoop();
4956
4957 // Check whether the backedge-taken count is SCEVCouldNotCompute.
4958 // Note that this serves two purposes: It filters out loops that are
4959 // simply not analyzable, and it covers the case where this code is
4960 // being called from within backedge-taken count analysis, such that
4961 // attempting to ask for the backedge-taken count would likely result
4962 // in infinite recursion. In the later case, the analysis code will
4963 // cope with a conservative value, and it will take care to purge
4964 // that value once it has finished.
4965 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
4966
4967 // Normally, in the cases we can prove no-overflow via a
4968 // backedge guarding condition, we can also compute a backedge
4969 // taken count for the loop. The exceptions are assumptions and
4970 // guards present in the loop -- SCEV is not great at exploiting
4971 // these to compute max backedge taken counts, but can still use
4972 // these to prove lack of overflow. Use this fact to avoid
4973 // doing extra work that may not pay off.
4974
4975 if (isa<SCEVCouldNotCompute>(MaxBECount) && !HasGuards &&
4976 AC.assumptions().empty())
4977 return Result;
4978
4979 // If the backedge is guarded by a comparison with the pre-inc value the
4980 // addrec is safe. Also, if the entry is guarded by a comparison with the
4981 // start value and the backedge is guarded by a comparison with the post-inc
4982 // value, the addrec is safe.
4984 const SCEV *OverflowLimit =
4985 getSignedOverflowLimitForStep(Step, &Pred, this);
4986 if (OverflowLimit &&
4987 (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) ||
4988 isKnownOnEveryIteration(Pred, AR, OverflowLimit))) {
4989 Result = setFlags(Result, SCEV::FlagNSW);
4990 }
4991 return Result;
4992}
4994ScalarEvolution::proveNoUnsignedWrapViaInduction(const SCEVAddRecExpr *AR) {
4996
4997 if (AR->hasNoUnsignedWrap())
4998 return Result;
4999
5000 if (!AR->isAffine())
5001 return Result;
5002
5003 // This function can be expensive, only try to prove NUW once per AddRec.
5004 if (!UnsignedWrapViaInductionTried.insert(AR).second)
5005 return Result;
5006
5007 const SCEV *Step = AR->getStepRecurrence(*this);
5008 const Loop *L = AR->getLoop();
5009
5010 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5011 // Note that this serves two purposes: It filters out loops that are
5012 // simply not analyzable, and it covers the case where this code is
5013 // being called from within backedge-taken count analysis, such that
5014 // attempting to ask for the backedge-taken count would likely result
5015 // in infinite recursion. In the later case, the analysis code will
5016 // cope with a conservative value, and it will take care to purge
5017 // that value once it has finished.
5018 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5019
5020 // Normally, in the cases we can prove no-overflow via a
5021 // backedge guarding condition, we can also compute a backedge
5022 // taken count for the loop. The exceptions are assumptions and
5023 // guards present in the loop -- SCEV is not great at exploiting
5024 // these to compute max backedge taken counts, but can still use
5025 // these to prove lack of overflow. Use this fact to avoid
5026 // doing extra work that may not pay off.
5027
5028 if (isa<SCEVCouldNotCompute>(MaxBECount) && !HasGuards &&
5029 AC.assumptions().empty())
5030 return Result;
5031
5032 // If the backedge is guarded by a comparison with the pre-inc value the
5033 // addrec is safe. Also, if the entry is guarded by a comparison with the
5034 // start value and the backedge is guarded by a comparison with the post-inc
5035 // value, the addrec is safe.
5036 if (isKnownPositive(Step)) {
5038 const SCEV *OverflowLimit =
5040 if (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) ||
5041 isKnownOnEveryIteration(Pred, AR, OverflowLimit))
5042 Result = setFlags(Result, SCEV::FlagNUW);
5043 }
5044 return Result;
5045}
5046
5047namespace {
5048
5049/// Represents an abstract binary operation. This may exist as a
5050/// normal instruction or constant expression, or may have been
5051/// derived from an expression tree.
5052struct BinaryOp {
5053 unsigned Opcode;
5054 Value *LHS;
5055 Value *RHS;
5056 bool IsNSW = false;
5057 bool IsNUW = false;
5058
5059 /// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or
5060 /// constant expression.
5061 Operator *Op = nullptr;
5062
5063 explicit BinaryOp(Operator *Op)
5064 : Opcode(Op->getOpcode()), LHS(Op->getOperand(0)), RHS(Op->getOperand(1)),
5065 Op(Op) {
5066 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Op)) {
5067 IsNSW = OBO->hasNoSignedWrap();
5068 IsNUW = OBO->hasNoUnsignedWrap();
5069 }
5070 }
5071
5072 explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false,
5073 bool IsNUW = false)
5074 : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW) {}
5075};
5076
5077} // end anonymous namespace
5078
5079/// Try to map \p V into a BinaryOp, and return \c std::nullopt on failure.
5080static std::optional<BinaryOp> MatchBinaryOp(Value *V,
5081 const DominatorTree &DT) {
5082 auto *Op = dyn_cast<Operator>(V);
5083 if (!Op)
5084 return std::nullopt;
5085
5086 // Implementation detail: all the cleverness here should happen without
5087 // creating new SCEV expressions -- our caller knowns tricks to avoid creating
5088 // SCEV expressions when possible, and we should not break that.
5089
5090 switch (Op->getOpcode()) {
5091 case Instruction::Add:
5092 case Instruction::Sub:
5093 case Instruction::Mul:
5094 case Instruction::UDiv:
5095 case Instruction::URem:
5096 case Instruction::And:
5097 case Instruction::AShr:
5098 case Instruction::Shl:
5099 return BinaryOp(Op);
5100
5101 case Instruction::Or: {
5102 // Convert or disjoint into add nuw nsw.
5103 if (cast<PossiblyDisjointInst>(Op)->isDisjoint()) {
5104 BinaryOp BinOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1),
5105 /*IsNSW=*/true, /*IsNUW=*/true);
5106 // Keep the reference to the original instruction so that we can later
5107 // check whether it can produce poison value or not.
5108 BinOp.Op = Op;
5109 return BinOp;
5110 }
5111 return BinaryOp(Op);
5112 }
5113
5114 case Instruction::Xor:
5115 if (auto *RHSC = dyn_cast<ConstantInt>(Op->getOperand(1)))
5116 // If the RHS of the xor is a signmask, then this is just an add.
5117 // Instcombine turns add of signmask into xor as a strength reduction step.
5118 if (RHSC->getValue().isSignMask())
5119 return BinaryOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1));
5120 // Binary `xor` is a bit-wise `add`.
5121 if (V->getType()->isIntegerTy(1))
5122 return BinaryOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1));
5123 return BinaryOp(Op);
5124
5125 case Instruction::LShr:
5126 // Turn logical shift right of a constant into a unsigned divide.
5127 if (ConstantInt *SA = dyn_cast<ConstantInt>(Op->getOperand(1))) {
5128 uint32_t BitWidth = cast<IntegerType>(Op->getType())->getBitWidth();
5129
5130 // If the shift count is not less than the bitwidth, the result of
5131 // the shift is undefined. Don't try to analyze it, because the
5132 // resolution chosen here may differ from the resolution chosen in
5133 // other parts of the compiler.
5134 if (SA->getValue().ult(BitWidth)) {
5135 Constant *X =
5136 ConstantInt::get(SA->getContext(),
5137 APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
5138 return BinaryOp(Instruction::UDiv, Op->getOperand(0), X);
5139 }
5140 }
5141 return BinaryOp(Op);
5142
5143 case Instruction::ExtractValue: {
5144 auto *EVI = cast<ExtractValueInst>(Op);
5145 if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0)
5146 break;
5147
5148 auto *WO = dyn_cast<WithOverflowInst>(EVI->getAggregateOperand());
5149 if (!WO)
5150 break;
5151
5152 Instruction::BinaryOps BinOp = WO->getBinaryOp();
5153 bool Signed = WO->isSigned();
5154 // TODO: Should add nuw/nsw flags for mul as well.
5155 if (BinOp == Instruction::Mul || !isOverflowIntrinsicNoWrap(WO, DT))
5156 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS());
5157
5158 // Now that we know that all uses of the arithmetic-result component of
5159 // CI are guarded by the overflow check, we can go ahead and pretend
5160 // that the arithmetic is non-overflowing.
5161 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS(),
5162 /* IsNSW = */ Signed, /* IsNUW = */ !Signed);
5163 }
5164
5165 default:
5166 break;
5167 }
5168
5169 // Recognise intrinsic loop.decrement.reg, and as this has exactly the same
5170 // semantics as a Sub, return a binary sub expression.
5171 if (auto *II = dyn_cast<IntrinsicInst>(V))
5172 if (II->getIntrinsicID() == Intrinsic::loop_decrement_reg)
5173 return BinaryOp(Instruction::Sub, II->getOperand(0), II->getOperand(1));
5174
5175 return std::nullopt;
5176}
5177
5178/// Helper function to createAddRecFromPHIWithCasts. We have a phi
5179/// node whose symbolic (unknown) SCEV is \p SymbolicPHI, which is updated via
5180/// the loop backedge by a SCEVAddExpr, possibly also with a few casts on the
5181/// way. This function checks if \p Op, an operand of this SCEVAddExpr,
5182/// follows one of the following patterns:
5183/// Op == (SExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5184/// Op == (ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5185/// If the SCEV expression of \p Op conforms with one of the expected patterns
5186/// we return the type of the truncation operation, and indicate whether the
5187/// truncated type should be treated as signed/unsigned by setting
5188/// \p Signed to true/false, respectively.
5189static Type *isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI,
5190 bool &Signed, ScalarEvolution &SE) {
5191 // The case where Op == SymbolicPHI (that is, with no type conversions on
5192 // the way) is handled by the regular add recurrence creating logic and
5193 // would have already been triggered in createAddRecForPHI. Reaching it here
5194 // means that createAddRecFromPHI had failed for this PHI before (e.g.,
5195 // because one of the other operands of the SCEVAddExpr updating this PHI is
5196 // not invariant).
5197 //
5198 // Here we look for the case where Op = (ext(trunc(SymbolicPHI))), and in
5199 // this case predicates that allow us to prove that Op == SymbolicPHI will
5200 // be added.
5201 if (Op == SymbolicPHI)
5202 return nullptr;
5203
5204 unsigned SourceBits = SE.getTypeSizeInBits(SymbolicPHI->getType());
5205 unsigned NewBits = SE.getTypeSizeInBits(Op->getType());
5206 if (SourceBits != NewBits)
5207 return nullptr;
5208
5209 if (match(Op, m_scev_SExt(m_scev_Trunc(m_scev_Specific(SymbolicPHI))))) {
5210 Signed = true;
5211 return cast<SCEVCastExpr>(Op)->getOperand()->getType();
5212 }
5213 if (match(Op, m_scev_ZExt(m_scev_Trunc(m_scev_Specific(SymbolicPHI))))) {
5214 Signed = false;
5215 return cast<SCEVCastExpr>(Op)->getOperand()->getType();
5216 }
5217 return nullptr;
5218}
5219
5220static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) {
5221 if (!PN->getType()->isIntegerTy())
5222 return nullptr;
5223 const Loop *L = LI.getLoopFor(PN->getParent());
5224 if (!L || L->getHeader() != PN->getParent())
5225 return nullptr;
5226 return L;
5227}
5228
5229// Analyze \p SymbolicPHI, a SCEV expression of a phi node, and check if the
5230// computation that updates the phi follows the following pattern:
5231// (SExt/ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy) + InvariantAccum
5232// which correspond to a phi->trunc->sext/zext->add->phi update chain.
5233// If so, try to see if it can be rewritten as an AddRecExpr under some
5234// Predicates. If successful, return them as a pair. Also cache the results
5235// of the analysis.
5236//
5237// Example usage scenario:
5238// Say the Rewriter is called for the following SCEV:
5239// 8 * ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5240// where:
5241// %X = phi i64 (%Start, %BEValue)
5242// It will visitMul->visitAdd->visitSExt->visitTrunc->visitUnknown(%X),
5243// and call this function with %SymbolicPHI = %X.
5244//
5245// The analysis will find that the value coming around the backedge has
5246// the following SCEV:
5247// BEValue = ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5248// Upon concluding that this matches the desired pattern, the function
5249// will return the pair {NewAddRec, SmallPredsVec} where:
5250// NewAddRec = {%Start,+,%Step}
5251// SmallPredsVec = {P1, P2, P3} as follows:
5252// P1(WrapPred): AR: {trunc(%Start),+,(trunc %Step)}<nsw> Flags: <nssw>
5253// P2(EqualPred): %Start == (sext i32 (trunc i64 %Start to i32) to i64)
5254// P3(EqualPred): %Step == (sext i32 (trunc i64 %Step to i32) to i64)
5255// The returned pair means that SymbolicPHI can be rewritten into NewAddRec
5256// under the predicates {P1,P2,P3}.
5257// This predicated rewrite will be cached in PredicatedSCEVRewrites:
5258// PredicatedSCEVRewrites[{%X,L}] = {NewAddRec, {P1,P2,P3)}
5259//
5260// TODO's:
5261//
5262// 1) Extend the Induction descriptor to also support inductions that involve
5263// casts: When needed (namely, when we are called in the context of the
5264// vectorizer induction analysis), a Set of cast instructions will be
5265// populated by this method, and provided back to isInductionPHI. This is
5266// needed to allow the vectorizer to properly record them to be ignored by
5267// the cost model and to avoid vectorizing them (otherwise these casts,
5268// which are redundant under the runtime overflow checks, will be
5269// vectorized, which can be costly).
5270//
5271// 2) Support additional induction/PHISCEV patterns: We also want to support
5272// inductions where the sext-trunc / zext-trunc operations (partly) occur
5273// after the induction update operation (the induction increment):
5274//
5275// (Trunc iy (SExt/ZExt ix (%SymbolicPHI + InvariantAccum) to iy) to ix)
5276// which correspond to a phi->add->trunc->sext/zext->phi update chain.
5277//
5278// (Trunc iy ((SExt/ZExt ix (%SymbolicPhi) to iy) + InvariantAccum) to ix)
5279// which correspond to a phi->trunc->add->sext/zext->phi update chain.
5280//
5281// 3) Outline common code with createAddRecFromPHI to avoid duplication.
5282std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5283ScalarEvolution::createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI) {
5285
5286 // *** Part1: Analyze if we have a phi-with-cast pattern for which we can
5287 // return an AddRec expression under some predicate.
5288
5289 auto *PN = cast<PHINode>(SymbolicPHI->getValue());
5290 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5291 assert(L && "Expecting an integer loop header phi");
5292
5293 // The loop may have multiple entrances or multiple exits; we can analyze
5294 // this phi as an addrec if it has a unique entry value and a unique
5295 // backedge value.
5296 Value *BEValueV = nullptr, *StartValueV = nullptr;
5297 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5298 Value *V = PN->getIncomingValue(i);
5299 if (L->contains(PN->getIncomingBlock(i))) {
5300 if (!BEValueV) {
5301 BEValueV = V;
5302 } else if (BEValueV != V) {
5303 BEValueV = nullptr;
5304 break;
5305 }
5306 } else if (!StartValueV) {
5307 StartValueV = V;
5308 } else if (StartValueV != V) {
5309 StartValueV = nullptr;
5310 break;
5311 }
5312 }
5313 if (!BEValueV || !StartValueV)
5314 return std::nullopt;
5315
5316 const SCEV *BEValue = getSCEV(BEValueV);
5317
5318 // If the value coming around the backedge is an add with the symbolic
5319 // value we just inserted, possibly with casts that we can ignore under
5320 // an appropriate runtime guard, then we found a simple induction variable!
5321 const auto *Add = dyn_cast<SCEVAddExpr>(BEValue);
5322 if (!Add)
5323 return std::nullopt;
5324
5325 // If there is a single occurrence of the symbolic value, possibly
5326 // casted, replace it with a recurrence.
5327 unsigned FoundIndex = Add->getNumOperands();
5328 Type *TruncTy = nullptr;
5329 bool Signed;
5330 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5331 if ((TruncTy =
5332 isSimpleCastedPHI(Add->getOperand(i), SymbolicPHI, Signed, *this)))
5333 if (FoundIndex == e) {
5334 FoundIndex = i;
5335 break;
5336 }
5337
5338 if (FoundIndex == Add->getNumOperands())
5339 return std::nullopt;
5340
5341 // Create an add with everything but the specified operand.
5343 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5344 if (i != FoundIndex)
5345 Ops.push_back(Add->getOperand(i));
5346 const SCEV *Accum = getAddExpr(Ops);
5347
5348 // The runtime checks will not be valid if the step amount is
5349 // varying inside the loop.
5350 if (!isLoopInvariant(Accum, L))
5351 return std::nullopt;
5352
5353 // *** Part2: Create the predicates
5354
5355 // Analysis was successful: we have a phi-with-cast pattern for which we
5356 // can return an AddRec expression under the following predicates:
5357 //
5358 // P1: A Wrap predicate that guarantees that Trunc(Start) + i*Trunc(Accum)
5359 // fits within the truncated type (does not overflow) for i = 0 to n-1.
5360 // P2: An Equal predicate that guarantees that
5361 // Start = (Ext ix (Trunc iy (Start) to ix) to iy)
5362 // P3: An Equal predicate that guarantees that
5363 // Accum = (Ext ix (Trunc iy (Accum) to ix) to iy)
5364 //
5365 // As we next prove, the above predicates guarantee that:
5366 // Start + i*Accum = (Ext ix (Trunc iy ( Start + i*Accum ) to ix) to iy)
5367 //
5368 //
5369 // More formally, we want to prove that:
5370 // Expr(i+1) = Start + (i+1) * Accum
5371 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5372 //
5373 // Given that:
5374 // 1) Expr(0) = Start
5375 // 2) Expr(1) = Start + Accum
5376 // = (Ext ix (Trunc iy (Start) to ix) to iy) + Accum :: from P2
5377 // 3) Induction hypothesis (step i):
5378 // Expr(i) = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum
5379 //
5380 // Proof:
5381 // Expr(i+1) =
5382 // = Start + (i+1)*Accum
5383 // = (Start + i*Accum) + Accum
5384 // = Expr(i) + Accum
5385 // = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum + Accum
5386 // :: from step i
5387 //
5388 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy) + Accum + Accum
5389 //
5390 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy)
5391 // + (Ext ix (Trunc iy (Accum) to ix) to iy)
5392 // + Accum :: from P3
5393 //
5394 // = (Ext ix (Trunc iy ((Start + (i-1)*Accum) + Accum) to ix) to iy)
5395 // + Accum :: from P1: Ext(x)+Ext(y)=>Ext(x+y)
5396 //
5397 // = (Ext ix (Trunc iy (Start + i*Accum) to ix) to iy) + Accum
5398 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5399 //
5400 // By induction, the same applies to all iterations 1<=i<n:
5401 //
5402
5403 // Create a truncated addrec for which we will add a no overflow check (P1).
5404 const SCEV *StartVal = getSCEV(StartValueV);
5405 const SCEV *PHISCEV =
5406 getAddRecExpr(getTruncateExpr(StartVal, TruncTy),
5407 getTruncateExpr(Accum, TruncTy), L, SCEV::FlagNone);
5408
5409 // PHISCEV can be either a SCEVConstant or a SCEVAddRecExpr.
5410 // ex: If truncated Accum is 0 and StartVal is a constant, then PHISCEV
5411 // will be constant.
5412 //
5413 // If PHISCEV is a constant, then P1 degenerates into P2 or P3, so we don't
5414 // add P1.
5415 if (const auto *AR = dyn_cast<SCEVAddRecExpr>(PHISCEV)) {
5419 const SCEVPredicate *AddRecPred = getWrapPredicate(AR, AddedFlags);
5420 Predicates.push_back(AddRecPred);
5421 }
5422
5423 // Create the Equal Predicates P2,P3:
5424
5425 // It is possible that the predicates P2 and/or P3 are computable at
5426 // compile time due to StartVal and/or Accum being constants.
5427 // If either one is, then we can check that now and escape if either P2
5428 // or P3 is false.
5429
5430 // Construct the extended SCEV: (Ext ix (Trunc iy (Expr) to ix) to iy)
5431 // for each of StartVal and Accum
5432 auto getExtendedExpr = [&](const SCEV *Expr,
5433 bool CreateSignExtend) -> const SCEV * {
5434 assert(isLoopInvariant(Expr, L) && "Expr is expected to be invariant");
5435 const SCEV *TruncatedExpr = getTruncateExpr(Expr, TruncTy);
5436 const SCEV *ExtendedExpr =
5437 CreateSignExtend ? getSignExtendExpr(TruncatedExpr, Expr->getType())
5438 : getZeroExtendExpr(TruncatedExpr, Expr->getType());
5439 return ExtendedExpr;
5440 };
5441
5442 // Given:
5443 // ExtendedExpr = (Ext ix (Trunc iy (Expr) to ix) to iy
5444 // = getExtendedExpr(Expr)
5445 // Determine whether the predicate P: Expr == ExtendedExpr
5446 // is known to be false at compile time
5447 auto PredIsKnownFalse = [&](const SCEV *Expr,
5448 const SCEV *ExtendedExpr) -> bool {
5449 return Expr != ExtendedExpr &&
5450 isKnownPredicate(ICmpInst::ICMP_NE, Expr, ExtendedExpr);
5451 };
5452
5453 const SCEV *StartExtended = getExtendedExpr(StartVal, Signed);
5454 if (PredIsKnownFalse(StartVal, StartExtended)) {
5455 LLVM_DEBUG(dbgs() << "P2 is compile-time false\n";);
5456 return std::nullopt;
5457 }
5458
5459 // The Step is always Signed (because the overflow checks are either
5460 // NSSW or NUSW)
5461 const SCEV *AccumExtended = getExtendedExpr(Accum, /*CreateSignExtend=*/true);
5462 if (PredIsKnownFalse(Accum, AccumExtended)) {
5463 LLVM_DEBUG(dbgs() << "P3 is compile-time false\n";);
5464 return std::nullopt;
5465 }
5466
5467 auto AppendPredicate = [&](const SCEV *Expr,
5468 const SCEV *ExtendedExpr) -> void {
5469 if (Expr != ExtendedExpr &&
5470 !isKnownPredicate(ICmpInst::ICMP_EQ, Expr, ExtendedExpr)) {
5471 const SCEVPredicate *Pred = getEqualPredicate(Expr, ExtendedExpr);
5472 LLVM_DEBUG(dbgs() << "Added Predicate: " << *Pred);
5473 Predicates.push_back(Pred);
5474 }
5475 };
5476
5477 AppendPredicate(StartVal, StartExtended);
5478 AppendPredicate(Accum, AccumExtended);
5479
5480 // *** Part3: Predicates are ready. Now go ahead and create the new addrec in
5481 // which the casts had been folded away. The caller can rewrite SymbolicPHI
5482 // into NewAR if it will also add the runtime overflow checks specified in
5483 // Predicates.
5484 const SCEV *NewAR = getAddRecExpr(StartVal, Accum, L, SCEV::FlagNone);
5485
5486 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> PredRewrite =
5487 std::make_pair(NewAR, Predicates);
5488 // Remember the result of the analysis for this SCEV at this locayyytion.
5489 PredicatedSCEVRewrites[{SymbolicPHI, L}] = PredRewrite;
5490 return PredRewrite;
5491}
5492
5493std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5495 auto *PN = cast<PHINode>(SymbolicPHI->getValue());
5496 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5497 if (!L)
5498 return std::nullopt;
5499
5500 // Check to see if we already analyzed this PHI.
5501 auto I = PredicatedSCEVRewrites.find({SymbolicPHI, L});
5502 if (I != PredicatedSCEVRewrites.end()) {
5503 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> Rewrite =
5504 I->second;
5505 // Analysis was done before and failed to create an AddRec:
5506 if (Rewrite.first == SymbolicPHI)
5507 return std::nullopt;
5508 // Analysis was done before and succeeded to create an AddRec under
5509 // a predicate:
5510 assert(isa<SCEVAddRecExpr>(Rewrite.first) && "Expected an AddRec");
5511 assert(!(Rewrite.second).empty() && "Expected to find Predicates");
5512 return Rewrite;
5513 }
5514
5515 std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5516 Rewrite = createAddRecFromPHIWithCastsImpl(SymbolicPHI);
5517
5518 // Record in the cache that the analysis failed
5519 if (!Rewrite) {
5521 PredicatedSCEVRewrites[{SymbolicPHI, L}] = {SymbolicPHI, Predicates};
5522 return std::nullopt;
5523 }
5524
5525 return Rewrite;
5526}
5527
5528// FIXME: This utility is currently required because the Rewriter currently
5529// does not rewrite this expression:
5530// {0, +, (sext ix (trunc iy to ix) to iy)}
5531// into {0, +, %step},
5532// even when the following Equal predicate exists:
5533// "%step == (sext ix (trunc iy to ix) to iy)".
5535 const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2,
5536 ArrayRef<const SCEVPredicate *> NoWrapPreds) const {
5537 if (AR1 == AR2)
5538 return true;
5539
5540 SCEVUnionPredicate NoWrapUnionPred(NoWrapPreds, SE);
5541 SCEVUnionPredicate AllPreds = Preds->getUnionWith(&NoWrapUnionPred, SE);
5542 auto areExprsEqual = [&](const SCEV *Expr1, const SCEV *Expr2) -> bool {
5543 if (Expr1 != Expr2 &&
5544 !AllPreds.implies(SE.getEqualPredicate(Expr1, Expr2), SE) &&
5545 !AllPreds.implies(SE.getEqualPredicate(Expr2, Expr1), SE))
5546 return false;
5547 return true;
5548 };
5549
5550 if (!areExprsEqual(AR1->getStart(), AR2->getStart()) ||
5551 !areExprsEqual(AR1->getStepRecurrence(SE), AR2->getStepRecurrence(SE)))
5552 return false;
5553 return true;
5554}
5555
5557 ScalarEvolution &SE) {
5558 SCEVFlags Flags = SCEV::FlagNone;
5559 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
5560 // If the increment has any nowrap flags, then we know the address
5561 // space cannot be wrapped around.
5562 if (NW != GEPNoWrapFlags::none())
5564 // If the GEP is nuw or nusw with non-negative offset, we know that
5565 // no unsigned wrap occurs. We cannot set the nsw flag as only the
5566 // offset is treated as signed, while the base is unsigned.
5567 if (NW.hasNoUnsignedWrap() ||
5568 (NW.hasNoUnsignedSignedWrap() && SE.isKnownNonNegative(Accum)))
5570
5571 return Flags;
5572}
5573
5574/// A helper function for createAddRecFromPHI to handle simple cases.
5575///
5576/// This function tries to find an AddRec expression for the simplest (yet most
5577/// common) cases: PN = PHI(Start, OP(Self, LoopInvariant)).
5578/// If it fails, createAddRecFromPHI will use a more general, but slow,
5579/// technique for finding the AddRec expression.
5580const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
5581 Value *BEValueV,
5582 Value *StartValueV) {
5583 const Loop *L = LI.getLoopFor(PN->getParent());
5584 assert(L && L->getHeader() == PN->getParent());
5585 assert(BEValueV && StartValueV);
5586
5587 const SCEV *Accum = nullptr;
5589 if (auto BO = MatchBinaryOp(BEValueV, DT)) {
5590 if (BO->Opcode != Instruction::Add)
5591 return nullptr;
5592
5593 if (BO->LHS == PN && L->isLoopInvariant(BO->RHS))
5594 Accum = getSCEV(BO->RHS);
5595 else if (BO->RHS == PN && L->isLoopInvariant(BO->LHS))
5596 Accum = getSCEV(BO->LHS);
5597
5598 if (!Accum)
5599 return nullptr;
5600
5601 if (BO->IsNUW)
5602 Flags = setFlags(Flags, SCEV::FlagNUW);
5603 if (BO->IsNSW)
5604 Flags = setFlags(Flags, SCEV::FlagNSW);
5605 } else {
5606 // Handle pointer induction variable: PN = PHI(Start, gep PN,
5607 // LoopInvariant).
5608 auto *GEP = dyn_cast<GEPOperator>(BEValueV);
5609 if (!GEP || GEP->getPointerOperand() != PN || GEP->getNumIndices() != 1)
5610 return nullptr;
5611 Value *Idx = *GEP->idx_begin();
5612 if (!L->isLoopInvariant(Idx))
5613 return nullptr;
5614
5615 Type *IntIdxTy = getEffectiveSCEVType(GEP->getType());
5616 Accum = getMulExpr(getTruncateOrSignExtend(getSCEV(Idx), IntIdxTy),
5617 getSizeOfExpr(IntIdxTy, GEP->getSourceElementType()));
5618 Flags = getNoWrapFlagsForGEP(GEP, Accum, *this);
5619 }
5620
5621 const SCEV *StartVal = getSCEV(StartValueV);
5622 const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags);
5623 insertValueToMap(PN, PHISCEV);
5624
5625 if (auto *AR = dyn_cast<SCEVAddRecExpr>(PHISCEV))
5626 inferNoWrapViaConstantRanges(AR);
5627
5628 // We can add Flags to the post-inc expression only if we
5629 // know that it is *undefined behavior* for BEValueV to
5630 // overflow.
5631 if (auto *BEInst = dyn_cast<Instruction>(BEValueV)) {
5632 assert(isLoopInvariant(Accum, L) &&
5633 "Accum is defined outside L, but is not invariant?");
5634 if (isAddRecNeverPoison(BEInst, L))
5635 (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, Flags);
5636 }
5637
5638 return PHISCEV;
5639}
5640
5641const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
5642 const Loop *L = LI.getLoopFor(PN->getParent());
5643 if (!L || L->getHeader() != PN->getParent())
5644 return nullptr;
5645
5646 // The loop may have multiple entrances or multiple exits; we can analyze
5647 // this phi as an addrec if it has a unique entry value and a unique
5648 // backedge value.
5649 Value *BEValueV = nullptr, *StartValueV = nullptr;
5650 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5651 Value *V = PN->getIncomingValue(i);
5652 if (L->contains(PN->getIncomingBlock(i))) {
5653 if (!BEValueV) {
5654 BEValueV = V;
5655 } else if (BEValueV != V) {
5656 BEValueV = nullptr;
5657 break;
5658 }
5659 } else if (!StartValueV) {
5660 StartValueV = V;
5661 } else if (StartValueV != V) {
5662 StartValueV = nullptr;
5663 break;
5664 }
5665 }
5666 if (!BEValueV || !StartValueV)
5667 return nullptr;
5668
5669 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
5670 "PHI node already processed?");
5671
5672 // First, try to find AddRec expression without creating a fictituos symbolic
5673 // value for PN.
5674 if (auto *S = createSimpleAffineAddRec(PN, BEValueV, StartValueV))
5675 return S;
5676
5677 // Handle PHI node value symbolically.
5678 const SCEV *SymbolicName = getUnknown(PN);
5679 insertValueToMap(PN, SymbolicName);
5680
5681 // Using this symbolic name for the PHI, analyze the value coming around
5682 // the back-edge.
5683 const SCEV *BEValue = getSCEV(BEValueV);
5684
5685 // NOTE: If BEValue is loop invariant, we know that the PHI node just
5686 // has a special value for the first iteration of the loop.
5687
5688 // If the value coming around the backedge is an add with the symbolic
5689 // value we just inserted, then we found a simple induction variable!
5690 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BEValue)) {
5691 // If there is a single occurrence of the symbolic value, replace it
5692 // with a recurrence.
5693 unsigned FoundIndex = Add->getNumOperands();
5694 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5695 if (Add->getOperand(i) == SymbolicName)
5696 if (FoundIndex == e) {
5697 FoundIndex = i;
5698 break;
5699 }
5700
5701 if (FoundIndex != Add->getNumOperands()) {
5702 // Create an add with everything but the specified operand.
5704 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5705 if (i != FoundIndex)
5706 Ops.push_back(SCEVBackedgeConditionFolder::rewrite(Add->getOperand(i),
5707 L, *this));
5708 const SCEV *Accum = getAddExpr(Ops);
5709
5710 // This is not a valid addrec if the step amount is varying each
5711 // loop iteration, but is not itself an addrec in this loop.
5712 if (isLoopInvariant(Accum, L) ||
5713 (isa<SCEVAddRecExpr>(Accum) &&
5714 cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) {
5716
5717 if (auto BO = MatchBinaryOp(BEValueV, DT)) {
5718 if (BO->Opcode == Instruction::Add && BO->LHS == PN) {
5719 if (BO->IsNUW)
5720 Flags = setFlags(Flags, SCEV::FlagNUW);
5721 if (BO->IsNSW)
5722 Flags = setFlags(Flags, SCEV::FlagNSW);
5723 }
5724 } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(BEValueV)) {
5725 if (GEP->getOperand(0) == PN)
5726 Flags = getNoWrapFlagsForGEP(GEP, Accum, *this);
5727
5728 // We cannot transfer nuw and nsw flags from subtraction
5729 // operations -- sub nuw X, Y is not the same as add nuw X, -Y
5730 // for instance.
5731 }
5732
5733 const SCEV *StartVal = getSCEV(StartValueV);
5734 const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags);
5735
5736 // Okay, for the entire analysis of this edge we assumed the PHI
5737 // to be symbolic. We now need to go back and purge all of the
5738 // entries for the scalars that use the symbolic expression.
5739 forgetMemoizedResults({SymbolicName});
5740 insertValueToMap(PN, PHISCEV);
5741
5742 if (auto *AR = dyn_cast<SCEVAddRecExpr>(PHISCEV))
5743 inferNoWrapViaConstantRanges(AR);
5744
5745 // We can add Flags to the post-inc expression only if we
5746 // know that it is *undefined behavior* for BEValueV to
5747 // overflow.
5748 if (auto *BEInst = dyn_cast<Instruction>(BEValueV))
5749 if (isLoopInvariant(Accum, L) && isAddRecNeverPoison(BEInst, L))
5750 (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, Flags);
5751
5752 return PHISCEV;
5753 }
5754 }
5755 } else {
5756 // Otherwise, this could be a loop like this:
5757 // i = 0; for (j = 1; ..; ++j) { .... i = j; }
5758 // In this case, j = {1,+,1} and BEValue is j.
5759 // Because the other in-value of i (0) fits the evolution of BEValue
5760 // i really is an addrec evolution.
5761 //
5762 // We can generalize this saying that i is the shifted value of BEValue
5763 // by one iteration:
5764 // PHI(f(0), f({1,+,1})) --> f({0,+,1})
5765
5766 // Do not allow refinement in rewriting of BEValue.
5767 const SCEV *Shifted = SCEVShiftRewriter::rewrite(BEValue, L, *this);
5768 const SCEV *Start = SCEVInitRewriter::rewrite(Shifted, L, *this, false);
5769 if (Shifted != getCouldNotCompute() && Start != getCouldNotCompute() &&
5770 isGuaranteedNotToCauseUB(Shifted) && ::impliesPoison(Shifted, Start)) {
5771 const SCEV *StartVal = getSCEV(StartValueV);
5772 if (Start == StartVal) {
5773 // Okay, for the entire analysis of this edge we assumed the PHI
5774 // to be symbolic. We now need to go back and purge all of the
5775 // entries for the scalars that use the symbolic expression.
5776 forgetMemoizedResults({SymbolicName});
5777 insertValueToMap(PN, Shifted);
5778 return Shifted;
5779 }
5780 }
5781 }
5782
5783 // Remove the temporary PHI node SCEV that has been inserted while intending
5784 // to create an AddRecExpr for this PHI node. We can not keep this temporary
5785 // as it will prevent later (possibly simpler) SCEV expressions to be added
5786 // to the ValueExprMap.
5787 eraseValueFromMap(PN);
5788
5789 return nullptr;
5790}
5791
5792// Try to match a control flow sequence that branches out at BI and merges back
5793// at Merge into a "C ? LHS : RHS" select pattern. Return true on a successful
5794// match.
5796 Value *&C, Value *&LHS, Value *&RHS) {
5797 C = BI->getCondition();
5798
5799 BasicBlockEdge LeftEdge(BI->getParent(), BI->getSuccessor(0));
5800 BasicBlockEdge RightEdge(BI->getParent(), BI->getSuccessor(1));
5801
5802 Use &LeftUse = Merge->getOperandUse(0);
5803 Use &RightUse = Merge->getOperandUse(1);
5804
5805 if (DT.dominates(LeftEdge, LeftUse) && DT.dominates(RightEdge, RightUse)) {
5806 LHS = LeftUse;
5807 RHS = RightUse;
5808 return true;
5809 }
5810
5811 if (DT.dominates(LeftEdge, RightUse) && DT.dominates(RightEdge, LeftUse)) {
5812 LHS = RightUse;
5813 RHS = LeftUse;
5814 return true;
5815 }
5816
5817 return false;
5818}
5819
5821 Value *&Cond, Value *&LHS,
5822 Value *&RHS) {
5823 auto IsReachable =
5824 [&](BasicBlock *BB) { return DT.isReachableFromEntry(BB); };
5825 if (PN->getNumIncomingValues() == 2 && all_of(PN->blocks(), IsReachable)) {
5826 // Try to match
5827 //
5828 // br %cond, label %left, label %right
5829 // left:
5830 // br label %merge
5831 // right:
5832 // br label %merge
5833 // merge:
5834 // V = phi [ %x, %left ], [ %y, %right ]
5835 //
5836 // as "select %cond, %x, %y"
5837
5838 BasicBlock *IDom = DT[PN->getParent()]->getIDom()->getBlock();
5839 assert(IDom && "At least the entry block should dominate PN");
5840
5841 auto *BI = dyn_cast<CondBrInst>(IDom->getTerminator());
5842 return BI && BrPHIToSelect(DT, BI, PN, Cond, LHS, RHS);
5843 }
5844 return false;
5845}
5846
5847const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
5848 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
5849 if (getOperandsForSelectLikePHI(DT, PN, Cond, LHS, RHS) &&
5852 return createNodeForSelectOrPHI(PN, Cond, LHS, RHS);
5853
5854 return nullptr;
5855}
5856
5858 BinaryOperator *CommonInst = nullptr;
5859 // Check if instructions are identical.
5860 for (Value *Incoming : PN->incoming_values()) {
5861 auto *IncomingInst = dyn_cast<BinaryOperator>(Incoming);
5862 if (!IncomingInst)
5863 return nullptr;
5864 if (CommonInst) {
5865 if (!CommonInst->isIdenticalToWhenDefined(IncomingInst))
5866 return nullptr; // Not identical, give up
5867 } else {
5868 // Remember binary operator
5869 CommonInst = IncomingInst;
5870 }
5871 }
5872 return CommonInst;
5873}
5874
5875/// Returns SCEV for the first operand of a phi if all phi operands have
5876/// identical opcodes and operands
5877/// eg.
5878/// a: %add = %a + %b
5879/// br %c
5880/// b: %add1 = %a + %b
5881/// br %c
5882/// c: %phi = phi [%add, a], [%add1, b]
5883/// scev(%phi) => scev(%add)
5884const SCEV *
5885ScalarEvolution::createNodeForPHIWithIdenticalOperands(PHINode *PN) {
5886 BinaryOperator *CommonInst = getCommonInstForPHI(PN);
5887 if (!CommonInst)
5888 return nullptr;
5889
5890 // Check if SCEV exprs for instructions are identical.
5891 const SCEV *CommonSCEV = getSCEV(CommonInst);
5892 bool SCEVExprsIdentical =
5894 [this, CommonSCEV](Value *V) { return CommonSCEV == getSCEV(V); });
5895 return SCEVExprsIdentical ? CommonSCEV : nullptr;
5896}
5897
5898const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
5899 if (const SCEV *S = createAddRecFromPHI(PN))
5900 return S;
5901
5902 // We do not allow simplifying phi (undef, X) to X here, to avoid reusing the
5903 // phi node for X.
5905 PN, {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
5906 /*UseInstrInfo=*/true, /*CanUseUndef=*/false}))
5907 return getSCEV(V);
5908
5909 if (const SCEV *S = createNodeForPHIWithIdenticalOperands(PN))
5910 return S;
5911
5912 if (const SCEV *S = createNodeFromSelectLikePHI(PN))
5913 return S;
5914
5915 // If it's not a loop phi, we can't handle it yet.
5916 return getUnknown(PN);
5917}
5918
5919bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind,
5920 SCEVTypes RootKind) {
5921 struct FindClosure {
5922 const SCEV *OperandToFind;
5923 const SCEVTypes RootKind; // Must be a sequential min/max expression.
5924 const SCEVTypes NonSequentialRootKind; // Non-seq variant of RootKind.
5925
5926 bool Found = false;
5927
5928 bool canRecurseInto(SCEVTypes Kind) const {
5929 // We can only recurse into the SCEV expression of the same effective type
5930 // as the type of our root SCEV expression, and into zero-extensions.
5931 return RootKind == Kind || NonSequentialRootKind == Kind ||
5932 scZeroExtend == Kind;
5933 };
5934
5935 FindClosure(const SCEV *OperandToFind, SCEVTypes RootKind)
5936 : OperandToFind(OperandToFind), RootKind(RootKind),
5937 NonSequentialRootKind(
5939 RootKind)) {}
5940
5941 bool follow(const SCEV *S) {
5942 Found = S == OperandToFind;
5943
5944 return !isDone() && canRecurseInto(S->getSCEVType());
5945 }
5946
5947 bool isDone() const { return Found; }
5948 };
5949
5950 FindClosure FC(OperandToFind, RootKind);
5951 visitAll(Root, FC);
5952 return FC.Found;
5953}
5954
5955std::optional<const SCEV *>
5956ScalarEvolution::createNodeForSelectOrPHIInstWithICmpInstCond(Type *Ty,
5957 ICmpInst *Cond,
5958 Value *TrueVal,
5959 Value *FalseVal) {
5960 // Try to match some simple smax or umax patterns.
5961 auto *ICI = Cond;
5962
5963 Value *LHS = ICI->getOperand(0);
5964 Value *RHS = ICI->getOperand(1);
5965
5966 switch (ICI->getPredicate()) {
5967 case ICmpInst::ICMP_SLT:
5968 case ICmpInst::ICMP_SLE:
5969 case ICmpInst::ICMP_ULT:
5970 case ICmpInst::ICMP_ULE:
5971 std::swap(LHS, RHS);
5972 [[fallthrough]];
5973 case ICmpInst::ICMP_SGT:
5974 case ICmpInst::ICMP_SGE:
5975 case ICmpInst::ICMP_UGT:
5976 case ICmpInst::ICMP_UGE:
5977 // a > b ? a+x : b+x -> max(a, b)+x
5978 // a > b ? b+x : a+x -> min(a, b)+x
5980 bool Signed = ICI->isSigned();
5981 const SCEV *LA = getSCEV(TrueVal);
5982 const SCEV *RA = getSCEV(FalseVal);
5983 const SCEV *LS = getSCEV(LHS);
5984 const SCEV *RS = getSCEV(RHS);
5985 if (LA->getType()->isPointerTy()) {
5986 // FIXME: Handle cases where LS/RS are pointers not equal to LA/RA.
5987 // Need to make sure we can't produce weird expressions involving
5988 // negated pointers.
5989 if (LA == LS && RA == RS)
5990 return Signed ? getSMaxExpr(LS, RS) : getUMaxExpr(LS, RS);
5991 if (LA == RS && RA == LS)
5992 return Signed ? getSMinExpr(LS, RS) : getUMinExpr(LS, RS);
5993 }
5994 auto CoerceOperand = [&](const SCEV *Op) -> const SCEV * {
5995 if (Op->getType()->isPointerTy()) {
5998 return Op;
5999 }
6000 if (Signed)
6001 Op = getNoopOrSignExtend(Op, Ty);
6002 else
6003 Op = getNoopOrZeroExtend(Op, Ty);
6004 return Op;
6005 };
6006 LS = CoerceOperand(LS);
6007 RS = CoerceOperand(RS);
6009 break;
6010 const SCEV *LDiff = getMinusSCEV(LA, LS);
6011 const SCEV *RDiff = getMinusSCEV(RA, RS);
6012 if (LDiff == RDiff)
6013 return getAddExpr(Signed ? getSMaxExpr(LS, RS) : getUMaxExpr(LS, RS),
6014 LDiff);
6015 LDiff = getMinusSCEV(LA, RS);
6016 RDiff = getMinusSCEV(RA, LS);
6017 if (LDiff == RDiff)
6018 return getAddExpr(Signed ? getSMinExpr(LS, RS) : getUMinExpr(LS, RS),
6019 LDiff);
6020 }
6021 break;
6022 case ICmpInst::ICMP_NE:
6023 // x != 0 ? x+y : C+y -> x == 0 ? C+y : x+y
6024 std::swap(TrueVal, FalseVal);
6025 [[fallthrough]];
6026 case ICmpInst::ICMP_EQ:
6027 // x == 0 ? C+y : x+y -> umax(x, C)+y iff C u<= 1
6030 const SCEV *X = getNoopOrZeroExtend(getSCEV(LHS), Ty);
6031 const SCEV *TrueValExpr = getSCEV(TrueVal); // C+y
6032 const SCEV *FalseValExpr = getSCEV(FalseVal); // x+y
6033 const SCEV *Y = getMinusSCEV(FalseValExpr, X); // y = (x+y)-x
6034 const SCEV *C = getMinusSCEV(TrueValExpr, Y); // C = (C+y)-y
6035 if (isa<SCEVConstant>(C) && cast<SCEVConstant>(C)->getAPInt().ule(1))
6036 return getAddExpr(getUMaxExpr(X, C), Y);
6037 }
6038 // x == 0 ? 0 : umin (..., x, ...) -> umin_seq(x, umin (...))
6039 // x == 0 ? 0 : umin_seq(..., x, ...) -> umin_seq(x, umin_seq(...))
6040 // x == 0 ? 0 : umin (..., umin_seq(..., x, ...), ...)
6041 // -> umin_seq(x, umin (..., umin_seq(...), ...))
6043 isa<ConstantInt>(TrueVal) && cast<ConstantInt>(TrueVal)->isZero()) {
6044 const SCEV *X = getSCEV(LHS);
6045 while (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(X))
6046 X = ZExt->getOperand();
6047 if (getTypeSizeInBits(X->getType()) <= getTypeSizeInBits(Ty)) {
6048 const SCEV *FalseValExpr = getSCEV(FalseVal);
6049 if (SCEVMinMaxExprContains(FalseValExpr, X, scSequentialUMinExpr))
6050 return getUMinExpr(getNoopOrZeroExtend(X, Ty), FalseValExpr,
6051 /*Sequential=*/true);
6052 }
6053 }
6054 break;
6055 default:
6056 break;
6057 }
6058
6059 return std::nullopt;
6060}
6061
6062static std::optional<const SCEV *>
6064 const SCEV *TrueExpr, const SCEV *FalseExpr) {
6065 assert(CondExpr->getType()->isIntegerTy(1) &&
6066 TrueExpr->getType() == FalseExpr->getType() &&
6067 TrueExpr->getType()->isIntegerTy(1) &&
6068 "Unexpected operands of a select.");
6069
6070 // i1 cond ? i1 x : i1 C --> C + (i1 cond ? (i1 x - i1 C) : i1 0)
6071 // --> C + (umin_seq cond, x - C)
6072 //
6073 // i1 cond ? i1 C : i1 x --> C + (i1 cond ? i1 0 : (i1 x - i1 C))
6074 // --> C + (i1 ~cond ? (i1 x - i1 C) : i1 0)
6075 // --> C + (umin_seq ~cond, x - C)
6076
6077 // FIXME: while we can't legally model the case where both of the hands
6078 // are fully variable, we only require that the *difference* is constant.
6079 if (!isa<SCEVConstant>(TrueExpr) && !isa<SCEVConstant>(FalseExpr))
6080 return std::nullopt;
6081
6082 const SCEV *X, *C;
6083 if (isa<SCEVConstant>(TrueExpr)) {
6084 CondExpr = SE->getNotSCEV(CondExpr);
6085 X = FalseExpr;
6086 C = TrueExpr;
6087 } else {
6088 X = TrueExpr;
6089 C = FalseExpr;
6090 }
6091 return SE->getAddExpr(C, SE->getUMinExpr(CondExpr, SE->getMinusSCEV(X, C),
6092 /*Sequential=*/true));
6093}
6094
6095static std::optional<const SCEV *>
6097 Value *FalseVal) {
6098 if (!isa<ConstantInt>(TrueVal) && !isa<ConstantInt>(FalseVal))
6099 return std::nullopt;
6100
6101 const auto *SECond = SE->getSCEV(Cond);
6102 const auto *SETrue = SE->getSCEV(TrueVal);
6103 const auto *SEFalse = SE->getSCEV(FalseVal);
6104 return createNodeForSelectViaUMinSeq(SE, SECond, SETrue, SEFalse);
6105}
6106
6107const SCEV *ScalarEvolution::createNodeForSelectOrPHIViaUMinSeq(
6108 Value *V, Value *Cond, Value *TrueVal, Value *FalseVal) {
6109 assert(Cond->getType()->isIntegerTy(1) && "Select condition is not an i1?");
6110 assert(TrueVal->getType() == FalseVal->getType() &&
6111 V->getType() == TrueVal->getType() &&
6112 "Types of select hands and of the result must match.");
6113
6114 // For now, only deal with i1-typed `select`s.
6115 if (!V->getType()->isIntegerTy(1))
6116 return getUnknown(V);
6117
6118 if (std::optional<const SCEV *> S =
6119 createNodeForSelectViaUMinSeq(this, Cond, TrueVal, FalseVal))
6120 return *S;
6121
6122 return getUnknown(V);
6123}
6124
6125const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Value *V, Value *Cond,
6126 Value *TrueVal,
6127 Value *FalseVal) {
6128 // Handle "constant" branch or select. This can occur for instance when a
6129 // loop pass transforms an inner loop and moves on to process the outer loop.
6130 if (auto *CI = dyn_cast<ConstantInt>(Cond))
6131 return getSCEV(CI->isOne() ? TrueVal : FalseVal);
6132
6133 if (auto *I = dyn_cast<Instruction>(V)) {
6134 if (auto *ICI = dyn_cast<ICmpInst>(Cond)) {
6135 if (std::optional<const SCEV *> S =
6136 createNodeForSelectOrPHIInstWithICmpInstCond(I->getType(), ICI,
6137 TrueVal, FalseVal))
6138 return *S;
6139 }
6140 }
6141
6142 return createNodeForSelectOrPHIViaUMinSeq(V, Cond, TrueVal, FalseVal);
6143}
6144
6145/// Expand GEP instructions into add and multiply operations. This allows them
6146/// to be analyzed by regular SCEV code.
6147const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
6148 assert(GEP->getSourceElementType()->isSized() &&
6149 "GEP source element type must be sized");
6150
6151 SmallVector<SCEVUse, 4> IndexExprs;
6152 for (Value *Index : GEP->indices())
6153 IndexExprs.push_back(getSCEV(Index));
6154 return getGEPExpr(GEP, IndexExprs);
6155}
6156
6157APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S,
6158 const Instruction *CtxI) {
6160 auto GetShiftedByZeros = [BitWidth](uint32_t TrailingZeros) {
6161 return TrailingZeros >= BitWidth
6163 : APInt::getOneBitSet(BitWidth, TrailingZeros);
6164 };
6165 auto GetGCDMultiple = [this, CtxI](const SCEVNAryExpr *N) {
6166 // The result is GCD of all operands results.
6167 APInt Res = getConstantMultiple(N->getOperand(0), CtxI);
6168 for (unsigned I = 1, E = N->getNumOperands(); I < E && Res != 1; ++I)
6170 Res, getConstantMultiple(N->getOperand(I), CtxI));
6171 return Res;
6172 };
6173
6174 switch (S->getSCEVType()) {
6175 case scConstant:
6176 return cast<SCEVConstant>(S)->getAPInt();
6177 case scPtrToAddr:
6178 return getConstantMultiple(cast<SCEVCastExpr>(S)->getOperand());
6179 case scUDivExpr:
6180 case scVScale:
6181 return APInt(BitWidth, 1);
6182 case scTruncate: {
6183 // Only multiples that are a power of 2 will hold after truncation.
6184 const SCEVTruncateExpr *T = cast<SCEVTruncateExpr>(S);
6185 uint32_t TZ = getMinTrailingZeros(T->getOperand(), CtxI);
6186 return GetShiftedByZeros(TZ);
6187 }
6188 case scZeroExtend: {
6189 const SCEVZeroExtendExpr *Z = cast<SCEVZeroExtendExpr>(S);
6190 return getConstantMultiple(Z->getOperand(), CtxI).zext(BitWidth);
6191 }
6192 case scSignExtend: {
6193 // Only multiples that are a power of 2 will hold after sext.
6194 const SCEVSignExtendExpr *E = cast<SCEVSignExtendExpr>(S);
6195 uint32_t TZ = getMinTrailingZeros(E->getOperand(), CtxI);
6196 return GetShiftedByZeros(TZ);
6197 }
6198 case scMulExpr: {
6199 const SCEVMulExpr *M = cast<SCEVMulExpr>(S);
6200 if (M->hasNoUnsignedWrap()) {
6201 // The result is the product of all operand results.
6202 APInt Res = getConstantMultiple(M->getOperand(0), CtxI);
6203 for (const SCEV *Operand : M->operands().drop_front())
6204 Res = Res * getConstantMultiple(Operand, CtxI);
6205 return Res;
6206 }
6207
6208 // If there are no wrap guarentees, find the trailing zeros, which is the
6209 // sum of trailing zeros for all its operands.
6210 uint32_t TZ = 0;
6211 for (const SCEV *Operand : M->operands())
6212 TZ += getMinTrailingZeros(Operand, CtxI);
6213 return GetShiftedByZeros(TZ);
6214 }
6215 case scAddExpr:
6216 case scAddRecExpr: {
6217 const SCEVNAryExpr *N = cast<SCEVNAryExpr>(S);
6218 if (N->hasNoUnsignedWrap())
6219 return GetGCDMultiple(N);
6220 // Find the trailing bits, which is the minimum of its operands.
6221 uint32_t TZ = getMinTrailingZeros(N->getOperand(0), CtxI);
6222 for (const SCEV *Operand : N->operands().drop_front())
6223 TZ = std::min(TZ, getMinTrailingZeros(Operand, CtxI));
6224 return GetShiftedByZeros(TZ);
6225 }
6226 case scUMaxExpr:
6227 case scSMaxExpr:
6228 case scUMinExpr:
6229 case scSMinExpr:
6231 return GetGCDMultiple(cast<SCEVNAryExpr>(S));
6232 case scUnknown: {
6233 // Ask ValueTracking for known bits. SCEVUnknown only become available at
6234 // the point their underlying IR instruction has been defined. If CtxI was
6235 // not provided, use:
6236 // * the first instruction in the entry block if it is an argument
6237 // * the instruction itself otherwise.
6238 const SCEVUnknown *U = cast<SCEVUnknown>(S);
6239 if (!CtxI) {
6240 if (isa<Argument>(U->getValue()))
6241 CtxI = &*F.getEntryBlock().begin();
6242 else if (auto *I = dyn_cast<Instruction>(U->getValue()))
6243 CtxI = I;
6244 }
6245 unsigned Known =
6246 computeKnownBits(U->getValue(),
6247 SimplifyQuery(getDataLayout(), &DT, &AC, CtxI)
6248 .allowEphemerals(true))
6249 .countMinTrailingZeros();
6250 return GetShiftedByZeros(Known);
6251 }
6252 case scCouldNotCompute:
6253 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6254 }
6255 llvm_unreachable("Unknown SCEV kind!");
6256}
6257
6259 const Instruction *CtxI) {
6260 // Skip looking up and updating the cache if there is a context instruction,
6261 // as the result will only be valid in the specified context.
6262 if (CtxI)
6263 return getConstantMultipleImpl(S, CtxI);
6264
6265 auto I = ConstantMultipleCache.find(S);
6266 if (I != ConstantMultipleCache.end())
6267 return I->second;
6268
6269 APInt Result = getConstantMultipleImpl(S, CtxI);
6270 auto InsertPair = ConstantMultipleCache.insert({S, Result});
6271 assert(InsertPair.second && "Should insert a new key");
6272 return InsertPair.first->second;
6273}
6274
6276 APInt Multiple = getConstantMultiple(S);
6277 return Multiple == 0 ? APInt(Multiple.getBitWidth(), 1) : Multiple;
6278}
6279
6281 const Instruction *CtxI) {
6282 return std::min(getConstantMultiple(S, CtxI).countTrailingZeros(),
6283 (unsigned)getTypeSizeInBits(S->getType()));
6284}
6285
6286/// Helper method to assign a range to V from metadata present in the IR.
6287static std::optional<ConstantRange> GetRangeFromMetadata(Value *V) {
6289 if (MDNode *MD = I->getMetadata(LLVMContext::MD_range))
6290 return getConstantRangeFromMetadata(*MD);
6291 if (const auto *CB = dyn_cast<CallBase>(V))
6292 if (std::optional<ConstantRange> Range = CB->getRange())
6293 return Range;
6294 }
6295 if (auto *A = dyn_cast<Argument>(V))
6296 if (std::optional<ConstantRange> Range = A->getRange())
6297 return Range;
6298
6299 return std::nullopt;
6300}
6301
6303 SCEVFlags NWFlags = Flags & SCEV::FlagsNoWrapMask;
6304 if (AddRec->getNoWrapFlags(NWFlags) != NWFlags) {
6305 AddRec->setNoWrapFlags(NWFlags);
6306 UnsignedRanges.erase(AddRec);
6307 SignedRanges.erase(AddRec);
6308 ConstantMultipleCache.erase(AddRec);
6309 }
6310}
6311
6312ConstantRange ScalarEvolution::
6313getRangeForUnknownRecurrence(const SCEVUnknown *U) {
6314 const DataLayout &DL = getDataLayout();
6315
6316 unsigned BitWidth = getTypeSizeInBits(U->getType());
6317 const ConstantRange FullSet(BitWidth, /*isFullSet=*/true);
6318
6319 // Match a simple recurrence of the form: <start, ShiftOp, Step>, and then
6320 // use information about the trip count to improve our available range. Note
6321 // that the trip count independent cases are already handled by known bits.
6322 // WARNING: The definition of recurrence used here is subtly different than
6323 // the one used by AddRec (and thus most of this file). Step is allowed to
6324 // be arbitrarily loop varying here, where AddRec allows only loop invariant
6325 // and other addrecs in the same loop (for non-affine addrecs). The code
6326 // below intentionally handles the case where step is not loop invariant.
6327 auto *P = dyn_cast<PHINode>(U->getValue());
6328 if (!P)
6329 return FullSet;
6330
6331 // Make sure that no Phi input comes from an unreachable block. Otherwise,
6332 // even the values that are not available in these blocks may come from them,
6333 // and this leads to false-positive recurrence test.
6334 for (auto *Pred : predecessors(P->getParent()))
6335 if (!DT.isReachableFromEntry(Pred))
6336 return FullSet;
6337
6338 BinaryOperator *BO;
6339 Value *Start, *Step;
6340 if (!matchSimpleRecurrence(P, BO, Start, Step))
6341 return FullSet;
6342
6343 // If we found a recurrence in reachable code, we must be in a loop. Note
6344 // that BO might be in some subloop of L, and that's completely okay.
6345 auto *L = LI.getLoopFor(P->getParent());
6346 assert(L && L->getHeader() == P->getParent());
6347 if (!L->contains(BO->getParent()))
6348 // NOTE: This bailout should be an assert instead. However, asserting
6349 // the condition here exposes a case where LoopFusion is querying SCEV
6350 // with malformed loop information during the midst of the transform.
6351 // There doesn't appear to be an obvious fix, so for the moment bailout
6352 // until the caller issue can be fixed. PR49566 tracks the bug.
6353 return FullSet;
6354
6355 // TODO: Extend to other opcodes such as mul, and div
6356 switch (BO->getOpcode()) {
6357 default:
6358 return FullSet;
6359 case Instruction::AShr:
6360 case Instruction::LShr:
6361 case Instruction::Shl:
6362 break;
6363 };
6364
6365 if (BO->getOperand(0) != P)
6366 // TODO: Handle the power function forms some day.
6367 return FullSet;
6368
6369 unsigned TC = getSmallConstantMaxTripCount(L);
6370 if (!TC || TC >= BitWidth)
6371 return FullSet;
6372
6373 auto KnownStart = computeKnownBits(Start, DL, &AC, nullptr, &DT);
6374 auto KnownStep = computeKnownBits(Step, DL, &AC, nullptr, &DT);
6375 assert(KnownStart.getBitWidth() == BitWidth &&
6376 KnownStep.getBitWidth() == BitWidth);
6377
6378 // Compute total shift amount, being careful of overflow and bitwidths.
6379 auto MaxShiftAmt = KnownStep.getMaxValue();
6380 APInt TCAP(BitWidth, TC-1);
6381 bool Overflow = false;
6382 auto TotalShift = MaxShiftAmt.umul_ov(TCAP, Overflow);
6383 if (Overflow)
6384 return FullSet;
6385
6386 switch (BO->getOpcode()) {
6387 default:
6388 llvm_unreachable("filtered out above");
6389 case Instruction::AShr: {
6390 // For each ashr, three cases:
6391 // shift = 0 => unchanged value
6392 // saturation => 0 or -1
6393 // other => a value closer to zero (of the same sign)
6394 // Thus, the end value is closer to zero than the start.
6395 auto KnownEnd = KnownBits::ashr(KnownStart,
6396 KnownBits::makeConstant(TotalShift));
6397 if (KnownStart.isNonNegative())
6398 // Analogous to lshr (simply not yet canonicalized)
6399 return ConstantRange::getNonEmpty(KnownEnd.getMinValue(),
6400 KnownStart.getMaxValue() + 1);
6401 if (KnownStart.isNegative())
6402 // End >=u Start && End <=s Start
6403 return ConstantRange::getNonEmpty(KnownStart.getMinValue(),
6404 KnownEnd.getMaxValue() + 1);
6405 break;
6406 }
6407 case Instruction::LShr: {
6408 // For each lshr, three cases:
6409 // shift = 0 => unchanged value
6410 // saturation => 0
6411 // other => a smaller positive number
6412 // Thus, the low end of the unsigned range is the last value produced.
6413 auto KnownEnd = KnownBits::lshr(KnownStart,
6414 KnownBits::makeConstant(TotalShift));
6415 return ConstantRange::getNonEmpty(KnownEnd.getMinValue(),
6416 KnownStart.getMaxValue() + 1);
6417 }
6418 case Instruction::Shl: {
6419 // Iff no bits are shifted out, value increases on every shift.
6420 auto KnownEnd = KnownBits::shl(KnownStart,
6421 KnownBits::makeConstant(TotalShift));
6422 if (TotalShift.ult(KnownStart.countMinLeadingZeros()))
6423 return ConstantRange(KnownStart.getMinValue(),
6424 KnownEnd.getMaxValue() + 1);
6425 break;
6426 }
6427 };
6428 return FullSet;
6429}
6430
6431// The goal of this function is to check if recursively visiting the operands
6432// of this PHI might lead to an infinite loop. If we do see such a loop,
6433// there's no good way to break it, so we avoid analyzing such cases.
6434//
6435// getRangeRef previously used a visited set to avoid infinite loops, but this
6436// caused other issues: the result was dependent on the order of getRangeRef
6437// calls, and the interaction with createSCEVIter could cause a stack overflow
6438// in some cases (see issue #148253).
6439//
6440// FIXME: The way this is implemented is overly conservative; this checks
6441// for a few obviously safe patterns, but anything that doesn't lead to
6442// recursion is fine.
6444 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
6446 return true;
6447
6448 if (all_of(PHI->operands(),
6449 [&](Value *Operand) { return DT.dominates(Operand, PHI); }))
6450 return true;
6451
6452 return false;
6453}
6454
6455const ConstantRange &
6456ScalarEvolution::getRangeRefIter(const SCEV *S,
6457 ScalarEvolution::RangeSignHint SignHint) {
6458 DenseMap<const SCEV *, ConstantRange> &Cache =
6459 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6460 : SignedRanges;
6461 SmallVector<SCEVUse> WorkList;
6462 SmallPtrSet<const SCEV *, 8> Seen;
6463
6464 // Add Expr to the worklist, if Expr is either an N-ary expression or a
6465 // SCEVUnknown PHI node.
6466 auto AddToWorklist = [&WorkList, &Seen, &Cache](const SCEV *Expr) {
6467 if (!Seen.insert(Expr).second)
6468 return;
6469 if (Cache.contains(Expr))
6470 return;
6471 switch (Expr->getSCEVType()) {
6472 case scUnknown:
6474 break;
6475 [[fallthrough]];
6476 case scConstant:
6477 case scVScale:
6478 case scTruncate:
6479 case scZeroExtend:
6480 case scSignExtend:
6481 case scPtrToAddr:
6482 case scAddExpr:
6483 case scMulExpr:
6484 case scUDivExpr:
6485 case scAddRecExpr:
6486 case scUMaxExpr:
6487 case scSMaxExpr:
6488 case scUMinExpr:
6489 case scSMinExpr:
6491 WorkList.push_back(Expr);
6492 break;
6493 case scCouldNotCompute:
6494 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6495 }
6496 };
6497 AddToWorklist(S);
6498
6499 // Build worklist by queuing operands of N-ary expressions and phi nodes.
6500 for (unsigned I = 0; I != WorkList.size(); ++I) {
6501 const SCEV *P = WorkList[I];
6502 auto *UnknownS = dyn_cast<SCEVUnknown>(P);
6503 // If it is not a `SCEVUnknown`, just recurse into operands.
6504 if (!UnknownS) {
6505 for (const SCEV *Op : P->operands())
6506 AddToWorklist(Op);
6507 continue;
6508 }
6509 // `SCEVUnknown`'s require special treatment.
6510 if (PHINode *P = dyn_cast<PHINode>(UnknownS->getValue())) {
6511 if (!RangeRefPHIAllowedOperands(DT, P))
6512 continue;
6513 for (auto &Op : reverse(P->operands()))
6514 AddToWorklist(getSCEV(Op));
6515 }
6516 }
6517
6518 if (!WorkList.empty()) {
6519 // Use getRangeRef to compute ranges for items in the worklist in reverse
6520 // order. This will force ranges for earlier operands to be computed before
6521 // their users in most cases.
6522 for (const SCEV *P : reverse(drop_begin(WorkList))) {
6523 getRangeRef(P, SignHint);
6524 }
6525 }
6526
6527 return getRangeRef(S, SignHint, 0);
6528}
6529
6530const APInt *ScalarEvolution::getConstantAPIntOrNull(const SCEV *S) {
6531 if (const auto *C = dyn_cast<SCEVConstant>(S))
6532 return &C->getAPInt();
6533 return nullptr;
6534}
6535
6536/// Determine the range for a particular SCEV. If SignHint is
6537/// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges
6538/// with a "cleaner" unsigned (resp. signed) representation.
6539const ConstantRange &ScalarEvolution::getRangeRef(
6540 const SCEV *S, ScalarEvolution::RangeSignHint SignHint, unsigned Depth) {
6541 DenseMap<const SCEV *, ConstantRange> &Cache =
6542 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6543 : SignedRanges;
6545 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? ConstantRange::Unsigned
6547
6548 // See if we've computed this range already.
6549 auto I = Cache.find(S);
6550 if (I != Cache.end())
6551 return I->second;
6552
6553 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
6554 return setRange(C, SignHint, ConstantRange(C->getAPInt()));
6555
6556 // Switch to iteratively computing the range for S, if it is part of a deeply
6557 // nested expression.
6558 if (Depth > Opts.scev_range_iter_threshold)
6559 return getRangeRefIter(S, SignHint);
6560
6561 unsigned BitWidth = getTypeSizeInBits(S->getType());
6562 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
6563 using OBO = OverflowingBinaryOperator;
6564
6565 // If the value has known zeros, the maximum value will have those known zeros
6566 // as well.
6567 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
6568 APInt Multiple = getNonZeroConstantMultiple(S);
6569 APInt Remainder = APInt::getMaxValue(BitWidth).urem(Multiple);
6570 if (!Remainder.isZero())
6571 ConservativeResult =
6572 ConstantRange(APInt::getMinValue(BitWidth),
6573 APInt::getMaxValue(BitWidth) - Remainder + 1);
6574 }
6575 else {
6576 uint32_t TZ = getMinTrailingZeros(S);
6577 if (TZ != 0) {
6578 ConservativeResult = ConstantRange(
6580 APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1);
6581 }
6582 }
6583
6584 switch (S->getSCEVType()) {
6585 case scConstant:
6586 llvm_unreachable("Already handled above.");
6587 case scVScale:
6588 return setRange(S, SignHint, getVScaleRange(&F, BitWidth));
6589 case scTruncate: {
6590 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(S);
6591 ConstantRange X = getRangeRef(Trunc->getOperand(), SignHint, Depth + 1);
6592 return setRange(
6593 Trunc, SignHint,
6594 ConservativeResult.intersectWith(X.truncate(BitWidth), RangeType));
6595 }
6596 case scZeroExtend: {
6597 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(S);
6598 ConstantRange X = getRangeRef(ZExt->getOperand(), SignHint, Depth + 1);
6599 return setRange(
6600 ZExt, SignHint,
6601 ConservativeResult.intersectWith(X.zeroExtend(BitWidth), RangeType));
6602 }
6603 case scSignExtend: {
6604 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(S);
6605 ConstantRange X = getRangeRef(SExt->getOperand(), SignHint, Depth + 1);
6606 return setRange(
6607 SExt, SignHint,
6608 ConservativeResult.intersectWith(X.signExtend(BitWidth), RangeType));
6609 }
6610 case scPtrToAddr: {
6611 const SCEVCastExpr *Cast = cast<SCEVCastExpr>(S);
6612 ConstantRange X = getRangeRef(Cast->getOperand(), SignHint, Depth + 1);
6613 return setRange(Cast, SignHint, X);
6614 }
6615 case scAddExpr: {
6616 const SCEVAddExpr *Add = cast<SCEVAddExpr>(S);
6617 // Check if this is a URem pattern: A - (A / B) * B, which is always < B.
6618 const SCEV *URemLHS = nullptr, *URemRHS = nullptr;
6619 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED &&
6620 match(S, m_scev_URem(m_SCEV(URemLHS), m_SCEV(URemRHS), *this))) {
6621 ConstantRange LHSRange = getRangeRef(URemLHS, SignHint, Depth + 1);
6622 ConstantRange RHSRange = getRangeRef(URemRHS, SignHint, Depth + 1);
6623 ConservativeResult =
6624 ConservativeResult.intersectWith(LHSRange.urem(RHSRange), RangeType);
6625 }
6626 ConstantRange X = getRangeRef(Add->getOperand(0), SignHint, Depth + 1);
6627 unsigned WrapType = OBO::AnyWrap;
6628 if (Add->hasNoSignedWrap())
6629 WrapType |= OBO::NoSignedWrap;
6630 if (Add->hasNoUnsignedWrap())
6631 WrapType |= OBO::NoUnsignedWrap;
6632 for (const SCEV *Op : drop_begin(Add->operands()))
6633 X = X.addWithNoWrap(getRangeRef(Op, SignHint, Depth + 1), WrapType,
6634 RangeType);
6635 return setRange(Add, SignHint,
6636 ConservativeResult.intersectWith(X, RangeType));
6637 }
6638 case scMulExpr: {
6639 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(S);
6640 ConstantRange X = getRangeRef(Mul->getOperand(0), SignHint, Depth + 1);
6641 for (const SCEV *Op : drop_begin(Mul->operands()))
6642 X = X.multiply(getRangeRef(Op, SignHint, Depth + 1));
6643 return setRange(Mul, SignHint,
6644 ConservativeResult.intersectWith(X, RangeType));
6645 }
6646 case scUDivExpr: {
6647 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S);
6648 ConstantRange X = getRangeRef(UDiv->getLHS(), SignHint, Depth + 1);
6649 ConstantRange Y = getRangeRef(UDiv->getRHS(), SignHint, Depth + 1);
6650 return setRange(UDiv, SignHint,
6651 ConservativeResult.intersectWith(X.udiv(Y), RangeType));
6652 }
6653 case scAddRecExpr: {
6654 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(S);
6655 // If there's no unsigned wrap, the value will never be less than its
6656 // initial value.
6657 if (AddRec->hasNoUnsignedWrap()) {
6658 APInt UnsignedMinValue = getUnsignedRangeMin(AddRec->getStart());
6659 if (!UnsignedMinValue.isZero())
6660 ConservativeResult = ConservativeResult.intersectWith(
6661 ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), RangeType);
6662 }
6663
6664 // If there's no signed wrap, and all the operands except initial value have
6665 // the same sign or zero, the value won't ever be:
6666 // 1: smaller than initial value if operands are non negative,
6667 // 2: bigger than initial value if operands are non positive.
6668 // For both cases, value can not cross signed min/max boundary.
6669 if (AddRec->hasNoSignedWrap()) {
6670 bool AllNonNeg = true;
6671 bool AllNonPos = true;
6672 for (unsigned i = 1, e = AddRec->getNumOperands(); i != e; ++i) {
6673 if (!isKnownNonNegative(AddRec->getOperand(i)))
6674 AllNonNeg = false;
6675 if (!isKnownNonPositive(AddRec->getOperand(i)))
6676 AllNonPos = false;
6677 }
6678 if (AllNonNeg)
6679 ConservativeResult = ConservativeResult.intersectWith(
6682 RangeType);
6683 else if (AllNonPos)
6684 ConservativeResult = ConservativeResult.intersectWith(
6686 getSignedRangeMax(AddRec->getStart()) +
6687 1),
6688 RangeType);
6689 }
6690
6691 // TODO: non-affine addrec
6692 if (AddRec->isAffine()) {
6693 const SCEV *MaxBEScev =
6695 if (!isa<SCEVCouldNotCompute>(MaxBEScev)) {
6696 APInt MaxBECount = cast<SCEVConstant>(MaxBEScev)->getAPInt();
6697
6698 // Adjust MaxBECount to the same bitwidth as AddRec. We can truncate if
6699 // MaxBECount's active bits are all <= AddRec's bit width.
6700 if (MaxBECount.getBitWidth() > BitWidth &&
6701 MaxBECount.getActiveBits() <= BitWidth)
6702 MaxBECount = MaxBECount.trunc(BitWidth);
6703 else if (MaxBECount.getBitWidth() < BitWidth)
6704 MaxBECount = MaxBECount.zext(BitWidth);
6705
6706 if (MaxBECount.getBitWidth() == BitWidth) {
6707 auto [RangeFromAffine, Flags] = getRangeForAffineAR(
6708 AddRec->getStart(), AddRec->getStepRecurrence(*this), MaxBECount);
6709 ConservativeResult =
6710 ConservativeResult.intersectWith(RangeFromAffine, RangeType);
6711 const_cast<SCEVAddRecExpr *>(AddRec)->setNoWrapFlags(Flags);
6712
6713 auto RangeFromFactoring = getRangeViaFactoring(
6714 AddRec->getStart(), AddRec->getStepRecurrence(*this), MaxBECount);
6715 ConservativeResult =
6716 ConservativeResult.intersectWith(RangeFromFactoring, RangeType);
6717 }
6718 }
6719
6720 // Now try symbolic BE count and more powerful methods.
6721 if (Opts.scalar_evolution_use_expensive_range_sharpening) {
6722 const SCEV *SymbolicMaxBECount =
6724 if (!isa<SCEVCouldNotCompute>(SymbolicMaxBECount) &&
6725 getTypeSizeInBits(MaxBEScev->getType()) <= BitWidth &&
6726 AddRec->hasNoSelfWrap()) {
6727 auto RangeFromAffineNew = getRangeForAffineNoSelfWrappingAR(
6728 AddRec, SymbolicMaxBECount, BitWidth, SignHint);
6729 ConservativeResult =
6730 ConservativeResult.intersectWith(RangeFromAffineNew, RangeType);
6731 }
6732 }
6733 }
6734
6735 return setRange(AddRec, SignHint, std::move(ConservativeResult));
6736 }
6737 case scUMaxExpr:
6738 case scSMaxExpr:
6739 case scUMinExpr:
6740 case scSMinExpr:
6741 case scSequentialUMinExpr: {
6743 switch (S->getSCEVType()) {
6744 case scUMaxExpr:
6745 ID = Intrinsic::umax;
6746 break;
6747 case scSMaxExpr:
6748 ID = Intrinsic::smax;
6749 break;
6750 case scUMinExpr:
6752 ID = Intrinsic::umin;
6753 break;
6754 case scSMinExpr:
6755 ID = Intrinsic::smin;
6756 break;
6757 default:
6758 llvm_unreachable("Unknown SCEVMinMaxExpr/SCEVSequentialMinMaxExpr.");
6759 }
6760
6761 const auto *NAry = cast<SCEVNAryExpr>(S);
6762 ConstantRange X = getRangeRef(NAry->getOperand(0), SignHint, Depth + 1);
6763 for (unsigned i = 1, e = NAry->getNumOperands(); i != e; ++i)
6764 X = X.intrinsic(
6765 ID, {X, getRangeRef(NAry->getOperand(i), SignHint, Depth + 1)});
6766 return setRange(S, SignHint,
6767 ConservativeResult.intersectWith(X, RangeType));
6768 }
6769 case scUnknown: {
6770 const SCEVUnknown *U = cast<SCEVUnknown>(S);
6771 Value *V = U->getValue();
6772
6773 // Check if the IR explicitly contains !range metadata.
6774 std::optional<ConstantRange> MDRange = GetRangeFromMetadata(V);
6775 if (MDRange)
6776 ConservativeResult =
6777 ConservativeResult.intersectWith(*MDRange, RangeType);
6778
6779 // Use facts about recurrences in the underlying IR. Note that add
6780 // recurrences are AddRecExprs and thus don't hit this path. This
6781 // primarily handles shift recurrences.
6782 auto CR = getRangeForUnknownRecurrence(U);
6783 ConservativeResult = ConservativeResult.intersectWith(CR);
6784
6785 // See if ValueTracking can give us a useful range.
6786 const DataLayout &DL = getDataLayout();
6787 KnownBits Known = computeKnownBits(V, DL, &AC, nullptr, &DT);
6788 if (Known.getBitWidth() != BitWidth)
6789 Known = Known.zextOrTrunc(BitWidth);
6790
6791 // ValueTracking may be able to compute a tighter result for the number of
6792 // sign bits than for the value of those sign bits.
6793 unsigned NS = ComputeNumSignBits(V, DL, &AC, nullptr, &DT);
6794 if (U->getType()->isPointerTy()) {
6795 // NS counts the sign bits of the whole pointer; drop those above the
6796 // index bits.
6797 unsigned PtrIdxDiff =
6798 DL.getPointerTypeSizeInBits(U->getType()) - BitWidth;
6799 NS = NS > PtrIdxDiff ? NS - PtrIdxDiff : 1;
6800 }
6801
6802 if (NS > 1) {
6803 // If we know any of the sign bits, we know all of the sign bits.
6804 if (!Known.Zero.getHiBits(NS).isZero())
6805 Known.Zero.setHighBits(NS);
6806 if (!Known.One.getHiBits(NS).isZero())
6807 Known.One.setHighBits(NS);
6808 }
6809
6810 if (Known.getMinValue() != Known.getMaxValue() + 1)
6811 ConservativeResult = ConservativeResult.intersectWith(
6812 ConstantRange(Known.getMinValue(), Known.getMaxValue() + 1),
6813 RangeType);
6814 if (NS > 1)
6815 ConservativeResult = ConservativeResult.intersectWith(
6816 ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1),
6817 APInt::getSignedMaxValue(BitWidth).ashr(NS - 1) + 1),
6818 RangeType);
6819
6820 if (U->getType()->isPointerTy() && SignHint == HINT_RANGE_UNSIGNED) {
6821 // Strengthen the range if the underlying IR value is a
6822 // global/alloca/heap allocation using the size of the object.
6823 bool CanBeNull;
6824 uint64_t DerefBytes = V->getPointerDereferenceableBytes(
6825 DL, CanBeNull, /*CanBeFreed=*/nullptr);
6826 if (DerefBytes > 1 && isUIntN(BitWidth, DerefBytes)) {
6827 // The highest address the object can start is DerefBytes bytes before
6828 // the end (unsigned max value). If this value is not a multiple of the
6829 // alignment, the last possible start value is the next lowest multiple
6830 // of the alignment. Note: The computations below cannot overflow,
6831 // because if they would there's no possible start address for the
6832 // object.
6833 APInt MaxVal =
6834 APInt::getMaxValue(BitWidth) - APInt(BitWidth, DerefBytes);
6835 uint64_t Align = U->getValue()->getPointerAlignment(DL).value();
6836 uint64_t Rem = MaxVal.urem(Align);
6837 MaxVal -= APInt(BitWidth, Rem);
6838 APInt MinVal = APInt::getZero(BitWidth);
6839 if (llvm::isKnownNonZero(V, DL))
6840 MinVal = Align;
6841 ConservativeResult = ConservativeResult.intersectWith(
6842 ConstantRange::getNonEmpty(MinVal, MaxVal + 1), RangeType);
6843 }
6844 }
6845
6846 // A range of Phi is a subset of union of all ranges of its input.
6847 if (PHINode *Phi = dyn_cast<PHINode>(V)) {
6848 // SCEVExpander sometimes creates SCEVUnknowns that are secretly
6849 // AddRecs; return the range for the corresponding AddRec.
6850 if (auto *AR = dyn_cast<SCEVAddRecExpr>(getSCEV(V)))
6851 return getRangeRef(AR, SignHint, Depth + 1);
6852
6853 // Make sure that we do not run over cycled Phis.
6854 if (RangeRefPHIAllowedOperands(DT, Phi)) {
6855 ConstantRange RangeFromOps(BitWidth, /*isFullSet=*/false);
6856
6857 for (const auto &Op : Phi->operands()) {
6858 auto OpRange = getRangeRef(getSCEV(Op), SignHint, Depth + 1);
6859 RangeFromOps = RangeFromOps.unionWith(OpRange, RangeType);
6860 // No point to continue if we already have a full set.
6861 if (RangeFromOps.isFullSet())
6862 break;
6863 }
6864 ConservativeResult =
6865 ConservativeResult.intersectWith(RangeFromOps, RangeType);
6866 }
6867 }
6868
6869 const APInt *TrueC, *FalseC;
6870 if (match(V, m_Select(m_Value(), m_APInt(TrueC), m_APInt(FalseC))))
6871 ConservativeResult = ConservativeResult.intersectWith(
6872 ConstantRange(*TrueC).unionWith(*FalseC, RangeType), RangeType);
6873
6874 // vscale can't be equal to zero
6875 if (const auto *II = dyn_cast<IntrinsicInst>(V))
6876 if (II->getIntrinsicID() == Intrinsic::vscale) {
6877 ConstantRange Disallowed = APInt::getZero(BitWidth);
6878 ConservativeResult = ConservativeResult.difference(Disallowed);
6879 }
6880
6881 return setRange(U, SignHint, std::move(ConservativeResult));
6882 }
6883 case scCouldNotCompute:
6884 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6885 }
6886
6887 return setRange(S, SignHint, std::move(ConservativeResult));
6888}
6889
6890// Given a StartRange, Step and MaxBECount for an expression compute a range of
6891// values that the expression can take. Initially, the expression has a value
6892// from StartRange and then is changed by Step up to MaxBECount times. Signed
6893// argument defines if we treat Step as signed or unsigned. The second return
6894// value indicates that no wrapping occurred.
6895static std::pair<ConstantRange, bool>
6897 const APInt &MaxBECount, bool Signed) {
6898 unsigned BitWidth = Step.getBitWidth();
6899 assert(BitWidth == StartRange.getBitWidth() &&
6900 BitWidth == MaxBECount.getBitWidth() && "mismatched bit widths");
6901 // If either Step or MaxBECount is 0, then the expression won't change, and we
6902 // just need to return the initial range.
6903 if (Step == 0 || MaxBECount == 0)
6904 return {StartRange, true};
6905
6906 // If we don't know anything about the initial value (i.e. StartRange is
6907 // FullRange), then we don't know anything about the final range either.
6908 // Return FullRange.
6909 if (StartRange.isFullSet())
6910 return {ConstantRange::getFull(BitWidth), false};
6911
6912 // If Step is signed and negative, then we use its absolute value, but we also
6913 // note that we're moving in the opposite direction.
6914 bool Descending = Signed && Step.isNegative();
6915
6916 if (Signed)
6917 // This is correct even for INT_SMIN. Let's look at i8 to illustrate this:
6918 // abs(INT_SMIN) = abs(-128) = abs(0x80) = -0x80 = 0x80 = 128.
6919 // This equations hold true due to the well-defined wrap-around behavior of
6920 // APInt.
6921 Step = Step.abs();
6922
6923 // Check if Offset is more than full span of BitWidth. If it is, the
6924 // expression is guaranteed to overflow.
6925 if (APInt::getMaxValue(StartRange.getBitWidth()).udiv(Step).ult(MaxBECount))
6926 return {ConstantRange::getFull(BitWidth), false};
6927
6928 // Offset is by how much the expression can change. Checks above guarantee no
6929 // overflow here.
6930 APInt Offset = Step * MaxBECount;
6931
6932 // Minimum value of the final range will match the minimal value of StartRange
6933 // if the expression is increasing and will be decreased by Offset otherwise.
6934 // Maximum value of the final range will match the maximal value of StartRange
6935 // if the expression is decreasing and will be increased by Offset otherwise.
6936 APInt StartLower = StartRange.getLower();
6937 APInt StartUpper = StartRange.getUpper() - 1;
6938 bool Overflow;
6939 APInt MovedBoundary;
6940 if (Signed) {
6941 // This does not use sadd_ov, as we want to check overflow for a signed
6942 // start with an unsigned offset.
6943 if (Descending) {
6944 MovedBoundary = StartLower - std::move(Offset);
6945 Overflow = MovedBoundary.sgt(StartLower) || StartRange.isSignWrappedSet();
6946 } else {
6947 MovedBoundary = StartUpper + std::move(Offset);
6948 Overflow = MovedBoundary.slt(StartUpper) || StartRange.isSignWrappedSet();
6949 }
6950 } else {
6951 MovedBoundary = StartUpper.uadd_ov(std::move(Offset), Overflow);
6952 Overflow |= StartRange.isWrappedSet();
6953 }
6954
6955 // It's possible that the new minimum/maximum value will fall into the initial
6956 // range (due to wrap around). This means that the expression can take any
6957 // value in this bitwidth, and we have to return full range.
6958 if (StartRange.contains(MovedBoundary))
6959 return {ConstantRange::getFull(BitWidth), false};
6960
6961 APInt NewLower =
6962 Descending ? std::move(MovedBoundary) : std::move(StartLower);
6963 APInt NewUpper =
6964 Descending ? std::move(StartUpper) : std::move(MovedBoundary);
6965 NewUpper += 1;
6966
6967 // No overflow detected, return [StartLower, StartUpper + Offset + 1) range.
6968 return {ConstantRange::getNonEmpty(std::move(NewLower), std::move(NewUpper)),
6969 !Overflow};
6970}
6971
6972std::pair<ConstantRange, SCEVFlags>
6973ScalarEvolution::getRangeForAffineAR(const SCEV *Start, const SCEV *Step,
6974 const APInt &MaxBECount) {
6975 assert(getTypeSizeInBits(Start->getType()) ==
6976 getTypeSizeInBits(Step->getType()) &&
6977 getTypeSizeInBits(Start->getType()) == MaxBECount.getBitWidth() &&
6978 "mismatched bit widths");
6979
6980 // First, consider step signed.
6981 ConstantRange StartSRange = getSignedRange(Start);
6982 ConstantRange StepSRange = getSignedRange(Step);
6983
6984 // If Step can be both positive and negative, we need to find ranges for the
6985 // maximum absolute step values in both directions and union them.
6986 auto [SR, NSW] = getRangeForAffineARHelper(
6987 StepSRange.getSignedMin(), StartSRange, MaxBECount, /*Signed=*/true);
6988 // A singleton step has identical signed bounds, so the second computation
6989 // would produce the same range and no-wrap result.
6990 if (!StepSRange.isSingleElement()) {
6991 auto [SRMax, NSWMax] = getRangeForAffineARHelper(
6992 StepSRange.getSignedMax(), StartSRange, MaxBECount, /*Signed=*/true);
6993 SR = SR.unionWith(SRMax);
6994 NSW &= NSWMax;
6995 }
6996
6997 // Next, consider step unsigned.
6998 auto [UR, NUW] = getRangeForAffineARHelper(
6999 getUnsignedRangeMax(Step), getUnsignedRange(Start), MaxBECount,
7000 /*Signed=*/false);
7001
7003 if (NUW)
7005 if (NSW)
7007
7008 // Finally, intersect signed and unsigned ranges.
7009 return {SR.intersectWith(UR, ConstantRange::Smallest), Flags};
7010}
7011
7012ConstantRange ScalarEvolution::getRangeForAffineNoSelfWrappingAR(
7013 const SCEVAddRecExpr *AddRec, const SCEV *MaxBECount, unsigned BitWidth,
7014 ScalarEvolution::RangeSignHint SignHint) {
7015 assert(AddRec->isAffine() && "Non-affine AddRecs are not suppored!\n");
7016 assert(AddRec->hasNoSelfWrap() &&
7017 "This only works for non-self-wrapping AddRecs!");
7018 const bool IsSigned = SignHint == HINT_RANGE_SIGNED;
7019 const SCEV *Step = AddRec->getStepRecurrence(*this);
7020 // Only deal with constant step to save compile time.
7021 if (!isa<SCEVConstant>(Step))
7022 return ConstantRange::getFull(BitWidth);
7023 // Let's make sure that we can prove that we do not self-wrap during
7024 // MaxBECount iterations. We need this because MaxBECount is a maximum
7025 // iteration count estimate, and we might infer nw from some exit for which we
7026 // do not know max exit count (or any other side reasoning).
7027 // TODO: Turn into assert at some point.
7028 if (getTypeSizeInBits(MaxBECount->getType()) >
7029 getTypeSizeInBits(AddRec->getType()))
7030 return ConstantRange::getFull(BitWidth);
7031 MaxBECount = getNoopOrZeroExtend(MaxBECount, AddRec->getType());
7032 const SCEV *RangeWidth = getMinusOne(AddRec->getType());
7033 const SCEV *StepAbs = getUMinExpr(Step, getNegativeSCEV(Step));
7034 const SCEV *MaxItersWithoutWrap = getUDivExpr(RangeWidth, StepAbs);
7035 if (!isKnownPredicateViaConstantRanges(ICmpInst::ICMP_ULE, MaxBECount,
7036 MaxItersWithoutWrap))
7037 return ConstantRange::getFull(BitWidth);
7038
7039 ICmpInst::Predicate LEPred =
7041 ICmpInst::Predicate GEPred =
7043 const SCEV *End = AddRec->evaluateAtIteration(MaxBECount, *this);
7044
7045 // We know that there is no self-wrap. Let's take Start and End values and
7046 // look at all intermediate values V1, V2, ..., Vn that IndVar takes during
7047 // the iteration. They either lie inside the range [Min(Start, End),
7048 // Max(Start, End)] or outside it:
7049 //
7050 // Case 1: RangeMin ... Start V1 ... VN End ... RangeMax;
7051 // Case 2: RangeMin Vk ... V1 Start ... End Vn ... Vk + 1 RangeMax;
7052 //
7053 // No self wrap flag guarantees that the intermediate values cannot be BOTH
7054 // outside and inside the range [Min(Start, End), Max(Start, End)]. Using that
7055 // knowledge, let's try to prove that we are dealing with Case 1. It is so if
7056 // Start <= End and step is positive, or Start >= End and step is negative.
7057 const SCEV *Start = applyLoopGuards(AddRec->getStart(), AddRec->getLoop());
7058 ConstantRange StartRange = getRangeRef(Start, SignHint);
7059 ConstantRange EndRange = getRangeRef(End, SignHint);
7060 ConstantRange RangeBetween = StartRange.unionWith(EndRange);
7061 // If they already cover full iteration space, we will know nothing useful
7062 // even if we prove what we want to prove.
7063 if (RangeBetween.isFullSet())
7064 return RangeBetween;
7065 // Only deal with ranges that do not wrap (i.e. RangeMin < RangeMax).
7066 bool IsWrappedSet = IsSigned ? RangeBetween.isSignWrappedSet()
7067 : RangeBetween.isWrappedSet();
7068 if (IsWrappedSet)
7069 return ConstantRange::getFull(BitWidth);
7070
7071 if (isKnownPositive(Step) &&
7072 isKnownPredicateViaConstantRanges(LEPred, Start, End))
7073 return RangeBetween;
7074 if (isKnownNegative(Step) &&
7075 isKnownPredicateViaConstantRanges(GEPred, Start, End))
7076 return RangeBetween;
7077 return ConstantRange::getFull(BitWidth);
7078}
7079
7080ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start,
7081 const SCEV *Step,
7082 const APInt &MaxBECount) {
7083 // RangeOf({C?A:B,+,C?P:Q}) == RangeOf(C?{A,+,P}:{B,+,Q})
7084 // == RangeOf({A,+,P}) union RangeOf({B,+,Q})
7085
7086 unsigned BitWidth = MaxBECount.getBitWidth();
7087 assert(getTypeSizeInBits(Start->getType()) == BitWidth &&
7088 getTypeSizeInBits(Step->getType()) == BitWidth &&
7089 "mismatched bit widths");
7090
7091 struct SelectPattern {
7092 Value *Condition = nullptr;
7093 APInt TrueValue;
7094 APInt FalseValue;
7095
7096 explicit SelectPattern(ScalarEvolution &SE, unsigned BitWidth,
7097 const SCEV *S) {
7098 std::optional<unsigned> CastOp;
7099 APInt Offset(BitWidth, 0);
7100
7102 "Should be!");
7103
7104 // Peel off a constant offset. In the future we could consider being
7105 // smarter here and handle {Start+Step,+,Step} too.
7106 const APInt *Off;
7107 if (match(S, m_scev_Add(m_scev_APInt(Off), m_SCEV(S))))
7108 Offset = *Off;
7109
7110 // Peel off a cast operation
7111 if (auto *SCast = dyn_cast<SCEVIntegralCastExpr>(S)) {
7112 CastOp = SCast->getSCEVType();
7113 S = SCast->getOperand();
7114 }
7115
7116 using namespace llvm::PatternMatch;
7117
7118 auto *SU = dyn_cast<SCEVUnknown>(S);
7119 const APInt *TrueVal, *FalseVal;
7120 if (!SU ||
7121 !match(SU->getValue(), m_Select(m_Value(Condition), m_APInt(TrueVal),
7122 m_APInt(FalseVal)))) {
7123 Condition = nullptr;
7124 return;
7125 }
7126
7127 TrueValue = *TrueVal;
7128 FalseValue = *FalseVal;
7129
7130 // Re-apply the cast we peeled off earlier
7131 if (CastOp)
7132 switch (*CastOp) {
7133 default:
7134 llvm_unreachable("Unknown SCEV cast type!");
7135
7136 case scTruncate:
7137 TrueValue = TrueValue.trunc(BitWidth);
7138 FalseValue = FalseValue.trunc(BitWidth);
7139 break;
7140 case scZeroExtend:
7141 TrueValue = TrueValue.zext(BitWidth);
7142 FalseValue = FalseValue.zext(BitWidth);
7143 break;
7144 case scSignExtend:
7145 TrueValue = TrueValue.sext(BitWidth);
7146 FalseValue = FalseValue.sext(BitWidth);
7147 break;
7148 }
7149
7150 // Re-apply the constant offset we peeled off earlier
7151 TrueValue += Offset;
7152 FalseValue += Offset;
7153 }
7154
7155 bool isRecognized() { return Condition != nullptr; }
7156 };
7157
7158 SelectPattern StartPattern(*this, BitWidth, Start);
7159 if (!StartPattern.isRecognized())
7160 return ConstantRange::getFull(BitWidth);
7161
7162 SelectPattern StepPattern(*this, BitWidth, Step);
7163 if (!StepPattern.isRecognized())
7164 return ConstantRange::getFull(BitWidth);
7165
7166 if (StartPattern.Condition != StepPattern.Condition) {
7167 // We don't handle this case today; but we could, by considering four
7168 // possibilities below instead of two. I'm not sure if there are cases where
7169 // that will help over what getRange already does, though.
7170 return ConstantRange::getFull(BitWidth);
7171 }
7172
7173 // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to
7174 // construct arbitrary general SCEV expressions here. This function is called
7175 // from deep in the call stack, and calling getSCEV (on a sext instruction,
7176 // say) can end up caching a suboptimal value.
7177
7178 // FIXME: without the explicit `this` receiver below, MSVC errors out with
7179 // C2352 and C2512 (otherwise it isn't needed).
7180
7181 const SCEV *TrueStart = this->getConstant(StartPattern.TrueValue);
7182 const SCEV *TrueStep = this->getConstant(StepPattern.TrueValue);
7183 const SCEV *FalseStart = this->getConstant(StartPattern.FalseValue);
7184 const SCEV *FalseStep = this->getConstant(StepPattern.FalseValue);
7185
7186 ConstantRange TrueRange =
7187 this->getRangeForAffineAR(TrueStart, TrueStep, MaxBECount).first;
7188 ConstantRange FalseRange =
7189 this->getRangeForAffineAR(FalseStart, FalseStep, MaxBECount).first;
7190
7191 return TrueRange.unionWith(FalseRange);
7192}
7193
7194SCEVFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) {
7195 if (isa<ConstantExpr>(V))
7196 return SCEV::FlagNone;
7197 const BinaryOperator *BinOp = cast<BinaryOperator>(V);
7198
7199 // Return early if there are no flags to propagate to the SCEV.
7201 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(BinOp);
7202 PDI && PDI->isDisjoint()) {
7204 } else {
7205 if (BinOp->hasNoUnsignedWrap())
7207 if (BinOp->hasNoSignedWrap())
7209 }
7210 if (Flags == SCEV::FlagNone)
7211 return SCEV::FlagNone;
7212
7213 return isSCEVExprNeverPoison(BinOp) ? Flags : SCEV::FlagNone;
7214}
7215
7216const Instruction *
7217ScalarEvolution::getNonTrivialDefiningScopeBound(const SCEV *S) {
7218 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(S))
7219 return &*AddRec->getLoop()->getHeader()->begin();
7220 if (auto *U = dyn_cast<SCEVUnknown>(S))
7221 if (auto *I = dyn_cast<Instruction>(U->getValue()))
7222 return I;
7223 return nullptr;
7224}
7225
7226const Instruction *ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops,
7227 bool &Precise) {
7228 Precise = true;
7229 // Do a bounded search of the def relation of the requested SCEVs.
7230 SmallPtrSet<const SCEV *, 16> Visited;
7231 SmallVector<SCEVUse> Worklist;
7232 auto pushOp = [&](const SCEV *S) {
7233 if (!Visited.insert(S).second)
7234 return;
7235 // Threshold of 30 here is arbitrary.
7236 if (Visited.size() > 30) {
7237 Precise = false;
7238 return;
7239 }
7240 Worklist.push_back(S);
7241 };
7242
7243 for (SCEVUse S : Ops)
7244 pushOp(S);
7245
7246 const Instruction *Bound = nullptr;
7247 while (!Worklist.empty()) {
7248 SCEVUse S = Worklist.pop_back_val();
7249 if (auto *DefI = getNonTrivialDefiningScopeBound(S)) {
7250 if (!Bound || DT.dominates(Bound, DefI))
7251 Bound = DefI;
7252 } else {
7253 for (SCEVUse Op : S->operands())
7254 pushOp(Op);
7255 }
7256 }
7257 return Bound ? Bound : &*F.getEntryBlock().begin();
7258}
7259
7260const Instruction *
7261ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops) {
7262 bool Discard;
7263 return getDefiningScopeBound(Ops, Discard);
7264}
7265
7266bool ScalarEvolution::isGuaranteedToTransferExecutionTo(const Instruction *A,
7267 const Instruction *B) {
7268 if (A->getParent() == B->getParent() &&
7270 B->getIterator()))
7271 return true;
7272
7273 auto *BLoop = LI.getLoopFor(B->getParent());
7274 if (BLoop && BLoop->getHeader() == B->getParent() &&
7275 BLoop->getLoopPreheader() == A->getParent() &&
7277 A->getParent()->end()) &&
7278 isGuaranteedToTransferExecutionToSuccessor(B->getParent()->begin(),
7279 B->getIterator()))
7280 return true;
7281 return false;
7282}
7283
7285 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ true);
7286 visitAll(Op, PC);
7287 return PC.MaybePoison.empty();
7288}
7289
7291 return !SCEVExprContains(Op, [this](const SCEV *S) {
7292 const SCEV *Op1;
7293 bool M = match(S, m_scev_UDiv(m_SCEV(), m_SCEV(Op1)));
7294 // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
7295 // is a non-zero constant, we have to assume the UDiv may be UB.
7296 return M && (!isKnownNonZero(Op1) || !isGuaranteedNotToBePoison(Op1));
7297 });
7298}
7299
7300bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) {
7301 // Only proceed if we can prove that I does not yield poison.
7303 return false;
7304
7305 // At this point we know that if I is executed, then it does not wrap
7306 // according to at least one of NSW or NUW. If I is not executed, then we do
7307 // not know if the calculation that I represents would wrap. Multiple
7308 // instructions can map to the same SCEV. If we apply NSW or NUW from I to
7309 // the SCEV, we must guarantee no wrapping for that SCEV also when it is
7310 // derived from other instructions that map to the same SCEV. We cannot make
7311 // that guarantee for cases where I is not executed. So we need to find a
7312 // upper bound on the defining scope for the SCEV, and prove that I is
7313 // executed every time we enter that scope. When the bounding scope is a
7314 // loop (the common case), this is equivalent to proving I executes on every
7315 // iteration of that loop.
7316 SmallVector<SCEVUse> SCEVOps;
7317 for (const Use &Op : I->operands()) {
7318 // I could be an extractvalue from a call to an overflow intrinsic.
7319 // TODO: We can do better here in some cases.
7320 if (isSCEVable(Op->getType()))
7321 SCEVOps.push_back(getSCEV(Op));
7322 }
7323 auto *DefI = getDefiningScopeBound(SCEVOps);
7324 return isGuaranteedToTransferExecutionTo(DefI, I);
7325}
7326
7327bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) {
7328 // If we know that \c I can never be poison period, then that's enough.
7329 if (isSCEVExprNeverPoison(I))
7330 return true;
7331
7332 // If the loop only has one exit, then we know that, if the loop is entered,
7333 // any instruction dominating that exit will be executed. If any such
7334 // instruction would result in UB, the addrec cannot be poison.
7335 //
7336 // This is basically the same reasoning as in isSCEVExprNeverPoison(), but
7337 // also handles uses outside the loop header (they just need to dominate the
7338 // single exit).
7339
7340 auto *ExitingBB = L->getExitingBlock();
7341 if (!ExitingBB || !loopHasNoAbnormalExits(L))
7342 return false;
7343
7344 SmallPtrSet<const Value *, 16> KnownPoison;
7346
7347 // We start by assuming \c I, the post-inc add recurrence, is poison. Only
7348 // things that are known to be poison under that assumption go on the
7349 // Worklist.
7350 KnownPoison.insert(I);
7351 Worklist.push_back(I);
7352
7353 while (!Worklist.empty()) {
7354 const Instruction *Poison = Worklist.pop_back_val();
7355
7356 for (const Use &U : Poison->uses()) {
7357 const Instruction *PoisonUser = cast<Instruction>(U.getUser());
7358 if (mustTriggerUB(PoisonUser, KnownPoison) &&
7359 DT.dominates(PoisonUser->getParent(), ExitingBB))
7360 return true;
7361
7362 if (propagatesPoison(U) && L->contains(PoisonUser))
7363 if (KnownPoison.insert(PoisonUser).second)
7364 Worklist.push_back(PoisonUser);
7365 }
7366 }
7367
7368 return false;
7369}
7370
7371ScalarEvolution::LoopProperties
7372ScalarEvolution::getLoopProperties(const Loop *L) {
7373 using LoopProperties = ScalarEvolution::LoopProperties;
7374
7375 auto Itr = LoopPropertiesCache.find(L);
7376 if (Itr == LoopPropertiesCache.end()) {
7377 auto HasSideEffects = [](Instruction *I) {
7378 if (auto *SI = dyn_cast<StoreInst>(I))
7379 return !SI->isSimple();
7380
7381 if (I->mayThrow())
7382 return true;
7383
7384 // Non-volatile memset / memcpy do not count as side-effect for forward
7385 // progress.
7386 if (auto *MI = dyn_cast<MemIntrinsic>(I))
7387 if (!MI->isVolatile())
7388 return false;
7389
7390 // Droppable intrinsics only model a memory write; they are not progress.
7391 if (I->isDroppable())
7392 return false;
7393
7394 return I->mayWriteToMemory();
7395 };
7396
7397 LoopProperties LP = {/* HasNoAbnormalExits */ true,
7398 /*HasNoSideEffects*/ true};
7399
7400 for (auto *BB : L->getBlocks())
7401 for (auto &I : *BB) {
7403 LP.HasNoAbnormalExits = false;
7404 if (HasSideEffects(&I))
7405 LP.HasNoSideEffects = false;
7406 if (!LP.HasNoAbnormalExits && !LP.HasNoSideEffects)
7407 break; // We're already as pessimistic as we can get.
7408 }
7409
7410 auto InsertPair = LoopPropertiesCache.insert({L, LP});
7411 assert(InsertPair.second && "We just checked!");
7412 Itr = InsertPair.first;
7413 }
7414
7415 return Itr->second;
7416}
7417
7419 // A mustprogress loop without side effects must be finite.
7420 // TODO: The check used here is very conservative. It's only *specific*
7421 // side effects which are well defined in infinite loops.
7422 return isFinite(L) || (isMustProgress(L) && loopHasNoSideEffects(L));
7423}
7424
7425const SCEV *ScalarEvolution::createSCEVIter(Value *V) {
7426 // Worklist item with a Value and a bool indicating whether all operands have
7427 // been visited already.
7430
7431 Stack.emplace_back(V, false);
7432 while (!Stack.empty()) {
7433 auto E = Stack.back();
7434 Value *CurV = E.getPointer();
7435
7436 if (getExistingSCEV(CurV)) {
7437 Stack.pop_back();
7438 continue;
7439 }
7440
7442 const SCEV *CreatedSCEV = nullptr;
7443 // If all operands have been visited already, create the SCEV.
7444 if (E.getInt()) {
7445 CreatedSCEV = createSCEV(CurV);
7446 } else {
7447 // Otherwise get the operands we need to create SCEV's for before creating
7448 // the SCEV for CurV. If the SCEV for CurV can be constructed trivially,
7449 // just use it.
7450 CreatedSCEV = getOperandsToCreate(CurV, Ops);
7451 }
7452
7453 if (CreatedSCEV) {
7454 insertValueToMap(CurV, CreatedSCEV);
7455 Stack.pop_back();
7456 } else {
7457 Stack.back().setInt(true);
7458 // Queue its operands which need to be constructed.
7459 for (Value *Op : Ops)
7460 Stack.emplace_back(Op, false);
7461 }
7462 }
7463
7464 return getExistingSCEV(V);
7465}
7466
7467const SCEV *
7468ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
7469 if (!isSCEVable(V->getType()))
7470 return getUnknown(V);
7471
7472 if (Instruction *I = dyn_cast<Instruction>(V)) {
7473 // Don't attempt to analyze instructions in blocks that aren't
7474 // reachable. Such instructions don't matter, and they aren't required
7475 // to obey basic rules for definitions dominating uses which this
7476 // analysis depends on.
7477 if (!DT.isReachableFromEntry(I->getParent()))
7478 return getUnknown(PoisonValue::get(V->getType()));
7479 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(V))
7480 return getConstant(CI);
7481 else if (isa<GlobalAlias>(V))
7482 return getUnknown(V);
7483 else if (!isa<ConstantExpr>(V))
7484 return getUnknown(V);
7485
7487 if (auto BO = MatchBinaryOp(U, DT)) {
7488 bool IsConstArg = isa<ConstantInt>(BO->RHS);
7489 switch (BO->Opcode) {
7490 case Instruction::Add:
7491 case Instruction::Mul: {
7492 // For additions and multiplications, traverse add/mul chains for which we
7493 // can potentially create a single SCEV, to reduce the number of
7494 // get{Add,Mul}Expr calls.
7495 do {
7496 if (BO->Op) {
7497 if (BO->Op != V && getExistingSCEV(BO->Op)) {
7498 Ops.push_back(BO->Op);
7499 break;
7500 }
7501 }
7502 Ops.push_back(BO->RHS);
7503 auto NewBO = MatchBinaryOp(BO->LHS, DT);
7504 if (!NewBO ||
7505 (BO->Opcode == Instruction::Add &&
7506 (NewBO->Opcode != Instruction::Add &&
7507 NewBO->Opcode != Instruction::Sub)) ||
7508 (BO->Opcode == Instruction::Mul &&
7509 NewBO->Opcode != Instruction::Mul)) {
7510 Ops.push_back(BO->LHS);
7511 break;
7512 }
7513 // CreateSCEV calls getNoWrapFlagsFromUB, which under certain conditions
7514 // requires a SCEV for the LHS.
7515 if (BO->Op && (BO->IsNSW || BO->IsNUW)) {
7516 auto *I = dyn_cast<Instruction>(BO->Op);
7517 if (I && programUndefinedIfPoison(I)) {
7518 Ops.push_back(BO->LHS);
7519 break;
7520 }
7521 }
7522 BO = NewBO;
7523 } while (true);
7524 return nullptr;
7525 }
7526 case Instruction::Sub:
7527 case Instruction::UDiv:
7528 case Instruction::URem:
7529 break;
7530 case Instruction::AShr:
7531 case Instruction::Shl:
7532 case Instruction::Xor:
7533 if (!IsConstArg)
7534 return nullptr;
7535 break;
7536 case Instruction::And:
7537 case Instruction::Or:
7538 if (!IsConstArg && !BO->LHS->getType()->isIntegerTy(1))
7539 return nullptr;
7540 break;
7541 case Instruction::LShr:
7542 return getUnknown(V);
7543 default:
7544 llvm_unreachable("Unhandled binop");
7545 break;
7546 }
7547
7548 Ops.push_back(BO->LHS);
7549 Ops.push_back(BO->RHS);
7550 return nullptr;
7551 }
7552
7553 switch (U->getOpcode()) {
7554 case Instruction::Trunc:
7555 case Instruction::ZExt:
7556 case Instruction::SExt:
7557 case Instruction::PtrToAddr:
7558 case Instruction::PtrToInt:
7559 Ops.push_back(U->getOperand(0));
7560 return nullptr;
7561
7562 case Instruction::BitCast:
7563 if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType())) {
7564 Ops.push_back(U->getOperand(0));
7565 return nullptr;
7566 }
7567 return getUnknown(V);
7568
7569 case Instruction::SDiv:
7570 case Instruction::SRem:
7571 Ops.push_back(U->getOperand(0));
7572 Ops.push_back(U->getOperand(1));
7573 return nullptr;
7574
7575 case Instruction::GetElementPtr:
7576 assert(cast<GEPOperator>(U)->getSourceElementType()->isSized() &&
7577 "GEP source element type must be sized");
7578 llvm::append_range(Ops, U->operands());
7579 return nullptr;
7580
7581 case Instruction::IntToPtr:
7582 return getUnknown(V);
7583
7584 case Instruction::PHI:
7585 // getNodeForPHI has four ways to turn a PHI into a SCEV; retrieve the
7586 // relevant nodes for each of them.
7587 //
7588 // The first is just to call simplifyInstruction, and get something back
7589 // that isn't a PHI.
7591 cast<PHINode>(U),
7592 {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
7593 /*UseInstrInfo=*/true, /*CanUseUndef=*/false})) {
7594 assert(V);
7595 Ops.push_back(V);
7596 return nullptr;
7597 }
7598 // The second is createNodeForPHIWithIdenticalOperands: this looks for
7599 // operands which all perform the same operation, but haven't been
7600 // CSE'ed for whatever reason.
7601 if (BinaryOperator *BO = getCommonInstForPHI(cast<PHINode>(U))) {
7602 assert(BO);
7603 Ops.push_back(BO);
7604 return nullptr;
7605 }
7606 // The third is createNodeFromSelectLikePHI; this takes a PHI which
7607 // is equivalent to a select, and analyzes it like a select.
7608 {
7609 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
7611 assert(Cond);
7612 assert(LHS);
7613 assert(RHS);
7614 if (auto *CondICmp = dyn_cast<ICmpInst>(Cond)) {
7615 Ops.push_back(CondICmp->getOperand(0));
7616 Ops.push_back(CondICmp->getOperand(1));
7617 }
7618 Ops.push_back(Cond);
7619 Ops.push_back(LHS);
7620 Ops.push_back(RHS);
7621 return nullptr;
7622 }
7623 }
7624 // The fourth way is createAddRecFromPHI. It's complicated to handle here,
7625 // so just construct it recursively.
7626 //
7627 // In addition to getNodeForPHI, also construct nodes which might be needed
7628 // by getRangeRef.
7630 for (Value *V : cast<PHINode>(U)->operands())
7631 Ops.push_back(V);
7632 return nullptr;
7633 }
7634 return nullptr;
7635
7636 case Instruction::Select: {
7637 // Check if U is a select that can be simplified to a SCEVUnknown.
7638 auto CanSimplifyToUnknown = [this, U]() {
7639 if (U->getType()->isIntegerTy(1) || isa<ConstantInt>(U->getOperand(0)))
7640 return false;
7641
7642 auto *ICI = dyn_cast<ICmpInst>(U->getOperand(0));
7643 if (!ICI)
7644 return false;
7645 Value *LHS = ICI->getOperand(0);
7646 Value *RHS = ICI->getOperand(1);
7647 if (ICI->getPredicate() == CmpInst::ICMP_EQ ||
7648 ICI->getPredicate() == CmpInst::ICMP_NE) {
7650 return true;
7651 } else if (getTypeSizeInBits(LHS->getType()) >
7652 getTypeSizeInBits(U->getType()))
7653 return true;
7654 return false;
7655 };
7656 if (CanSimplifyToUnknown())
7657 return getUnknown(U);
7658
7659 llvm::append_range(Ops, U->operands());
7660 return nullptr;
7661 break;
7662 }
7663 case Instruction::Call:
7664 case Instruction::Invoke:
7665 if (Value *RV = cast<CallBase>(U)->getReturnedArgOperand()) {
7666 Ops.push_back(RV);
7667 return nullptr;
7668 }
7669
7670 if (auto *II = dyn_cast<IntrinsicInst>(U)) {
7671 switch (II->getIntrinsicID()) {
7672 case Intrinsic::abs:
7673 Ops.push_back(II->getArgOperand(0));
7674 return nullptr;
7675 case Intrinsic::umax:
7676 case Intrinsic::umin:
7677 case Intrinsic::smax:
7678 case Intrinsic::smin:
7679 case Intrinsic::usub_sat:
7680 case Intrinsic::uadd_sat:
7681 Ops.push_back(II->getArgOperand(0));
7682 Ops.push_back(II->getArgOperand(1));
7683 return nullptr;
7684 case Intrinsic::start_loop_iterations:
7685 case Intrinsic::annotation:
7686 case Intrinsic::ptr_annotation:
7687 Ops.push_back(II->getArgOperand(0));
7688 return nullptr;
7689 default:
7690 break;
7691 }
7692 }
7693 break;
7694 }
7695
7696 return nullptr;
7697}
7698
7699const SCEV *ScalarEvolution::createSCEV(Value *V) {
7700 if (!isSCEVable(V->getType()))
7701 return getUnknown(V);
7702
7703 if (Instruction *I = dyn_cast<Instruction>(V)) {
7704 // Don't attempt to analyze instructions in blocks that aren't
7705 // reachable. Such instructions don't matter, and they aren't required
7706 // to obey basic rules for definitions dominating uses which this
7707 // analysis depends on.
7708 if (!DT.isReachableFromEntry(I->getParent()))
7709 return getUnknown(PoisonValue::get(V->getType()));
7710 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(V))
7711 return getConstant(CI);
7712 else if (isa<GlobalAlias>(V))
7713 return getUnknown(V);
7714 else if (!isa<ConstantExpr>(V))
7715 return getUnknown(V);
7716
7717 const SCEV *LHS;
7718 const SCEV *RHS;
7719
7721 if (auto BO = MatchBinaryOp(U, DT)) {
7722 switch (BO->Opcode) {
7723 case Instruction::Add: {
7724 // The simple thing to do would be to just call getSCEV on both operands
7725 // and call getAddExpr with the result. However if we're looking at a
7726 // bunch of things all added together, this can be quite inefficient,
7727 // because it leads to N-1 getAddExpr calls for N ultimate operands.
7728 // Instead, gather up all the operands and make a single getAddExpr call.
7729 // LLVM IR canonical form means we need only traverse the left operands.
7731 do {
7732 if (BO->Op) {
7733 if (auto *OpSCEV = getExistingSCEV(BO->Op)) {
7734 AddOps.push_back(OpSCEV);
7735 break;
7736 }
7737
7738 // If a NUW or NSW flag can be applied to the SCEV for this
7739 // addition, then compute the SCEV for this addition by itself
7740 // with a separate call to getAddExpr. We need to do that
7741 // instead of pushing the operands of the addition onto AddOps,
7742 // since the flags are only known to apply to this particular
7743 // addition - they may not apply to other additions that can be
7744 // formed with operands from AddOps.
7745 const SCEV *RHS = getSCEV(BO->RHS);
7746 SCEVFlags Flags = getNoWrapFlagsFromUB(BO->Op);
7747 if (Flags != SCEV::FlagNone) {
7748 const SCEV *LHS = getSCEV(BO->LHS);
7749 if (BO->Opcode == Instruction::Sub)
7750 AddOps.push_back(getMinusSCEV(LHS, RHS, Flags));
7751 else
7752 AddOps.push_back(getAddExpr(LHS, RHS, Flags));
7753 break;
7754 }
7755 }
7756
7757 if (BO->Opcode == Instruction::Sub)
7758 AddOps.push_back(getNegativeSCEV(getSCEV(BO->RHS)));
7759 else
7760 AddOps.push_back(getSCEV(BO->RHS));
7761
7762 auto NewBO = MatchBinaryOp(BO->LHS, DT);
7763 if (!NewBO || (NewBO->Opcode != Instruction::Add &&
7764 NewBO->Opcode != Instruction::Sub)) {
7765 AddOps.push_back(getSCEV(BO->LHS));
7766 break;
7767 }
7768 BO = NewBO;
7769 } while (true);
7770
7771 return getAddExpr(AddOps);
7772 }
7773
7774 case Instruction::Mul: {
7776 do {
7777 if (BO->Op) {
7778 if (auto *OpSCEV = getExistingSCEV(BO->Op)) {
7779 MulOps.push_back(OpSCEV);
7780 break;
7781 }
7782
7783 SCEVFlags Flags = getNoWrapFlagsFromUB(BO->Op);
7784 if (Flags != SCEV::FlagNone) {
7785 LHS = getSCEV(BO->LHS);
7786 RHS = getSCEV(BO->RHS);
7787 MulOps.push_back(getMulExpr(LHS, RHS, Flags));
7788 break;
7789 }
7790 }
7791
7792 MulOps.push_back(getSCEV(BO->RHS));
7793 auto NewBO = MatchBinaryOp(BO->LHS, DT);
7794 if (!NewBO || NewBO->Opcode != Instruction::Mul) {
7795 MulOps.push_back(getSCEV(BO->LHS));
7796 break;
7797 }
7798 BO = NewBO;
7799 } while (true);
7800
7801 return getMulExpr(MulOps);
7802 }
7803 case Instruction::UDiv:
7804 LHS = getSCEV(BO->LHS);
7805 RHS = getSCEV(BO->RHS);
7806 return getUDivExpr(LHS, RHS);
7807 case Instruction::URem:
7808 LHS = getSCEV(BO->LHS);
7809 RHS = getSCEV(BO->RHS);
7810 return getURemExpr(LHS, RHS);
7811 case Instruction::Sub: {
7813 if (BO->Op)
7814 Flags = getNoWrapFlagsFromUB(BO->Op);
7815
7816 // Try to use ptrtoaddr for subtracts with at least one ptrtoint
7817 // operand. While we don't model ptrtoint directly in SCEV, the
7818 // difference between two pointer addresses is well-defined.
7819 Value *PtrLHS = nullptr, *PtrRHS = nullptr;
7820 bool HasPtrLHS = match(BO->LHS, m_PtrToInt(m_Value(PtrLHS)));
7821 bool HasPtrRHS = match(BO->RHS, m_PtrToInt(m_Value(PtrRHS)));
7822 if (HasPtrLHS || HasPtrRHS) {
7823 // Convert a ptrtoint operand (OrigOp) to ptrtoaddr of its pointer
7824 // PtrOp. When only one side is ptrtoint (BothPtr is false), skip
7825 // SCEVUnknown pointers since wrapping them in ptrtoaddr adds no
7826 // useful structure.
7827 auto GetOp = [&](bool HasPtr, Value *PtrOp, Value *OrigOp,
7828 bool BothPtr) -> const SCEV * {
7829 if (!HasPtr)
7830 return getSCEV(OrigOp);
7831 const SCEV *PtrSCEV = getSCEV(PtrOp);
7832 if (BothPtr || !isa<SCEVUnknown>(PtrSCEV)) {
7833 const SCEV *Addr = getPtrToAddrExpr(PtrSCEV);
7834 if (!isa<SCEVCouldNotCompute>(Addr) &&
7835 getTypeSizeInBits(OrigOp->getType()) <=
7836 getTypeSizeInBits(Addr->getType()))
7837 return getTruncateOrNoop(Addr, OrigOp->getType());
7838 }
7839 return getSCEV(OrigOp);
7840 };
7841 const SCEV *L = GetOp(HasPtrLHS, PtrLHS, BO->LHS, HasPtrRHS);
7842 const SCEV *R = GetOp(HasPtrRHS, PtrRHS, BO->RHS, HasPtrLHS);
7843 return getMinusSCEV(L, R, Flags);
7844 }
7845
7846 LHS = getSCEV(BO->LHS);
7847 RHS = getSCEV(BO->RHS);
7848 return getMinusSCEV(LHS, RHS, Flags);
7849 }
7850 case Instruction::And:
7851 // For an expression like x&255 that merely masks off the high bits,
7852 // use zext(trunc(x)) as the SCEV expression.
7853 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) {
7854 if (CI->isZero())
7855 return getSCEV(BO->RHS);
7856 if (CI->isMinusOne())
7857 return getSCEV(BO->LHS);
7858 const APInt &A = CI->getValue();
7859
7860 // Instcombine's ShrinkDemandedConstant may strip bits out of
7861 // constants, obscuring what would otherwise be a low-bits mask.
7862 // Use computeKnownBits to compute what ShrinkDemandedConstant
7863 // knew about to reconstruct a low-bits mask value.
7864 unsigned LZ = A.countl_zero();
7865 unsigned TZ = A.countr_zero();
7866 unsigned BitWidth = A.getBitWidth();
7867 KnownBits Known(BitWidth);
7868 computeKnownBits(BO->LHS, Known, getDataLayout(), &AC, nullptr, &DT);
7869
7870 APInt EffectiveMask =
7871 APInt::getLowBitsSet(BitWidth, BitWidth - LZ - TZ).shl(TZ);
7872 if ((LZ != 0 || TZ != 0) && !((~A & ~Known.Zero) & EffectiveMask)) {
7873 const SCEV *MulCount = getConstant(APInt::getOneBitSet(BitWidth, TZ));
7874 const SCEV *LHS = getSCEV(BO->LHS);
7875 const SCEV *ShiftedLHS = nullptr;
7876 if (auto *LHSMul = dyn_cast<SCEVMulExpr>(LHS)) {
7877 if (auto *OpC = dyn_cast<SCEVConstant>(LHSMul->getOperand(0))) {
7878 // For an expression like (x * 8) & 8, simplify the multiply.
7879 unsigned MulZeros = OpC->getAPInt().countr_zero();
7880 unsigned GCD = std::min(MulZeros, TZ);
7881 APInt DivAmt = APInt::getOneBitSet(BitWidth, TZ - GCD);
7883 MulOps.push_back(getConstant(OpC->getAPInt().ashr(GCD)));
7884 append_range(MulOps, LHSMul->operands().drop_front());
7885 const SCEV *NewMul = getMulExpr(MulOps, LHSMul->getNoWrapFlags());
7886 ShiftedLHS = getUDivExpr(NewMul, getConstant(DivAmt));
7887 }
7888 }
7889 if (!ShiftedLHS)
7890 ShiftedLHS = getUDivExpr(LHS, MulCount);
7891 return getMulExpr(
7893 getTruncateExpr(ShiftedLHS,
7894 IntegerType::get(getContext(), BitWidth - LZ - TZ)),
7895 BO->LHS->getType()),
7896 MulCount);
7897 }
7898 }
7899 // Binary `and` is a bit-wise `umin`.
7900 if (BO->LHS->getType()->isIntegerTy(1)) {
7901 LHS = getSCEV(BO->LHS);
7902 RHS = getSCEV(BO->RHS);
7903 return getUMinExpr(LHS, RHS);
7904 }
7905 break;
7906
7907 case Instruction::Or:
7908 // Binary `or` is a bit-wise `umax`.
7909 if (BO->LHS->getType()->isIntegerTy(1)) {
7910 LHS = getSCEV(BO->LHS);
7911 RHS = getSCEV(BO->RHS);
7912 return getUMaxExpr(LHS, RHS);
7913 }
7914 break;
7915
7916 case Instruction::Xor:
7917 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) {
7918 // If the RHS of xor is -1, then this is a not operation.
7919 if (CI->isMinusOne())
7920 return getNotSCEV(getSCEV(BO->LHS));
7921
7922 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
7923 // This is a variant of the check for xor with -1, and it handles
7924 // the case where instcombine has trimmed non-demanded bits out
7925 // of an xor with -1.
7926 if (auto *LBO = dyn_cast<BinaryOperator>(BO->LHS))
7927 if (ConstantInt *LCI = dyn_cast<ConstantInt>(LBO->getOperand(1)))
7928 if (LBO->getOpcode() == Instruction::And &&
7929 LCI->getValue() == CI->getValue())
7930 if (const SCEVZeroExtendExpr *Z =
7932 Type *UTy = BO->LHS->getType();
7933 const SCEV *Z0 = Z->getOperand();
7934 Type *Z0Ty = Z0->getType();
7935 unsigned Z0TySize = getTypeSizeInBits(Z0Ty);
7936
7937 // If C is a low-bits mask, the zero extend is serving to
7938 // mask off the high bits. Complement the operand and
7939 // re-apply the zext.
7940 if (CI->getValue().isMask(Z0TySize))
7941 return getZeroExtendExpr(getNotSCEV(Z0), UTy);
7942
7943 // If C is a single bit, it may be in the sign-bit position
7944 // before the zero-extend. In this case, represent the xor
7945 // using an add, which is equivalent, and re-apply the zext.
7946 APInt Trunc = CI->getValue().trunc(Z0TySize);
7947 if (Trunc.zext(getTypeSizeInBits(UTy)) == CI->getValue() &&
7948 Trunc.isSignMask())
7949 return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)),
7950 UTy);
7951 }
7952 }
7953 break;
7954
7955 case Instruction::Shl:
7956 // Turn shift left of a constant amount into a multiply.
7957 if (ConstantInt *SA = dyn_cast<ConstantInt>(BO->RHS)) {
7958 uint32_t BitWidth = cast<IntegerType>(SA->getType())->getBitWidth();
7959
7960 // If the shift count is not less than the bitwidth, the result of
7961 // the shift is undefined. Don't try to analyze it, because the
7962 // resolution chosen here may differ from the resolution chosen in
7963 // other parts of the compiler.
7964 if (SA->getValue().uge(BitWidth))
7965 break;
7966
7967 // We can safely preserve the nuw flag in all cases. It's also safe to
7968 // turn a nuw nsw shl into a nuw nsw mul. However, nsw in isolation
7969 // requires special handling. It can be preserved as long as we're not
7970 // left shifting by bitwidth - 1.
7971 auto Flags = SCEV::FlagNone;
7972 if (BO->Op) {
7973 auto MulFlags = getNoWrapFlagsFromUB(BO->Op);
7974 if (any(MulFlags & SCEV::FlagNSW) &&
7975 (any(MulFlags & SCEV::FlagNUW) ||
7976 SA->getValue().ult(BitWidth - 1)))
7978 if (any(MulFlags & SCEV::FlagNUW))
7980 }
7981
7982 ConstantInt *X = ConstantInt::get(
7983 getContext(), APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
7984 return getMulExpr(getSCEV(BO->LHS), getConstant(X), Flags);
7985 }
7986 break;
7987
7988 case Instruction::AShr:
7989 // AShr X, C, where C is a constant.
7990 ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS);
7991 if (!CI)
7992 break;
7993
7994 Type *OuterTy = BO->LHS->getType();
7996 // If the shift count is not less than the bitwidth, the result of
7997 // the shift is undefined. Don't try to analyze it, because the
7998 // resolution chosen here may differ from the resolution chosen in
7999 // other parts of the compiler.
8000 if (CI->getValue().uge(BitWidth))
8001 break;
8002
8003 if (CI->isZero())
8004 return getSCEV(BO->LHS); // shift by zero --> noop
8005
8006 uint64_t AShrAmt = CI->getZExtValue();
8007 Type *TruncTy = IntegerType::get(getContext(), BitWidth - AShrAmt);
8008
8009 Operator *L = dyn_cast<Operator>(BO->LHS);
8010 const SCEV *AddTruncateExpr = nullptr;
8011 ConstantInt *ShlAmtCI = nullptr;
8012 const SCEV *AddConstant = nullptr;
8013
8014 if (L && L->getOpcode() == Instruction::Add) {
8015 // X = Shl A, n
8016 // Y = Add X, c
8017 // Z = AShr Y, m
8018 // n, c and m are constants.
8019
8020 Operator *LShift = dyn_cast<Operator>(L->getOperand(0));
8021 ConstantInt *AddOperandCI = dyn_cast<ConstantInt>(L->getOperand(1));
8022 if (LShift && LShift->getOpcode() == Instruction::Shl) {
8023 if (AddOperandCI) {
8024 const SCEV *ShlOp0SCEV = getSCEV(LShift->getOperand(0));
8025 ShlAmtCI = dyn_cast<ConstantInt>(LShift->getOperand(1));
8026 // since we truncate to TruncTy, the AddConstant should be of the
8027 // same type, so create a new Constant with type same as TruncTy.
8028 // Also, the Add constant should be shifted right by AShr amount.
8029 APInt AddOperand = AddOperandCI->getValue().ashr(AShrAmt);
8030 AddConstant = getConstant(AddOperand.trunc(BitWidth - AShrAmt));
8031 // we model the expression as sext(add(trunc(A), c << n)), since the
8032 // sext(trunc) part is already handled below, we create a
8033 // AddExpr(TruncExp) which will be used later.
8034 AddTruncateExpr = getTruncateExpr(ShlOp0SCEV, TruncTy);
8035 }
8036 }
8037 } else if (L && L->getOpcode() == Instruction::Shl) {
8038 // X = Shl A, n
8039 // Y = AShr X, m
8040 // Both n and m are constant.
8041
8042 const SCEV *ShlOp0SCEV = getSCEV(L->getOperand(0));
8043 ShlAmtCI = dyn_cast<ConstantInt>(L->getOperand(1));
8044 AddTruncateExpr = getTruncateExpr(ShlOp0SCEV, TruncTy);
8045 }
8046
8047 if (AddTruncateExpr && ShlAmtCI) {
8048 // We can merge the two given cases into a single SCEV statement,
8049 // incase n = m, the mul expression will be 2^0, so it gets resolved to
8050 // a simpler case. The following code handles the two cases:
8051 //
8052 // 1) For a two-shift sext-inreg, i.e. n = m,
8053 // use sext(trunc(x)) as the SCEV expression.
8054 //
8055 // 2) When n > m, use sext(mul(trunc(x), 2^(n-m)))) as the SCEV
8056 // expression. We already checked that ShlAmt < BitWidth, so
8057 // the multiplier, 1 << (ShlAmt - AShrAmt), fits into TruncTy as
8058 // ShlAmt - AShrAmt < Amt.
8059 const APInt &ShlAmt = ShlAmtCI->getValue();
8060 if (ShlAmt.ult(BitWidth) && ShlAmt.uge(AShrAmt)) {
8061 APInt Mul = APInt::getOneBitSet(BitWidth - AShrAmt,
8062 ShlAmtCI->getZExtValue() - AShrAmt);
8063 const SCEV *CompositeExpr =
8064 getMulExpr(AddTruncateExpr, getConstant(Mul));
8065 if (L->getOpcode() != Instruction::Shl)
8066 CompositeExpr = getAddExpr(CompositeExpr, AddConstant);
8067
8068 return getSignExtendExpr(CompositeExpr, OuterTy);
8069 }
8070 }
8071 break;
8072 }
8073 }
8074
8075 switch (U->getOpcode()) {
8076 case Instruction::Trunc:
8077 return getTruncateExpr(getSCEV(U->getOperand(0)), U->getType());
8078
8079 case Instruction::ZExt:
8080 return getZeroExtendExpr(getSCEV(U->getOperand(0)), U->getType());
8081
8082 case Instruction::SExt:
8083 if (auto BO = MatchBinaryOp(U->getOperand(0), DT)) {
8084 // The NSW flag of a subtract does not always survive the conversion to
8085 // A + (-1)*B. By pushing sign extension onto its operands we are much
8086 // more likely to preserve NSW and allow later AddRec optimisations.
8087 //
8088 // NOTE: This is effectively duplicating this logic from getSignExtend:
8089 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
8090 // but by that point the NSW information has potentially been lost.
8091 if (BO->Opcode == Instruction::Sub && BO->IsNSW) {
8092 Type *Ty = U->getType();
8093 auto *V1 = getSignExtendExpr(getSCEV(BO->LHS), Ty);
8094 auto *V2 = getSignExtendExpr(getSCEV(BO->RHS), Ty);
8095 return getMinusSCEV(V1, V2, SCEV::FlagNSW);
8096 }
8097 }
8098 return getSignExtendExpr(getSCEV(U->getOperand(0)), U->getType());
8099
8100 case Instruction::BitCast:
8101 // BitCasts are no-op casts so we just eliminate the cast.
8102 if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType()))
8103 return getSCEV(U->getOperand(0));
8104 break;
8105
8106 case Instruction::PtrToAddr: {
8107 const SCEV *IntOp = getPtrToAddrExpr(getSCEV(U->getOperand(0)));
8108 if (isa<SCEVCouldNotCompute>(IntOp))
8109 return getUnknown(V);
8110 return IntOp;
8111 }
8112
8113 case Instruction::PtrToInt:
8114 // SCEV only models ptrtoaddr.
8115 return getUnknown(V);
8116
8117 case Instruction::IntToPtr:
8118 // Just don't deal with inttoptr casts.
8119 return getUnknown(V);
8120
8121 case Instruction::SDiv:
8122 // If both operands are non-negative, this is just an udiv.
8123 if (isKnownNonNegative(getSCEV(U->getOperand(0))) &&
8124 isKnownNonNegative(getSCEV(U->getOperand(1))))
8125 return getUDivExpr(getSCEV(U->getOperand(0)), getSCEV(U->getOperand(1)));
8126 break;
8127
8128 case Instruction::SRem:
8129 // If both operands are non-negative, this is just an urem.
8130 if (isKnownNonNegative(getSCEV(U->getOperand(0))) &&
8131 isKnownNonNegative(getSCEV(U->getOperand(1))))
8132 return getURemExpr(getSCEV(U->getOperand(0)), getSCEV(U->getOperand(1)));
8133 break;
8134
8135 case Instruction::GetElementPtr:
8136 return createNodeForGEP(cast<GEPOperator>(U));
8137
8138 case Instruction::PHI:
8139 return createNodeForPHI(cast<PHINode>(U));
8140
8141 case Instruction::Select:
8142 return createNodeForSelectOrPHI(U, U->getOperand(0), U->getOperand(1),
8143 U->getOperand(2));
8144
8145 case Instruction::Call:
8146 case Instruction::Invoke:
8147 if (Value *RV = cast<CallBase>(U)->getReturnedArgOperand())
8148 return getSCEV(RV);
8149
8150 if (auto *II = dyn_cast<IntrinsicInst>(U)) {
8151 switch (II->getIntrinsicID()) {
8152 case Intrinsic::abs:
8153 return getAbsExpr(
8154 getSCEV(II->getArgOperand(0)),
8155 /*IsNSW=*/cast<ConstantInt>(II->getArgOperand(1))->isOne());
8156 case Intrinsic::umax:
8157 LHS = getSCEV(II->getArgOperand(0));
8158 RHS = getSCEV(II->getArgOperand(1));
8159 return getUMaxExpr(LHS, RHS);
8160 case Intrinsic::umin:
8161 LHS = getSCEV(II->getArgOperand(0));
8162 RHS = getSCEV(II->getArgOperand(1));
8163 return getUMinExpr(LHS, RHS);
8164 case Intrinsic::smax:
8165 LHS = getSCEV(II->getArgOperand(0));
8166 RHS = getSCEV(II->getArgOperand(1));
8167 return getSMaxExpr(LHS, RHS);
8168 case Intrinsic::smin:
8169 LHS = getSCEV(II->getArgOperand(0));
8170 RHS = getSCEV(II->getArgOperand(1));
8171 return getSMinExpr(LHS, RHS);
8172 case Intrinsic::usub_sat: {
8173 const SCEV *X = getSCEV(II->getArgOperand(0));
8174 const SCEV *Y = getSCEV(II->getArgOperand(1));
8175 const SCEV *ClampedY = getUMinExpr(X, Y);
8176 return getMinusSCEV(X, ClampedY, SCEV::FlagNUW);
8177 }
8178 case Intrinsic::uadd_sat: {
8179 const SCEV *X = getSCEV(II->getArgOperand(0));
8180 const SCEV *Y = getSCEV(II->getArgOperand(1));
8181 const SCEV *ClampedX = getUMinExpr(X, getNotSCEV(Y));
8182 return getAddExpr(ClampedX, Y, SCEV::FlagNUW);
8183 }
8184 case Intrinsic::start_loop_iterations:
8185 case Intrinsic::annotation:
8186 case Intrinsic::ptr_annotation:
8187 // A start_loop_iterations or llvm.annotation or llvm.prt.annotation is
8188 // just eqivalent to the first operand for SCEV purposes.
8189 return getSCEV(II->getArgOperand(0));
8190 case Intrinsic::vscale:
8191 return getVScale(II->getType());
8192 default:
8193 break;
8194 }
8195 }
8196 break;
8197 }
8198
8199 return getUnknown(V);
8200}
8201
8202//===----------------------------------------------------------------------===//
8203// Iteration Count Computation Code
8204//
8205
8207 if (isa<SCEVCouldNotCompute>(ExitCount))
8208 return getCouldNotCompute();
8209
8210 auto *ExitCountType = ExitCount->getType();
8211 assert(ExitCountType->isIntegerTy());
8212 auto *EvalTy = Type::getIntNTy(ExitCountType->getContext(),
8213 1 + ExitCountType->getScalarSizeInBits());
8214 return getTripCountFromExitCount(ExitCount, EvalTy, nullptr);
8215}
8216
8218 Type *EvalTy,
8219 const Loop *L) {
8220 if (isa<SCEVCouldNotCompute>(ExitCount))
8221 return getCouldNotCompute();
8222
8223 unsigned ExitCountSize = getTypeSizeInBits(ExitCount->getType());
8224 unsigned EvalSize = EvalTy->getPrimitiveSizeInBits();
8225
8226 auto CanAddOneWithoutOverflow = [&]() {
8227 ConstantRange ExitCountRange =
8228 getRangeRef(ExitCount, RangeSignHint::HINT_RANGE_UNSIGNED);
8229 if (!ExitCountRange.contains(APInt::getMaxValue(ExitCountSize)))
8230 return true;
8231
8232 return L && isLoopEntryGuardedByCond(L, ICmpInst::ICMP_NE, ExitCount,
8233 getMinusOne(ExitCount->getType()));
8234 };
8235
8236 // If we need to zero extend the backedge count, check if we can add one to
8237 // it prior to zero extending without overflow. Provided this is safe, it
8238 // allows better simplification of the +1.
8239 if (EvalSize > ExitCountSize && CanAddOneWithoutOverflow())
8240 return getZeroExtendExpr(
8241 getAddExpr(ExitCount, getOne(ExitCount->getType())), EvalTy);
8242
8243 // Get the total trip count from the count by adding 1. This may wrap.
8244 return getAddExpr(getTruncateOrZeroExtend(ExitCount, EvalTy), getOne(EvalTy));
8245}
8246
8247static unsigned getConstantTripCount(const SCEVConstant *ExitCount) {
8248 if (!ExitCount)
8249 return 0;
8250
8251 ConstantInt *ExitConst = ExitCount->getValue();
8252
8253 // Guard against huge trip counts.
8254 if (ExitConst->getValue().getActiveBits() > 32)
8255 return 0;
8256
8257 // In case of integer overflow, this returns 0, which is correct.
8258 return ((unsigned)ExitConst->getZExtValue()) + 1;
8259}
8260
8262 auto *ExitCount = dyn_cast<SCEVConstant>(getBackedgeTakenCount(L, Exact));
8263 return getConstantTripCount(ExitCount);
8264}
8265
8266unsigned
8268 const BasicBlock *ExitingBlock) {
8269 assert(ExitingBlock && "Must pass a non-null exiting block!");
8270 assert(L->isLoopExiting(ExitingBlock) &&
8271 "Exiting block must actually branch out of the loop!");
8272 const SCEVConstant *ExitCount =
8273 dyn_cast<SCEVConstant>(getExitCount(L, ExitingBlock));
8274 return getConstantTripCount(ExitCount);
8275}
8276
8278 const Loop *L, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8279
8280 const auto *MaxExitCount =
8281 Predicates ? getPredicatedConstantMaxBackedgeTakenCount(L, *Predicates)
8283 return getConstantTripCount(dyn_cast<SCEVConstant>(MaxExitCount));
8284}
8285
8287 SmallVector<BasicBlock *, 8> ExitingBlocks;
8288 L->getExitingBlocks(ExitingBlocks);
8289
8290 // An exit with an uncomputable exit count makes the result 1.
8291 if (ExitingBlocks.empty() ||
8292 any_of(ExitingBlocks, [this, L](BasicBlock *ExitingBB) {
8293 return isa<SCEVCouldNotCompute>(getExitCount(L, ExitingBB));
8294 }))
8295 return 1;
8296
8297 LoopGuards Guards = LoopGuards::collectOnDemand(L, *this);
8298 unsigned Res = 0;
8299 for (BasicBlock *ExitingBB : ExitingBlocks)
8300 Res = std::gcd(
8301 Res, getSmallConstantTripMultiple(getExitCount(L, ExitingBB), Guards));
8302 return Res;
8303}
8304
8305unsigned ScalarEvolution::getSmallConstantTripMultiple(const SCEV *ExitCount,
8306 LoopGuards &Guards) {
8307 assert(!isa<SCEVCouldNotCompute>(ExitCount) && "Must be computable!");
8308
8309 // Get the trip count
8310 const SCEV *TCExpr =
8311 getTripCountFromExitCount(applyLoopGuards(ExitCount, Guards));
8312
8313 APInt Multiple = getNonZeroConstantMultiple(TCExpr);
8314 // If a trip multiple is huge (>=2^32), the trip count is still divisible by
8315 // the greatest power of 2 divisor less than 2^32.
8316 return Multiple.getActiveBits() > 32
8317 ? 1U << std::min(31U, Multiple.countTrailingZeros())
8318 : (unsigned)Multiple.getZExtValue();
8319}
8320
8322 const SCEV *ExitCount) {
8323 if (isa<SCEVCouldNotCompute>(ExitCount))
8324 return 1;
8325
8326 LoopGuards Guards = LoopGuards::collectOnDemand(L, *this);
8327 return getSmallConstantTripMultiple(ExitCount, Guards);
8328}
8329
8330/// Returns the largest constant divisor of the trip count of this loop as a
8331/// normal unsigned value, if possible. This means that the actual trip count is
8332/// always a multiple of the returned value (don't forget the trip count could
8333/// very well be zero as well!).
8334///
8335/// Returns 1 if the trip count is unknown or not guaranteed to be the
8336/// multiple of a constant (which is also the case if the trip count is simply
8337/// constant, use getSmallConstantTripCount for that case), Will also return 1
8338/// if the trip count is very large (>= 2^32).
8339///
8340/// As explained in the comments for getSmallConstantTripCount, this assumes
8341/// that control exits the loop via ExitingBlock.
8342unsigned
8344 const BasicBlock *ExitingBlock) {
8345 assert(ExitingBlock && "Must pass a non-null exiting block!");
8346 assert(L->isLoopExiting(ExitingBlock) &&
8347 "Exiting block must actually branch out of the loop!");
8348 const SCEV *ExitCount = getExitCount(L, ExitingBlock);
8349 return getSmallConstantTripMultiple(L, ExitCount);
8350}
8351
8353 const BasicBlock *ExitingBlock,
8354 ExitCountKind Kind) {
8355 switch (Kind) {
8356 case Exact:
8357 return getBackedgeTakenInfo(L).getExact(ExitingBlock, this);
8358 case SymbolicMaximum:
8359 return getBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, this);
8360 case ConstantMaximum:
8361 return getBackedgeTakenInfo(L).getConstantMax(ExitingBlock, this);
8362 };
8363 llvm_unreachable("Invalid ExitCountKind!");
8364}
8365
8367 const Loop *L, const BasicBlock *ExitingBlock,
8369 switch (Kind) {
8370 case Exact:
8371 return getPredicatedBackedgeTakenInfo(L).getExact(ExitingBlock, this,
8372 Predicates);
8373 case SymbolicMaximum:
8374 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, this,
8375 Predicates);
8376 case ConstantMaximum:
8377 return getPredicatedBackedgeTakenInfo(L).getConstantMax(ExitingBlock, this,
8378 Predicates);
8379 };
8380 llvm_unreachable("Invalid ExitCountKind!");
8381}
8382
8385 return getPredicatedBackedgeTakenInfo(L).getExact(L, this, &Preds);
8386}
8387
8389 ExitCountKind Kind) {
8390 switch (Kind) {
8391 case Exact:
8392 return getBackedgeTakenInfo(L).getExact(L, this);
8393 case ConstantMaximum:
8394 return getBackedgeTakenInfo(L).getConstantMax(this);
8395 case SymbolicMaximum:
8396 return getBackedgeTakenInfo(L).getSymbolicMax(L, this);
8397 };
8398 llvm_unreachable("Invalid ExitCountKind!");
8399}
8400
8403 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(L, this, &Preds);
8404}
8405
8408 return getPredicatedBackedgeTakenInfo(L).getConstantMax(this, &Preds);
8409}
8410
8412 return getBackedgeTakenInfo(L).isConstantMaxOrZero(this);
8413}
8414
8415/// Push PHI nodes in the header of the given loop onto the given Worklist.
8416static void PushLoopPHIs(const Loop *L,
8419 BasicBlock *Header = L->getHeader();
8420
8421 // Push all Loop-header PHIs onto the Worklist stack.
8422 for (PHINode &PN : Header->phis())
8423 if (Visited.insert(&PN).second)
8424 Worklist.push_back(&PN);
8425}
8426
8427ScalarEvolution::BackedgeTakenInfo &
8428ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) {
8429 auto &BTI = getBackedgeTakenInfo(L);
8430 if (BTI.hasFullInfo())
8431 return BTI;
8432
8433 auto Pair = PredicatedBackedgeTakenCounts.try_emplace(L);
8434
8435 if (!Pair.second)
8436 return Pair.first->second;
8437
8438 BackedgeTakenInfo Result =
8439 computeBackedgeTakenCount(L, /*AllowPredicates=*/true);
8440
8441 return PredicatedBackedgeTakenCounts.find(L)->second = std::move(Result);
8442}
8443
8444ScalarEvolution::BackedgeTakenInfo &
8445ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
8446 // Initially insert an invalid entry for this loop. If the insertion
8447 // succeeds, proceed to actually compute a backedge-taken count and
8448 // update the value. The temporary CouldNotCompute value tells SCEV
8449 // code elsewhere that it shouldn't attempt to request a new
8450 // backedge-taken count, which could result in infinite recursion.
8451 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
8452 BackedgeTakenCounts.try_emplace(L);
8453 if (!Pair.second)
8454 return Pair.first->second;
8455
8456 // computeBackedgeTakenCount may allocate memory for its result. Inserting it
8457 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
8458 // must be cleared in this scope.
8459 BackedgeTakenInfo Result = computeBackedgeTakenCount(L);
8460
8461 // Now that we know more about the trip count for this loop, forget any
8462 // existing SCEV values for PHI nodes in this loop since they are only
8463 // conservative estimates made without the benefit of trip count
8464 // information. This invalidation is not necessary for correctness, and is
8465 // only done to produce more precise results.
8466 if (Result.hasAnyInfo()) {
8467 // Invalidate any expression using an addrec in this loop.
8468 SmallVector<SCEVUse, 8> ToForget;
8469 auto LoopUsersIt = LoopUsers.find(L);
8470 if (LoopUsersIt != LoopUsers.end())
8471 append_range(ToForget, LoopUsersIt->second);
8472 forgetMemoizedResults(ToForget);
8473
8474 // Invalidate constant-evolved loop header phis.
8475 for (PHINode &PN : L->getHeader()->phis())
8476 ConstantEvolutionLoopExitValue.erase(&PN);
8477 }
8478
8479 // Re-lookup the insert position, since the call to
8480 // computeBackedgeTakenCount above could result in a
8481 // recusive call to getBackedgeTakenInfo (on a different
8482 // loop), which would invalidate the iterator computed
8483 // earlier.
8484 return BackedgeTakenCounts.find(L)->second = std::move(Result);
8485}
8486
8488 // This method is intended to forget all info about loops. It should
8489 // invalidate caches as if the following happened:
8490 // - The trip counts of all loops have changed arbitrarily
8491 // - Every llvm::Value has been updated in place to produce a different
8492 // result.
8493 BackedgeTakenCounts.clear();
8494 PredicatedBackedgeTakenCounts.clear();
8495 BECountUsers.clear();
8496 LoopPropertiesCache.clear();
8497 ConstantEvolutionLoopExitValue.clear();
8498 ValueExprMap.clear();
8499 ValuesAtScopes.clear();
8500 ValuesAtScopesUsers.clear();
8501 LoopDispositions.clear();
8502 BlockDispositions.clear();
8503 UnsignedRanges.clear();
8504 SignedRanges.clear();
8505 ExprValueMap.clear();
8506 HasRecMap.clear();
8507 ConstantMultipleCache.clear();
8508 PredicatedSCEVRewrites.clear();
8509 FoldCache.clear();
8510 FoldCacheUser.clear();
8511}
8512void ScalarEvolution::visitAndClearUsers(
8515 SmallVectorImpl<SCEVUse> &ToForget) {
8516 // Nothing can be invalidated if no value has a SCEV yet.
8517 if (ValueExprMap.empty()) {
8518 Worklist.clear();
8519 return;
8520 }
8521 while (!Worklist.empty()) {
8522 Instruction *I = Worklist.pop_back_val();
8523 if (!isSCEVable(I->getType()) && !isa<WithOverflowInst>(I))
8524 continue;
8525
8527 ValueExprMap.find_as(static_cast<Value *>(I));
8528 if (It != ValueExprMap.end()) {
8529 ToForget.push_back(It->second);
8530 eraseValueFromMap(It->first);
8531 if (PHINode *PN = dyn_cast<PHINode>(I))
8532 ConstantEvolutionLoopExitValue.erase(PN);
8533 }
8534
8535 PushDefUseChildren(I, Worklist, Visited);
8536 }
8537}
8538
8540 SmallVector<const Loop *, 16> LoopWorklist(1, L);
8543 SmallVector<SCEVUse, 16> ToForget;
8544
8545 // Iterate over all the loops and sub-loops to drop SCEV information.
8546 while (!LoopWorklist.empty()) {
8547 auto *CurrL = LoopWorklist.pop_back_val();
8548
8549 // Drop any stored trip count value.
8550 forgetBackedgeTakenCounts(CurrL, /* Predicated */ false);
8551 forgetBackedgeTakenCounts(CurrL, /* Predicated */ true);
8552
8553 // Drop information about predicated SCEV rewrites for this loop.
8554 PredicatedSCEVRewrites.remove_if(
8555 [&](const auto &Entry) { return Entry.first.second == CurrL; });
8556
8557 auto LoopUsersItr = LoopUsers.find(CurrL);
8558 if (LoopUsersItr != LoopUsers.end())
8559 llvm::append_range(ToForget, LoopUsersItr->second);
8560
8561 // Drop information about expressions based on loop-header PHIs.
8562 PushLoopPHIs(CurrL, Worklist, Visited);
8563 visitAndClearUsers(Worklist, Visited, ToForget);
8564
8565 LoopPropertiesCache.erase(CurrL);
8566 // Forget all contained loops too, to avoid dangling entries in the
8567 // ValuesAtScopes map.
8568 LoopWorklist.append(CurrL->begin(), CurrL->end());
8569 }
8570 forgetMemoizedResults(ToForget);
8571}
8572
8574 forgetLoop(L->getOutermostLoop());
8575}
8576
8579 if (!I) return;
8580
8581 // Drop information about expressions based on loop-header PHIs.
8584 SmallVector<SCEVUse, 8> ToForget;
8585 Worklist.push_back(I);
8586 Visited.insert(I);
8587 visitAndClearUsers(Worklist, Visited, ToForget);
8588
8589 forgetMemoizedResults(ToForget);
8590}
8591
8595 SmallVector<SCEVUse, 8> ToForget;
8596 for (Value *V : Values)
8597 if (auto *I = dyn_cast<Instruction>(V))
8598 if (Visited.insert(I).second)
8599 Worklist.push_back(I);
8600 visitAndClearUsers(Worklist, Visited, ToForget);
8601
8602 forgetMemoizedResults(ToForget);
8603}
8604
8606 // If SCEV looked through a trivial LCSSA phi node, we might have SCEV's
8607 // directly using a SCEVUnknown/SCEVAddRec defined in the loop. After an
8608 // extra predecessor is added, this is no longer valid. Find all Unknowns and
8609 // AddRecs defined in the loop and invalidate any SCEV's making use of them.
8610 auto InvalidateValue = [&](Value *Val) {
8611 if (!isSCEVable(Val->getType()))
8612 return;
8613 if (const SCEV *S = getExistingSCEV(Val)) {
8614 struct InvalidationRootCollector {
8615 Loop *L;
8617
8618 InvalidationRootCollector(Loop *L) : L(L) {}
8619
8620 bool follow(const SCEV *S) {
8621 if (auto *SU = dyn_cast<SCEVUnknown>(S)) {
8622 if (auto *I = dyn_cast<Instruction>(SU->getValue()))
8623 if (L->contains(I))
8624 Roots.push_back(S);
8625 } else if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(S)) {
8626 if (L->contains(AddRec->getLoop()))
8627 Roots.push_back(S);
8628 }
8629 return true;
8630 }
8631 bool isDone() const { return false; }
8632 };
8633
8634 InvalidationRootCollector C(L);
8635 visitAll(S, C);
8636 forgetMemoizedResults(C.Roots);
8637 }
8638 };
8639
8640 InvalidateValue(V);
8641
8642 // If V has a non-SCEV-able type (e.g. {i64, i1} from a with.overflow
8643 // intrinsic), its users (e.g. extractvalue) may have stale SCEV
8644 // expressions referencing loop-internal values.
8645 if (!isSCEVable(V->getType()) &&
8646 any_of(V->incoming_values(), IsaPred<WithOverflowInst>))
8647 for (User *U : V->users())
8648 InvalidateValue(U);
8649 // Also perform the normal invalidation.
8650 forgetValue(V);
8651}
8652
8653void ScalarEvolution::forgetLoopDispositions() { LoopDispositions.clear(); }
8654
8656 // Unless a specific value is passed to invalidation, completely clear both
8657 // caches.
8658 if (!V) {
8659 BlockDispositions.clear();
8660 LoopDispositions.clear();
8661 return;
8662 }
8663
8664 if (!isSCEVable(V->getType()))
8665 return;
8666
8667 const SCEV *S = getExistingSCEV(V);
8668 if (!S)
8669 return;
8670
8671 // Invalidate the block and loop dispositions cached for S. Dispositions of
8672 // S's users may change if S's disposition changes (i.e. a user may change to
8673 // loop-invariant, if S changes to loop invariant), so also invalidate
8674 // dispositions of S's users recursively.
8675 SmallVector<SCEVUse, 8> Worklist = {S};
8677 while (!Worklist.empty()) {
8678 const SCEV *Curr = Worklist.pop_back_val();
8679 bool LoopDispoRemoved = LoopDispositions.erase(Curr);
8680 bool BlockDispoRemoved = BlockDispositions.erase(Curr);
8681 if (!LoopDispoRemoved && !BlockDispoRemoved)
8682 continue;
8683 auto Users = SCEVUsers.find(Curr);
8684 if (Users != SCEVUsers.end())
8685 for (const auto *User : Users->second)
8686 if (Seen.insert(User).second)
8687 Worklist.push_back(User);
8688 }
8689}
8690
8691/// Get the exact loop backedge taken count considering all loop exits. A
8692/// computable result can only be returned for loops with all exiting blocks
8693/// dominating the latch. howFarToZero assumes that the limit of each loop test
8694/// is never skipped. This is a valid assumption as long as the loop exits via
8695/// that test. For precise results, it is the caller's responsibility to specify
8696/// the relevant loop exiting block using getExact(ExitingBlock, SE).
8697const SCEV *ScalarEvolution::BackedgeTakenInfo::getExact(
8698 const Loop *L, ScalarEvolution *SE,
8700 // If any exits were not computable, the loop is not computable.
8701 if (!isComplete() || ExitNotTaken.empty())
8702 return SE->getCouldNotCompute();
8703
8704 const BasicBlock *Latch = L->getLoopLatch();
8705 // All exiting blocks we have collected must dominate the only backedge.
8706 if (!Latch)
8707 return SE->getCouldNotCompute();
8708
8709 // All exiting blocks we have gathered dominate loop's latch, so exact trip
8710 // count is simply a minimum out of all these calculated exit counts.
8712 for (const auto &ENT : ExitNotTaken) {
8713 const SCEV *BECount = ENT.ExactNotTaken;
8714 assert(BECount != SE->getCouldNotCompute() && "Bad exit SCEV!");
8715 assert(SE->DT.dominates(ENT.ExitingBlock, Latch) &&
8716 "We should only have known counts for exiting blocks that dominate "
8717 "latch!");
8718
8719 Ops.push_back(BECount);
8720
8721 if (Preds)
8722 append_range(*Preds, ENT.Predicates);
8723
8724 assert((Preds || ENT.hasAlwaysTruePredicate()) &&
8725 "Predicate should be always true!");
8726 }
8727
8728 // If an earlier exit exits on the first iteration (exit count zero), then
8729 // a later poison exit count should not propagate into the result. This are
8730 // exactly the semantics provided by umin_seq.
8731 return SE->getUMinFromMismatchedTypes(Ops, /* Sequential */ true);
8732}
8733
8734const ScalarEvolution::ExitNotTakenInfo *
8735ScalarEvolution::BackedgeTakenInfo::getExitNotTaken(
8736 const BasicBlock *ExitingBlock,
8737 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8738 for (const auto &ENT : ExitNotTaken)
8739 if (ENT.ExitingBlock == ExitingBlock) {
8740 if (ENT.hasAlwaysTruePredicate())
8741 return &ENT;
8742 else if (Predicates) {
8743 append_range(*Predicates, ENT.Predicates);
8744 return &ENT;
8745 }
8746 }
8747
8748 return nullptr;
8749}
8750
8751/// getConstantMax - Get the constant max backedge taken count for the loop.
8752const SCEV *ScalarEvolution::BackedgeTakenInfo::getConstantMax(
8753 ScalarEvolution *SE,
8754 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8755 if (!getConstantMax())
8756 return SE->getCouldNotCompute();
8757
8758 for (const auto &ENT : ExitNotTaken)
8759 if (!ENT.hasAlwaysTruePredicate()) {
8760 if (!Predicates)
8761 return SE->getCouldNotCompute();
8762 append_range(*Predicates, ENT.Predicates);
8763 }
8764
8765 assert((isa<SCEVCouldNotCompute>(getConstantMax()) ||
8766 isa<SCEVConstant>(getConstantMax())) &&
8767 "No point in having a non-constant max backedge taken count!");
8768 return getConstantMax();
8769}
8770
8771const SCEV *ScalarEvolution::BackedgeTakenInfo::getSymbolicMax(
8772 const Loop *L, ScalarEvolution *SE,
8773 SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8774 if (!SymbolicMax) {
8775 // Form an expression for the maximum exit count possible for this loop. We
8776 // merge the max and exact information to approximate a version of
8777 // getConstantMaxBackedgeTakenCount which isn't restricted to just
8778 // constants.
8779 SmallVector<SCEVUse, 4> ExitCounts;
8780
8781 for (const auto &ENT : ExitNotTaken) {
8782 const SCEV *ExitCount = ENT.SymbolicMaxNotTaken;
8783 if (!isa<SCEVCouldNotCompute>(ExitCount)) {
8784 assert(SE->DT.dominates(ENT.ExitingBlock, L->getLoopLatch()) &&
8785 "We should only have known counts for exiting blocks that "
8786 "dominate latch!");
8787 ExitCounts.push_back(ExitCount);
8788 if (Predicates)
8789 append_range(*Predicates, ENT.Predicates);
8790
8791 assert((Predicates || ENT.hasAlwaysTruePredicate()) &&
8792 "Predicate should be always true!");
8793 }
8794 }
8795 if (ExitCounts.empty())
8796 SymbolicMax = SE->getCouldNotCompute();
8797 else
8798 SymbolicMax =
8799 SE->getUMinFromMismatchedTypes(ExitCounts, /*Sequential*/ true);
8800 }
8801 return SymbolicMax;
8802}
8803
8804bool ScalarEvolution::BackedgeTakenInfo::isConstantMaxOrZero(
8805 ScalarEvolution *SE) const {
8806 auto PredicateNotAlwaysTrue = [](const ExitNotTakenInfo &ENT) {
8807 return !ENT.hasAlwaysTruePredicate();
8808 };
8809 return MaxOrZero && !any_of(ExitNotTaken, PredicateNotAlwaysTrue);
8810}
8811
8814
8816 const SCEV *E, const SCEV *ConstantMaxNotTaken,
8817 const SCEV *SymbolicMaxNotTaken, bool MaxOrZero,
8821 // If we prove the max count is zero, so is the symbolic bound. This happens
8822 // in practice due to differences in a) how context sensitive we've chosen
8823 // to be and b) how we reason about bounds implied by UB.
8824 if (ConstantMaxNotTaken->isZero()) {
8825 this->ExactNotTaken = E = ConstantMaxNotTaken;
8826 this->SymbolicMaxNotTaken = SymbolicMaxNotTaken = ConstantMaxNotTaken;
8827 }
8828
8831 "Exact is not allowed to be less precise than Constant Max");
8834 "Exact is not allowed to be less precise than Symbolic Max");
8837 "Symbolic Max is not allowed to be less precise than Constant Max");
8840 "No point in having a non-constant max backedge taken count!");
8842 for (const auto PredList : PredLists)
8843 for (const auto *P : PredList) {
8844 if (SeenPreds.contains(P))
8845 continue;
8846 assert(!isa<SCEVUnionPredicate>(P) && "Only add leaf predicates here!");
8847 SeenPreds.insert(P);
8848 Predicates.push_back(P);
8849 }
8850 assert((isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) &&
8851 "Backedge count should be int");
8853 !ConstantMaxNotTaken->getType()->isPointerTy()) &&
8854 "Max backedge count should be int");
8855}
8856
8864
8865/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
8866/// computable exit into a persistent ExitNotTakenInfo array.
8867ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
8869 bool IsComplete, const SCEV *ConstantMax, bool MaxOrZero)
8870 : ConstantMax(ConstantMax), IsComplete(IsComplete), MaxOrZero(MaxOrZero) {
8871 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8872
8873 ExitNotTaken.reserve(ExitCounts.size());
8874 std::transform(ExitCounts.begin(), ExitCounts.end(),
8875 std::back_inserter(ExitNotTaken),
8876 [&](const EdgeExitInfo &EEI) {
8877 BasicBlock *ExitBB = EEI.first;
8878 const ExitLimit &EL = EEI.second;
8879 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken,
8880 EL.ConstantMaxNotTaken, EL.SymbolicMaxNotTaken,
8881 EL.Predicates);
8882 });
8883 assert((isa<SCEVCouldNotCompute>(ConstantMax) ||
8884 isa<SCEVConstant>(ConstantMax)) &&
8885 "No point in having a non-constant max backedge taken count!");
8886}
8887
8888/// Compute the number of times the backedge of the specified loop will execute.
8889ScalarEvolution::BackedgeTakenInfo
8890ScalarEvolution::computeBackedgeTakenCount(const Loop *L,
8891 bool AllowPredicates) {
8892 SmallVector<BasicBlock *, 8> ExitingBlocks;
8893 L->getExitingBlocks(ExitingBlocks);
8894
8895 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8896
8898 bool CouldComputeBECount = true;
8899 BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
8900 const SCEV *MustExitMaxBECount = nullptr;
8901 const SCEV *MayExitMaxBECount = nullptr;
8902 bool MustExitMaxOrZero = false;
8903 bool IsOnlyExit = ExitingBlocks.size() == 1;
8904
8905 // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts
8906 // and compute maxBECount.
8907 // Do a union of all the predicates here.
8908 for (BasicBlock *ExitBB : ExitingBlocks) {
8909 // We canonicalize untaken exits to br (constant), ignore them so that
8910 // proving an exit untaken doesn't negatively impact our ability to reason
8911 // about the loop as whole.
8912 if (auto *BI = dyn_cast<CondBrInst>(ExitBB->getTerminator()))
8913 if (auto *CI = dyn_cast<ConstantInt>(BI->getCondition())) {
8914 bool ExitIfTrue = !L->contains(BI->getSuccessor(0));
8915 if (ExitIfTrue == CI->isZero())
8916 continue;
8917 }
8918
8919 ExitLimit EL = computeExitLimit(L, ExitBB, IsOnlyExit, AllowPredicates);
8920
8921 assert((AllowPredicates || EL.Predicates.empty()) &&
8922 "Predicated exit limit when predicates are not allowed!");
8923
8924 // 1. For each exit that can be computed, add an entry to ExitCounts.
8925 // CouldComputeBECount is true only if all exits can be computed.
8926 if (EL.ExactNotTaken != getCouldNotCompute())
8927 ++NumExitCountsComputed;
8928 else
8929 // We couldn't compute an exact value for this exit, so
8930 // we won't be able to compute an exact value for the loop.
8931 CouldComputeBECount = false;
8932 // Remember exit count if either exact or symbolic is known. Because
8933 // Exact always implies symbolic, only check symbolic.
8934 if (EL.SymbolicMaxNotTaken != getCouldNotCompute())
8935 ExitCounts.emplace_back(ExitBB, EL);
8936 else {
8937 assert(EL.ExactNotTaken == getCouldNotCompute() &&
8938 "Exact is known but symbolic isn't?");
8939 ++NumExitCountsNotComputed;
8940 }
8941
8942 // 2. Derive the loop's MaxBECount from each exit's max number of
8943 // non-exiting iterations. Partition the loop exits into two kinds:
8944 // LoopMustExits and LoopMayExits.
8945 //
8946 // If the exit dominates the loop latch, it is a LoopMustExit otherwise it
8947 // is a LoopMayExit. If any computable LoopMustExit is found, then
8948 // MaxBECount is the minimum EL.ConstantMaxNotTaken of computable
8949 // LoopMustExits. Otherwise, MaxBECount is conservatively the maximum
8950 // EL.ConstantMaxNotTaken, where CouldNotCompute is considered greater than
8951 // any
8952 // computable EL.ConstantMaxNotTaken.
8953 if (EL.ConstantMaxNotTaken != getCouldNotCompute() && Latch &&
8954 DT.dominates(ExitBB, Latch)) {
8955 if (!MustExitMaxBECount) {
8956 MustExitMaxBECount = EL.ConstantMaxNotTaken;
8957 MustExitMaxOrZero = EL.MaxOrZero;
8958 } else {
8959 MustExitMaxBECount = getUMinFromMismatchedTypes(MustExitMaxBECount,
8960 EL.ConstantMaxNotTaken);
8961 }
8962 } else if (MayExitMaxBECount != getCouldNotCompute()) {
8963 if (!MayExitMaxBECount || EL.ConstantMaxNotTaken == getCouldNotCompute())
8964 MayExitMaxBECount = EL.ConstantMaxNotTaken;
8965 else {
8966 MayExitMaxBECount = getUMaxFromMismatchedTypes(MayExitMaxBECount,
8967 EL.ConstantMaxNotTaken);
8968 }
8969 }
8970 }
8971 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
8972 (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute());
8973 // The loop backedge will be taken the maximum or zero times if there's
8974 // a single exit that must be taken the maximum or zero times.
8975 bool MaxOrZero = (MustExitMaxOrZero && ExitingBlocks.size() == 1);
8976
8977 // Remember which SCEVs are used in exit limits for invalidation purposes.
8978 // We only care about non-constant SCEVs here, so we can ignore
8979 // EL.ConstantMaxNotTaken
8980 // and MaxBECount, which must be SCEVConstant.
8981 for (const auto &Pair : ExitCounts) {
8982 if (!isa<SCEVConstant>(Pair.second.ExactNotTaken))
8983 BECountUsers[Pair.second.ExactNotTaken].insert({L, AllowPredicates});
8984 if (!isa<SCEVConstant>(Pair.second.SymbolicMaxNotTaken))
8985 BECountUsers[Pair.second.SymbolicMaxNotTaken].insert(
8986 {L, AllowPredicates});
8987 }
8988 return BackedgeTakenInfo(std::move(ExitCounts), CouldComputeBECount,
8989 MaxBECount, MaxOrZero);
8990}
8991
8992ScalarEvolution::ExitLimit
8993ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock,
8994 bool IsOnlyExit, bool AllowPredicates) {
8995 assert(L->contains(ExitingBlock) && "Exit count for non-loop block?");
8996 // If our exiting block does not dominate the latch, then its connection with
8997 // loop's exit limit may be far from trivial.
8998 const BasicBlock *Latch = L->getLoopLatch();
8999 if (!Latch || !DT.dominates(ExitingBlock, Latch))
9000 return getCouldNotCompute();
9001
9002 Instruction *Term = ExitingBlock->getTerminator();
9003 if (CondBrInst *BI = dyn_cast<CondBrInst>(Term)) {
9004 bool ExitIfTrue = !L->contains(BI->getSuccessor(0));
9005 assert(ExitIfTrue == L->contains(BI->getSuccessor(1)) &&
9006 "It should have one successor in loop and one exit block!");
9007 // Proceed to the next level to examine the exit condition expression.
9008 return computeExitLimitFromCond(L, BI->getCondition(), ExitIfTrue,
9009 /*ControlsOnlyExit=*/IsOnlyExit,
9010 AllowPredicates);
9011 }
9012
9013 if (SwitchInst *SI = dyn_cast<SwitchInst>(Term)) {
9014 // For switch, make sure that there is a single exit from the loop.
9015 BasicBlock *Exit = nullptr;
9016 for (auto *SBB : successors(ExitingBlock))
9017 if (!L->contains(SBB)) {
9018 if (Exit) // Multiple exit successors.
9019 return getCouldNotCompute();
9020 Exit = SBB;
9021 }
9022 assert(Exit && "Exiting block must have at least one exit");
9023 return computeExitLimitFromSingleExitSwitch(
9024 L, SI, Exit, /*ControlsOnlyExit=*/IsOnlyExit);
9025 }
9026
9027 return getCouldNotCompute();
9028}
9029
9031 const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9032 bool AllowPredicates) {
9033 ScalarEvolution::ExitLimitCacheTy Cache(L, ExitIfTrue, AllowPredicates);
9034 return computeExitLimitFromCondCached(Cache, L, ExitCond, ExitIfTrue,
9035 ControlsOnlyExit, AllowPredicates);
9036}
9037
9038std::optional<ScalarEvolution::ExitLimit>
9039ScalarEvolution::ExitLimitCache::find(const Loop *L, Value *ExitCond,
9040 bool ExitIfTrue, bool ControlsOnlyExit,
9041 bool AllowPredicates) {
9042 (void)this->L;
9043 (void)this->ExitIfTrue;
9044 (void)this->AllowPredicates;
9045
9046 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9047 this->AllowPredicates == AllowPredicates &&
9048 "Variance in assumed invariant key components!");
9049 auto Itr = TripCountMap.find({ExitCond, ControlsOnlyExit});
9050 if (Itr == TripCountMap.end())
9051 return std::nullopt;
9052 return Itr->second;
9053}
9054
9055void ScalarEvolution::ExitLimitCache::insert(const Loop *L, Value *ExitCond,
9056 bool ExitIfTrue,
9057 bool ControlsOnlyExit,
9058 bool AllowPredicates,
9059 const ExitLimit &EL) {
9060 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9061 this->AllowPredicates == AllowPredicates &&
9062 "Variance in assumed invariant key components!");
9063
9064 auto InsertResult = TripCountMap.insert({{ExitCond, ControlsOnlyExit}, EL});
9065 assert(InsertResult.second && "Expected successful insertion!");
9066 (void)InsertResult;
9067 (void)ExitIfTrue;
9068}
9069
9070ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondCached(
9071 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9072 bool ControlsOnlyExit, bool AllowPredicates) {
9073
9074 if (auto MaybeEL = Cache.find(L, ExitCond, ExitIfTrue, ControlsOnlyExit,
9075 AllowPredicates))
9076 return *MaybeEL;
9077
9078 ExitLimit EL = computeExitLimitFromCondImpl(
9079 Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates);
9080 Cache.insert(L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates, EL);
9081 return EL;
9082}
9083
9084ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl(
9085 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9086 bool ControlsOnlyExit, bool AllowPredicates) {
9087 // Handle BinOp conditions (And, Or).
9088 if (auto LimitFromBinOp = computeExitLimitFromCondFromBinOp(
9089 Cache, L, ExitCond, ExitIfTrue, AllowPredicates))
9090 return *LimitFromBinOp;
9091
9092 // With an icmp, it may be feasible to compute an exact backedge-taken count.
9093 // Proceed to the next level to examine the icmp.
9094 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond)) {
9095 ExitLimit EL =
9096 computeExitLimitFromICmp(L, ExitCondICmp, ExitIfTrue, ControlsOnlyExit);
9097 if (EL.hasFullInfo() || !AllowPredicates)
9098 return EL;
9099
9100 // Try again, but use SCEV predicates this time.
9101 return computeExitLimitFromICmp(L, ExitCondICmp, ExitIfTrue,
9102 ControlsOnlyExit,
9103 /*AllowPredicates=*/true);
9104 }
9105
9106 // Check for a constant condition. These are normally stripped out by
9107 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
9108 // preserve the CFG and is temporarily leaving constant conditions
9109 // in place.
9110 if (ConstantInt *CI = dyn_cast<ConstantInt>(ExitCond)) {
9111 if (ExitIfTrue == !CI->getZExtValue())
9112 // The backedge is always taken.
9113 return getCouldNotCompute();
9114 // The backedge is never taken.
9115 return getZero(CI->getType());
9116 }
9117
9118 // If we're exiting based on the overflow flag of an x.with.overflow intrinsic
9119 // with a constant step, we can form an equivalent icmp predicate and figure
9120 // out how many iterations will be taken before we exit.
9121 const WithOverflowInst *WO;
9122 const APInt *C;
9123 if (match(ExitCond, m_ExtractValue<1>(m_WithOverflowInst(WO))) &&
9124 match(WO->getRHS(), m_APInt(C))) {
9125 ConstantRange NWR =
9127 WO->getNoWrapKind());
9129 APInt NewRHSC, Offset;
9130 NWR.getEquivalentICmp(Pred, NewRHSC, Offset);
9131 if (!ExitIfTrue)
9133 auto *LHS = getSCEV(WO->getLHS());
9134 if (Offset != 0)
9136 auto EL = computeExitLimitFromICmp(L, Pred, LHS, getConstant(NewRHSC),
9137 ControlsOnlyExit, AllowPredicates);
9138 if (EL.hasAnyInfo())
9139 return EL;
9140 }
9141
9142 // If it's not an integer or pointer comparison then compute it the hard way.
9143 return computeExitCountExhaustively(L, ExitCond, ExitIfTrue);
9144}
9145
9146std::optional<ScalarEvolution::ExitLimit>
9147ScalarEvolution::computeExitLimitFromCondFromBinOp(ExitLimitCacheTy &Cache,
9148 const Loop *L,
9149 Value *ExitCond,
9150 bool ExitIfTrue,
9151 bool AllowPredicates) {
9152 // Check if the controlling expression for this loop is an And or Or.
9153 Value *Op0, *Op1;
9154 bool IsAnd;
9155 if (match(ExitCond, m_LogicalAnd(m_Value(Op0), m_Value(Op1))))
9156 IsAnd = true;
9157 else if (match(ExitCond, m_LogicalOr(m_Value(Op0), m_Value(Op1))))
9158 IsAnd = false;
9159 else
9160 return std::nullopt;
9161
9162 // A sub-condition of a non-trivial binop never solely controls the exit,
9163 // whether we exit always depends on both conditions.
9164 ExitLimit EL0 = computeExitLimitFromCondCached(
9165 Cache, L, Op0, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9166 ExitLimit EL1 = computeExitLimitFromCondCached(
9167 Cache, L, Op1, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9168
9169 // EitherMayExit is true in these two cases:
9170 // br (and Op0 Op1), loop, exit
9171 // br (or Op0 Op1), exit, loop
9172 bool EitherMayExit = IsAnd ^ ExitIfTrue;
9173
9174 const SCEV *BECount = getCouldNotCompute();
9175 const SCEV *ConstantMaxBECount = getCouldNotCompute();
9176 const SCEV *SymbolicMaxBECount = getCouldNotCompute();
9177 if (EitherMayExit) {
9178 bool UseSequentialUMin = !isa<BinaryOperator>(ExitCond);
9179 // Both conditions must be same for the loop to continue executing.
9180 // Choose the less conservative count.
9181 if (EL0.ExactNotTaken != getCouldNotCompute() &&
9182 EL1.ExactNotTaken != getCouldNotCompute()) {
9183 BECount = getUMinFromMismatchedTypes(EL0.ExactNotTaken, EL1.ExactNotTaken,
9184 UseSequentialUMin);
9185 }
9186 if (EL0.ConstantMaxNotTaken == getCouldNotCompute())
9187 ConstantMaxBECount = EL1.ConstantMaxNotTaken;
9188 else if (EL1.ConstantMaxNotTaken == getCouldNotCompute())
9189 ConstantMaxBECount = EL0.ConstantMaxNotTaken;
9190 else
9191 ConstantMaxBECount = getUMinFromMismatchedTypes(EL0.ConstantMaxNotTaken,
9192 EL1.ConstantMaxNotTaken);
9193 if (EL0.SymbolicMaxNotTaken == getCouldNotCompute())
9194 SymbolicMaxBECount = EL1.SymbolicMaxNotTaken;
9195 else if (EL1.SymbolicMaxNotTaken == getCouldNotCompute())
9196 SymbolicMaxBECount = EL0.SymbolicMaxNotTaken;
9197 else
9198 SymbolicMaxBECount = getUMinFromMismatchedTypes(
9199 EL0.SymbolicMaxNotTaken, EL1.SymbolicMaxNotTaken, UseSequentialUMin);
9200 } else {
9201 // Both conditions must be same at the same time for the loop to exit.
9202 // For now, be conservative.
9203 if (EL0.ExactNotTaken == EL1.ExactNotTaken)
9204 BECount = EL0.ExactNotTaken;
9205 }
9206
9207 // There are cases (e.g. PR26207) where computeExitLimitFromCond is able
9208 // to be more aggressive when computing BECount than when computing
9209 // ConstantMaxBECount. In these cases it is possible for EL0.ExactNotTaken
9210 // and
9211 // EL1.ExactNotTaken to match, but for EL0.ConstantMaxNotTaken and
9212 // EL1.ConstantMaxNotTaken to not.
9213 if (isa<SCEVCouldNotCompute>(ConstantMaxBECount) &&
9214 !isa<SCEVCouldNotCompute>(BECount))
9215 ConstantMaxBECount = getConstant(getUnsignedRangeMax(BECount));
9216 if (isa<SCEVCouldNotCompute>(SymbolicMaxBECount))
9217 SymbolicMaxBECount =
9218 isa<SCEVCouldNotCompute>(BECount) ? ConstantMaxBECount : BECount;
9219 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
9220 {ArrayRef(EL0.Predicates), ArrayRef(EL1.Predicates)});
9221}
9222
9223ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9224 const Loop *L, ICmpInst *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9225 bool AllowPredicates) {
9226 // If the condition was exit on true, convert the condition to exit on false
9227 CmpPredicate Pred;
9228 if (!ExitIfTrue)
9229 Pred = ExitCond->getCmpPredicate();
9230 else
9231 Pred = ExitCond->getInverseCmpPredicate();
9232 const ICmpInst::Predicate OriginalPred = Pred;
9233
9234 const SCEV *LHS = getSCEV(ExitCond->getOperand(0));
9235 const SCEV *RHS = getSCEV(ExitCond->getOperand(1));
9236
9237 ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, ControlsOnlyExit,
9238 AllowPredicates);
9239 if (EL.hasAnyInfo())
9240 return EL;
9241
9242 auto *ExhaustiveCount =
9243 computeExitCountExhaustively(L, ExitCond, ExitIfTrue);
9244
9245 if (!isa<SCEVCouldNotCompute>(ExhaustiveCount))
9246 return ExhaustiveCount;
9247
9248 return computeShiftCompareExitLimit(ExitCond->getOperand(0),
9249 ExitCond->getOperand(1), L, OriginalPred);
9250}
9251ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9252 const Loop *L, CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS,
9253 bool ControlsOnlyExit, bool AllowPredicates) {
9254
9255 // Try to evaluate any dependencies out of the loop.
9256 LHS = getSCEVAtScope(LHS, L);
9257 RHS = getSCEVAtScope(RHS, L);
9258
9259 // At this point, we would like to compute how many iterations of the
9260 // loop the predicate will return true for these inputs.
9261 if (isLoopInvariant(LHS, L) && !isLoopInvariant(RHS, L)) {
9262 // If there is a loop-invariant, force it into the RHS.
9263 std::swap(LHS, RHS);
9265 }
9266
9267 bool ControllingFiniteLoop = ControlsOnlyExit && loopHasNoAbnormalExits(L) &&
9269 // Simplify the operands before analyzing them.
9270 (void)SimplifyICmpOperands(Pred, LHS, RHS, /*Depth=*/0);
9271
9272 // If we have a comparison of a chrec against a constant, try to use value
9273 // ranges to answer this query.
9274 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS))
9275 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS))
9276 if (AddRec->getLoop() == L) {
9277 // Form the constant range.
9278 ConstantRange CompRange =
9279 ConstantRange::makeExactICmpRegion(Pred, RHSC->getAPInt());
9280
9281 const SCEV *Ret = AddRec->getNumIterationsInRange(CompRange, *this);
9282 if (!isa<SCEVCouldNotCompute>(Ret)) return Ret;
9283 }
9284
9285 // If this loop must exit based on this condition (or execute undefined
9286 // behaviour), see if we can improve wrap flags. This is essentially
9287 // a must execute style proof.
9288 if (ControllingFiniteLoop && isLoopInvariant(RHS, L)) {
9289 // If we can prove the test sequence produced must repeat the same values
9290 // on self-wrap of the IV, then we can infer that IV doesn't self wrap
9291 // because if it did, we'd have an infinite (undefined) loop.
9292 // TODO: We can peel off any functions which are invertible *in L*. Loop
9293 // invariant terms are effectively constants for our purposes here.
9294 SCEVUse InnerLHS = LHS;
9295 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(LHS))
9296 InnerLHS = ZExt->getOperand();
9297 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(InnerLHS);
9298 AR && !AR->hasNoSelfWrap() && AR->getLoop() == L && AR->isAffine() &&
9299 isKnownToBeAPowerOfTwo(AR->getStepRecurrence(*this), /*OrZero=*/true,
9300 /*OrNegative=*/true)) {
9301 auto Flags = AR->getNoWrapFlags();
9302 Flags = setFlags(Flags, SCEV::FlagNW);
9305 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), Flags);
9306 }
9307
9308 // For a slt/ult condition with a positive step, can we prove nsw/nuw?
9309 // From no-self-wrap, this follows trivially from the fact that every
9310 // (un)signed-wrapped, but not self-wrapped value must be LT than the
9311 // last value before (un)signed wrap. Since we know that last value
9312 // didn't exit, nor will any smaller one.
9314 auto WrapType = Pred == ICmpInst::ICMP_SLT ? SCEV::FlagNSW : SCEV::FlagNUW;
9315 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS);
9316 AR && AR->getLoop() == L && AR->isAffine() &&
9317 !AR->getNoWrapFlags(WrapType) && AR->hasNoSelfWrap() &&
9318 isKnownPositive(AR->getStepRecurrence(*this))) {
9319 auto Flags = AR->getNoWrapFlags();
9320 Flags = setFlags(Flags, WrapType);
9323 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), Flags);
9324 }
9325 }
9326 }
9327
9328 switch (Pred) {
9329 case ICmpInst::ICMP_NE: { // while (X != Y)
9330 // Convert to: while (X-Y != 0)
9331 if (LHS->getType()->isPointerTy()) {
9334 return LHS;
9335 }
9336 if (RHS->getType()->isPointerTy()) {
9339 return RHS;
9340 }
9341 ExitLimit EL = howFarToZero(getMinusSCEV(LHS, RHS), L, ControlsOnlyExit,
9342 AllowPredicates);
9343 if (EL.hasAnyInfo())
9344 return EL;
9345 break;
9346 }
9347 case ICmpInst::ICMP_EQ: { // while (X == Y)
9348 // Convert to: while (X-Y == 0)
9349 if (LHS->getType()->isPointerTy()) {
9352 return LHS;
9353 }
9354 if (RHS->getType()->isPointerTy()) {
9357 return RHS;
9358 }
9359 ExitLimit EL = howFarToNonZero(getMinusSCEV(LHS, RHS), L);
9360 if (EL.hasAnyInfo()) return EL;
9361 break;
9362 }
9363 case ICmpInst::ICMP_SLE:
9364 case ICmpInst::ICMP_ULE:
9365 // Since the loop is finite, an invariant RHS cannot include the boundary
9366 // value, otherwise it would loop forever.
9367 if (!Opts.scalar_evolution_finite_loop || !ControllingFiniteLoop ||
9368 !isLoopInvariant(RHS, L)) {
9369 // Otherwise, perform the addition in a wider type, to avoid overflow.
9370 // If the LHS is an addrec with the appropriate nowrap flag, the
9371 // extension will be sunk into it and the exit count can be analyzed.
9372 auto *OldType = dyn_cast<IntegerType>(LHS->getType());
9373 if (!OldType)
9374 break;
9375 // Prefer doubling the bitwidth over adding a single bit to make it more
9376 // likely that we use a legal type.
9377 auto *NewType =
9378 Type::getIntNTy(OldType->getContext(), OldType->getBitWidth() * 2);
9379 if (ICmpInst::isSigned(Pred)) {
9380 LHS = getSignExtendExpr(LHS, NewType);
9381 RHS = getSignExtendExpr(RHS, NewType);
9382 } else {
9383 LHS = getZeroExtendExpr(LHS, NewType);
9384 RHS = getZeroExtendExpr(RHS, NewType);
9385 }
9386 }
9388 [[fallthrough]];
9389 case ICmpInst::ICMP_SLT:
9390 case ICmpInst::ICMP_ULT: { // while (X < Y)
9391 bool IsSigned = ICmpInst::isSigned(Pred);
9392 ExitLimit EL = howManyLessThans(LHS, RHS, L, IsSigned, /*Invert=*/false,
9393 ControlsOnlyExit, AllowPredicates);
9394 if (EL.hasAnyInfo())
9395 return EL;
9396 break;
9397 }
9398 case ICmpInst::ICMP_SGE:
9399 case ICmpInst::ICMP_UGE:
9400 // Since the loop is finite, an invariant RHS cannot include the boundary
9401 // value, otherwise it would loop forever.
9402 if (!Opts.scalar_evolution_finite_loop || !ControllingFiniteLoop ||
9403 !isLoopInvariant(RHS, L))
9404 break;
9406 [[fallthrough]];
9407 case ICmpInst::ICMP_SGT:
9408 case ICmpInst::ICMP_UGT: { // while (X > Y)
9409 // "X > Y" is analyzed as the equivalent "~X < ~Y".
9410 bool IsSigned = ICmpInst::isSigned(Pred);
9411 ExitLimit EL = howManyLessThans(LHS, RHS, L, IsSigned, /*Invert=*/true,
9412 ControlsOnlyExit, AllowPredicates);
9413 if (EL.hasAnyInfo())
9414 return EL;
9415 break;
9416 }
9417 default:
9418 break;
9419 }
9420
9421 return getCouldNotCompute();
9422}
9423
9424ScalarEvolution::ExitLimit
9425ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L,
9426 SwitchInst *Switch,
9427 BasicBlock *ExitingBlock,
9428 bool ControlsOnlyExit) {
9429 assert(!L->contains(ExitingBlock) && "Not an exiting block!");
9430
9431 // Give up if the exit is the default dest of a switch.
9432 if (Switch->getDefaultDest() == ExitingBlock)
9433 return getCouldNotCompute();
9434
9435 assert(L->contains(Switch->getDefaultDest()) &&
9436 "Default case must not exit the loop!");
9437 const SCEV *LHS = getSCEVAtScope(Switch->getCondition(), L);
9438 const SCEV *RHS = getConstant(Switch->findCaseDest(ExitingBlock));
9439
9440 // while (X != Y) --> while (X-Y != 0)
9441 ExitLimit EL = howFarToZero(getMinusSCEV(LHS, RHS), L, ControlsOnlyExit);
9442 if (EL.hasAnyInfo())
9443 return EL;
9444
9445 return getCouldNotCompute();
9446}
9447
9448static ConstantInt *
9450 ScalarEvolution &SE) {
9451 const SCEV *InVal = SE.getConstant(C);
9452 const SCEV *Val = AddRec->evaluateAtIteration(InVal, SE);
9454 "Evaluation of SCEV at constant didn't fold correctly?");
9455 return cast<SCEVConstant>(Val)->getValue();
9456}
9457
9458ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit(
9459 Value *LHS, Value *RHSV, const Loop *L, ICmpInst::Predicate Pred) {
9460 ConstantInt *RHS = dyn_cast<ConstantInt>(RHSV);
9461 if (!RHS)
9462 return getCouldNotCompute();
9463
9464 const BasicBlock *Latch = L->getLoopLatch();
9465 if (!Latch)
9466 return getCouldNotCompute();
9467
9468 const BasicBlock *Predecessor = L->getLoopPredecessor();
9469 if (!Predecessor)
9470 return getCouldNotCompute();
9471
9472 // Return true if V is of the form "LHS `shift_op` <positive constant>".
9473 // Return LHS in OutLHS, shift_op in OutOpCode, and the shift amount in
9474 // OutShiftAmt.
9475 auto MatchPositiveShift = [](Value *V, Value *&OutLHS,
9476 Instruction::BinaryOps &OutOpCode,
9477 unsigned &OutShiftAmt) {
9478 using namespace PatternMatch;
9479
9480 ConstantInt *ShiftAmt;
9481 if (match(V, m_LShr(m_Value(OutLHS), m_ConstantInt(ShiftAmt))))
9482 OutOpCode = Instruction::LShr;
9483 else if (match(V, m_AShr(m_Value(OutLHS), m_ConstantInt(ShiftAmt))))
9484 OutOpCode = Instruction::AShr;
9485 else if (match(V, m_Shl(m_Value(OutLHS), m_ConstantInt(ShiftAmt))))
9486 OutOpCode = Instruction::Shl;
9487 else
9488 return false;
9489
9490 uint64_t Amt = ShiftAmt->getValue().getLimitedValue();
9491 if (Amt == 0 || Amt >= OutLHS->getType()->getScalarSizeInBits())
9492 return false;
9493 OutShiftAmt = Amt;
9494 return true;
9495 };
9496
9497 // Recognize a "shift recurrence" either of the form %iv or of %iv.shifted in
9498 //
9499 // loop:
9500 // %iv = phi i32 [ %iv.shifted, %loop ], [ %val, %preheader ]
9501 // %iv.shifted = lshr i32 %iv, <positive constant>
9502 //
9503 // Return true on a successful match. Return the corresponding PHI node (%iv
9504 // above) in PNOut, the opcode of the shift operation in OpCodeOut, and the
9505 // shift amount in ShiftAmtOut.
9506 auto MatchShiftRecurrence = [&](Value *V, PHINode *&PNOut,
9507 Instruction::BinaryOps &OpCodeOut,
9508 unsigned &ShiftAmtOut) {
9509 std::optional<Instruction::BinaryOps> PostShiftOpCode;
9510
9511 {
9513 Value *V;
9514 unsigned Amt;
9515
9516 // If we encounter a shift instruction, "peel off" the shift operation,
9517 // and remember that we did so. Later when we inspect %iv's backedge
9518 // value, we will make sure that the backedge value uses the same
9519 // operation.
9520 //
9521 // Note: the peeled shift operation does not have to be the same
9522 // instruction as the one feeding into the PHI's backedge value. We only
9523 // really care about it being the same *kind* of shift instruction --
9524 // that's all that is required for our later inferences to hold.
9525 if (MatchPositiveShift(LHS, V, OpC, Amt)) {
9526 PostShiftOpCode = OpC;
9527 LHS = V;
9528 }
9529 }
9530
9531 PNOut = dyn_cast<PHINode>(LHS);
9532 if (!PNOut || PNOut->getParent() != L->getHeader())
9533 return false;
9534
9535 Value *BEValue = PNOut->getIncomingValueForBlock(Latch);
9536 Value *OpLHS;
9537
9538 return
9539 // The backedge value for the PHI node must be a shift by a positive
9540 // amount
9541 MatchPositiveShift(BEValue, OpLHS, OpCodeOut, ShiftAmtOut) &&
9542
9543 // of the PHI node itself
9544 OpLHS == PNOut &&
9545
9546 // and the kind of shift should be match the kind of shift we peeled
9547 // off, if any.
9548 (!PostShiftOpCode || *PostShiftOpCode == OpCodeOut);
9549 };
9550
9551 PHINode *PN;
9553 unsigned ShiftAmt;
9554 if (!MatchShiftRecurrence(LHS, PN, OpCode, ShiftAmt))
9555 return getCouldNotCompute();
9556
9557 const DataLayout &DL = getDataLayout();
9558
9559 // The key rationale for this optimization is that for some kinds of shift
9560 // recurrences, the value of the recurrence "stabilizes" to either 0 or -1
9561 // within a finite number of iterations. If the condition guarding the
9562 // backedge (in the sense that the backedge is taken if the condition is true)
9563 // is false for the value the shift recurrence stabilizes to, then we know
9564 // that the backedge is taken only a finite number of times.
9565
9566 ConstantInt *StableValue = nullptr;
9567 switch (OpCode) {
9568 default:
9569 llvm_unreachable("Impossible case!");
9570
9571 case Instruction::AShr: {
9572 // {K,ashr,<positive-constant>} stabilizes to signum(K) in at most
9573 // bitwidth(K) iterations.
9574 Value *FirstValue = PN->getIncomingValueForBlock(Predecessor);
9575 KnownBits Known = computeKnownBits(FirstValue, DL, &AC,
9576 Predecessor->getTerminator(), &DT);
9577 auto *Ty = cast<IntegerType>(RHS->getType());
9578 if (Known.isNonNegative())
9579 StableValue = ConstantInt::get(Ty, 0);
9580 else if (Known.isNegative())
9581 StableValue = ConstantInt::get(Ty, -1, true);
9582 else
9583 return getCouldNotCompute();
9584
9585 break;
9586 }
9587 case Instruction::LShr:
9588 case Instruction::Shl:
9589 // Both {K,lshr,<positive-constant>} and {K,shl,<positive-constant>}
9590 // stabilize to 0 in at most bitwidth(K) iterations.
9591 StableValue = ConstantInt::get(cast<IntegerType>(RHS->getType()), 0);
9592 break;
9593 }
9594
9595 auto *Result =
9596 ConstantFoldCompareInstOperands(Pred, StableValue, RHS, DL, &TLI);
9597 assert(Result->getType()->isIntegerTy(1) &&
9598 "Otherwise cannot be an operand to a branch instruction");
9599
9600 if (Result->isNullValue()) {
9601 unsigned BitWidth = getTypeSizeInBits(RHS->getType());
9602 unsigned MaxBTC = BitWidth;
9603
9604 // For right-shift recurrences (lshr/ashr with non-negative start), we can
9605 // compute a tighter max backedge-taken count from the range of the start
9606 // value. After k shifts of ShiftAmt, value = start >> (k * ShiftAmt).
9607 // The value reaches 0 (the stable value) when k * ShiftAmt >=
9608 // activeBits(start), so max BTC = ceil(activeBits(maxStart) / ShiftAmt).
9609 if (OpCode == Instruction::LShr || OpCode == Instruction::AShr) {
9610 Value *StartValue = PN->getIncomingValueForBlock(Predecessor);
9611 const SCEV *StartSCEV = getSCEV(StartValue);
9612 APInt MaxStart = getUnsignedRangeMax(StartSCEV);
9613 if (MaxStart.isStrictlyPositive()) {
9614 unsigned ActiveBits = MaxStart.getActiveBits();
9615 unsigned RangeBTC = divideCeil(ActiveBits, ShiftAmt);
9616 MaxBTC = std::min(MaxBTC, RangeBTC);
9617 }
9618 }
9619
9620 const SCEV *UpperBound =
9622 return ExitLimit(getCouldNotCompute(), UpperBound, UpperBound, false);
9623 }
9624
9625 return getCouldNotCompute();
9626}
9627
9628/// Return true if we can constant fold an instruction of the specified type,
9629/// assuming that all operands were constants.
9630static bool canConstantFold(const Instruction *I,
9631 const TargetLibraryInfo *TLI) {
9635 return true;
9636
9637 if (const CallInst *CI = dyn_cast<CallInst>(I))
9638 if (const Function *F = CI->getCalledFunction())
9639 return canConstantFoldCallTo(CI, F, TLI);
9640 return false;
9641}
9642
9643/// Determine whether this instruction can constant evolve within this loop
9644/// assuming its operands can all constant evolve.
9645static bool canConstantEvolve(Instruction *I, const Loop *L,
9646 const TargetLibraryInfo *TLI) {
9647 // An instruction outside of the loop can't be derived from a loop PHI.
9648 if (!L->contains(I)) return false;
9649
9650 if (isa<PHINode>(I)) {
9651 // We don't currently keep track of the control flow needed to evaluate
9652 // PHIs, so we cannot handle PHIs inside of loops.
9653 return L->getHeader() == I->getParent();
9654 }
9655
9656 // If we won't be able to constant fold this expression even if the operands
9657 // are constants, bail early.
9658 return canConstantFold(I, TLI);
9659}
9660
9661/// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
9662/// recursing through each instruction operand until reaching a loop header phi.
9663static PHINode *
9664getConstantEvolvingPHIOperands(const AnalysisOptions &Opts,
9665 Instruction *UseInst, const Loop *L,
9667 const TargetLibraryInfo *TLI, unsigned Depth) {
9668 if (Depth > Opts.scalar_evolution_max_constant_evolving_depth)
9669 return nullptr;
9670
9671 // Otherwise, we can evaluate this instruction if all of its operands are
9672 // constant or derived from a PHI node themselves.
9673 PHINode *PHI = nullptr;
9674 for (Value *Op : UseInst->operands()) {
9675 if (isa<Constant>(Op)) continue;
9676
9678 if (!OpInst || !canConstantEvolve(OpInst, L, TLI))
9679 return nullptr;
9680
9681 PHINode *P = dyn_cast<PHINode>(OpInst);
9682 if (!P)
9683 // If this operand is already visited, reuse the prior result.
9684 // We may have P != PHI if this is the deepest point at which the
9685 // inconsistent paths meet.
9686 P = PHIMap.lookup(OpInst);
9687 if (!P) {
9688 // Recurse and memoize the results, whether a phi is found or not.
9689 // This recursive call invalidates pointers into PHIMap.
9690 P = getConstantEvolvingPHIOperands(Opts, OpInst, L, PHIMap, TLI,
9691 Depth + 1);
9692 PHIMap[OpInst] = P;
9693 }
9694 if (!P)
9695 return nullptr; // Not evolving from PHI
9696 if (PHI && PHI != P)
9697 return nullptr; // Evolving from multiple different PHIs.
9698 PHI = P;
9699 }
9700 // This is a expression evolving from a constant PHI!
9701 return PHI;
9702}
9703
9704/// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
9705/// in the loop that V is derived from. We allow arbitrary operations along the
9706/// way, but the operands of an operation must either be constants or a value
9707/// derived from a constant PHI. If this expression does not fit with these
9708/// constraints, return null.
9709static PHINode *getConstantEvolvingPHI(const AnalysisOptions &Opts, Value *V,
9710 const Loop *L,
9711 const TargetLibraryInfo *TLI) {
9713 if (!I || !canConstantEvolve(I, L, TLI))
9714 return nullptr;
9715
9716 if (PHINode *PN = dyn_cast<PHINode>(I))
9717 return PN;
9718
9719 // Record non-constant instructions contained by the loop.
9721 return getConstantEvolvingPHIOperands(Opts, I, L, PHIMap, TLI, 0);
9722}
9723
9724/// EvaluateExpression - Given an expression that passes the
9725/// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
9726/// in the loop has the value PHIVal. If we can't fold this expression for some
9727/// reason, return null.
9730 const DataLayout &DL,
9731 const TargetLibraryInfo *TLI) {
9732 // Convenient constant check, but redundant for recursive calls.
9733 if (Constant *C = dyn_cast<Constant>(V)) return C;
9735 if (!I) return nullptr;
9736
9737 if (Constant *C = Vals.lookup(I)) return C;
9738
9739 // An instruction inside the loop depends on a value outside the loop that we
9740 // weren't given a mapping for, or a value such as a call inside the loop.
9741 if (!canConstantEvolve(I, L, TLI))
9742 return nullptr;
9743
9744 // An unmapped PHI can be due to a branch or another loop inside this loop,
9745 // or due to this not being the initial iteration through a loop where we
9746 // couldn't compute the evolution of this particular PHI last time.
9747 if (isa<PHINode>(I)) return nullptr;
9748
9749 std::vector<Constant*> Operands(I->getNumOperands());
9750
9751 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
9752 Instruction *Operand = dyn_cast<Instruction>(I->getOperand(i));
9753 if (!Operand) {
9754 Operands[i] = dyn_cast<Constant>(I->getOperand(i));
9755 if (!Operands[i]) return nullptr;
9756 continue;
9757 }
9758 Constant *C = EvaluateExpression(Operand, L, Vals, DL, TLI);
9759 Vals[Operand] = C;
9760 if (!C) return nullptr;
9761 Operands[i] = C;
9762 }
9763
9765 /*AllowNonDeterministic=*/false);
9766}
9767
9768
9769// If every incoming value to PN except the one for BB is a specific Constant,
9770// return that, else return nullptr.
9772 Constant *IncomingVal = nullptr;
9773
9774 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
9775 if (PN->getIncomingBlock(i) == BB)
9776 continue;
9777
9778 auto *CurrentVal = dyn_cast<Constant>(PN->getIncomingValue(i));
9779 if (!CurrentVal)
9780 return nullptr;
9781
9782 if (IncomingVal != CurrentVal) {
9783 if (IncomingVal)
9784 return nullptr;
9785 IncomingVal = CurrentVal;
9786 }
9787 }
9788
9789 return IncomingVal;
9790}
9791
9792/// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
9793/// in the header of its containing loop, we know the loop executes a
9794/// constant number of times, and the PHI node is just a recurrence
9795/// involving constants, fold it.
9796Constant *
9797ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
9798 const APInt &BEs,
9799 const Loop *L) {
9800 auto [I, Inserted] = ConstantEvolutionLoopExitValue.try_emplace(PN);
9801 if (!Inserted)
9802 return I->second;
9803
9804 if (BEs.ugt(Opts.scalar_evolution_max_iterations))
9805 return nullptr; // Not going to evaluate it.
9806
9807 Constant *&RetVal = I->second;
9808
9809 DenseMap<Instruction *, Constant *> CurrentIterVals;
9810 BasicBlock *Header = L->getHeader();
9811 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
9812
9813 BasicBlock *Latch = L->getLoopLatch();
9814 if (!Latch)
9815 return nullptr;
9816
9817 for (PHINode &PHI : Header->phis()) {
9818 if (auto *StartCST = getOtherIncomingValue(&PHI, Latch))
9819 CurrentIterVals[&PHI] = StartCST;
9820 }
9821 if (!CurrentIterVals.count(PN))
9822 return RetVal = nullptr;
9823
9824 Value *BEValue = PN->getIncomingValueForBlock(Latch);
9825
9826 // Execute the loop symbolically to determine the exit value.
9827 assert(BEs.getActiveBits() < CHAR_BIT * sizeof(unsigned) &&
9828 "BEs is <= -scalar-evolution-max-iterations which is an 'unsigned'!");
9829
9830 unsigned NumIterations = BEs.getZExtValue(); // must be in range
9831 unsigned IterationNum = 0;
9832 const DataLayout &DL = getDataLayout();
9833 for (; ; ++IterationNum) {
9834 if (IterationNum == NumIterations)
9835 return RetVal = CurrentIterVals[PN]; // Got exit value!
9836
9837 // Compute the value of the PHIs for the next iteration.
9838 // EvaluateExpression adds non-phi values to the CurrentIterVals map.
9839 DenseMap<Instruction *, Constant *> NextIterVals;
9840 Constant *NextPHI =
9841 EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI);
9842 if (!NextPHI)
9843 return nullptr; // Couldn't evaluate!
9844 NextIterVals[PN] = NextPHI;
9845
9846 bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
9847
9848 // Also evaluate the other PHI nodes. However, we don't get to stop if we
9849 // cease to be able to evaluate one of them or if they stop evolving,
9850 // because that doesn't necessarily prevent us from computing PN.
9852 for (const auto &I : CurrentIterVals) {
9853 PHINode *PHI = dyn_cast<PHINode>(I.first);
9854 if (!PHI || PHI == PN || PHI->getParent() != Header) continue;
9855 PHIsToCompute.emplace_back(PHI, I.second);
9856 }
9857 // We use two distinct loops because EvaluateExpression may invalidate any
9858 // iterators into CurrentIterVals.
9859 for (const auto &I : PHIsToCompute) {
9860 PHINode *PHI = I.first;
9861 Constant *&NextPHI = NextIterVals[PHI];
9862 if (!NextPHI) { // Not already computed.
9863 Value *BEValue = PHI->getIncomingValueForBlock(Latch);
9864 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI);
9865 }
9866 if (NextPHI != I.second)
9867 StoppedEvolving = false;
9868 }
9869
9870 // If all entries in CurrentIterVals == NextIterVals then we can stop
9871 // iterating, the loop can't continue to change.
9872 if (StoppedEvolving)
9873 return RetVal = CurrentIterVals[PN];
9874
9875 CurrentIterVals.swap(NextIterVals);
9876 }
9877}
9878
9879const SCEV *ScalarEvolution::computeExitCountExhaustively(const Loop *L,
9880 Value *Cond,
9881 bool ExitWhen) {
9882 PHINode *PN = getConstantEvolvingPHI(Opts, Cond, L, &TLI);
9883 if (!PN) return getCouldNotCompute();
9884
9885 // If the loop is canonicalized, the PHI will have exactly two entries.
9886 // That's the only form we support here.
9887 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
9888
9889 DenseMap<Instruction *, Constant *> CurrentIterVals;
9890 BasicBlock *Header = L->getHeader();
9891 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
9892
9893 BasicBlock *Latch = L->getLoopLatch();
9894 assert(Latch && "Should follow from NumIncomingValues == 2!");
9895
9896 for (PHINode &PHI : Header->phis()) {
9897 if (auto *StartCST = getOtherIncomingValue(&PHI, Latch))
9898 CurrentIterVals[&PHI] = StartCST;
9899 }
9900 if (!CurrentIterVals.count(PN))
9901 return getCouldNotCompute();
9902
9903 // Okay, we find a PHI node that defines the trip count of this loop. Execute
9904 // the loop symbolically to determine when the condition gets a value of
9905 // "ExitWhen".
9906 unsigned MaxIterations =
9907 Opts.scalar_evolution_max_iterations; // Limit analysis.
9908 const DataLayout &DL = getDataLayout();
9909 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
9910 auto *CondVal = dyn_cast_or_null<ConstantInt>(
9911 EvaluateExpression(Cond, L, CurrentIterVals, DL, &TLI));
9912
9913 // Couldn't symbolically evaluate.
9914 if (!CondVal) return getCouldNotCompute();
9915
9916 if (CondVal->getValue() == uint64_t(ExitWhen)) {
9917 ++NumBruteForceTripCountsComputed;
9918 return getConstant(Type::getInt32Ty(getContext()), IterationNum);
9919 }
9920
9921 // Update all the PHI nodes for the next iteration.
9922 DenseMap<Instruction *, Constant *> NextIterVals;
9923
9924 // Create a list of which PHIs we need to compute. We want to do this before
9925 // calling EvaluateExpression on them because that may invalidate iterators
9926 // into CurrentIterVals.
9927 SmallVector<PHINode *, 8> PHIsToCompute;
9928 for (const auto &I : CurrentIterVals) {
9929 PHINode *PHI = dyn_cast<PHINode>(I.first);
9930 if (!PHI || PHI->getParent() != Header) continue;
9931 PHIsToCompute.push_back(PHI);
9932 }
9933 for (PHINode *PHI : PHIsToCompute) {
9934 Constant *&NextPHI = NextIterVals[PHI];
9935 if (NextPHI) continue; // Already computed!
9936
9937 Value *BEValue = PHI->getIncomingValueForBlock(Latch);
9938 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI);
9939 }
9940 CurrentIterVals.swap(NextIterVals);
9941 }
9942
9943 // Too many iterations were needed to evaluate.
9944 return getCouldNotCompute();
9945}
9946
9948 if (isa<SCEVConstant>(V))
9949 return V;
9950
9951 auto &Values = ValuesAtScopes[V];
9952 // Check to see if we've folded this expression at this loop before.
9953 for (auto &LS : Values)
9954 if (LS.first == L)
9955 return LS.second ? LS.second : SCEVUse(V);
9956
9957 Values.emplace_back(L, nullptr);
9958
9959 // Otherwise compute it.
9960 SCEVUse C = computeSCEVAtScope(V, L);
9961 for (auto &LS : reverse(ValuesAtScopes[V]))
9962 if (LS.first == L) {
9963 LS.second = C;
9964 // Record the dependency under the bare expression: invalidation walks
9965 // expressions, and any use flags on C do not change which expression
9966 // this is the value at scope of.
9967 if (!isa<SCEVConstant>(C))
9968 ValuesAtScopesUsers[C.getPointer()].push_back({L, V});
9969 break;
9970 }
9971 return C;
9972}
9973
9975 const BasicBlock *ExitingBlock) {
9976 SCEVUse ExitValue = getSCEVAtScope(V, L->getParentLoop());
9977 if (!isLoopInvariant(ExitValue, L)) {
9978 // If we failed to evaluate it in the outer scope, try to evaluate an
9979 // addrec for the specific exit.
9980 // TODO: Generalize this to other expressions.
9981 const SCEV *ExitCount = getExitCount(L, ExitingBlock);
9982 if (!isa<SCEVCouldNotCompute>(ExitCount))
9983 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(V))
9984 if (AddRec->getLoop() == L)
9985 ExitValue = AddRec->evaluateAtIteration(ExitCount, *this);
9986 }
9987 return ExitValue;
9988}
9989
9990/// This builds up a Constant using the ConstantExpr interface. That way, we
9991/// will return Constants for objects which aren't represented by a
9992/// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
9993/// Returns NULL if the SCEV isn't representable as a Constant.
9995 switch (V->getSCEVType()) {
9996 case scCouldNotCompute:
9997 case scAddRecExpr:
9998 case scVScale:
9999 return nullptr;
10000 case scConstant:
10001 return cast<SCEVConstant>(V)->getValue();
10002 case scUnknown:
10004 case scPtrToAddr: {
10006 if (Constant *CastOp = BuildConstantFromSCEV(P2I->getOperand()))
10007 return ConstantExpr::getPtrToAddr(CastOp, P2I->getType());
10008
10009 return nullptr;
10010 }
10011 case scTruncate: {
10013 if (Constant *CastOp = BuildConstantFromSCEV(ST->getOperand()))
10014 return ConstantExpr::getTrunc(CastOp, ST->getType());
10015 return nullptr;
10016 }
10017 case scAddExpr: {
10018 const SCEVAddExpr *SA = cast<SCEVAddExpr>(V);
10019 Constant *C = nullptr;
10020 for (const SCEV *Op : SA->operands()) {
10022 if (!OpC)
10023 return nullptr;
10024 if (!C) {
10025 C = OpC;
10026 continue;
10027 }
10028 assert(!C->getType()->isPointerTy() &&
10029 "Can only have one pointer, and it must be last");
10030 if (OpC->getType()->isPointerTy()) {
10031 // The offsets have been converted to bytes. We can add bytes using
10032 // an i8 GEP.
10033 C = ConstantExpr::getPtrAdd(OpC, C);
10034 } else {
10035 C = ConstantExpr::getAdd(C, OpC);
10036 }
10037 }
10038 return C;
10039 }
10040 case scMulExpr:
10041 case scSignExtend:
10042 case scZeroExtend:
10043 case scUDivExpr:
10044 case scSMaxExpr:
10045 case scUMaxExpr:
10046 case scSMinExpr:
10047 case scUMinExpr:
10049 return nullptr;
10050 }
10051 llvm_unreachable("Unknown SCEV kind!");
10052}
10053
10054const SCEV *ScalarEvolution::getWithOperands(const SCEV *S,
10055 SmallVectorImpl<SCEVUse> &NewOps) {
10056 switch (S->getSCEVType()) {
10057 case scTruncate:
10058 case scZeroExtend:
10059 case scSignExtend:
10060 case scPtrToAddr:
10061 return getCastExpr(S->getSCEVType(), NewOps[0], S->getType());
10062 case scAddRecExpr: {
10063 auto *AddRec = cast<SCEVAddRecExpr>(S);
10064 return getAddRecExpr(NewOps, AddRec->getLoop(), AddRec->getNoWrapFlags());
10065 }
10066 case scAddExpr:
10067 return getAddExpr(NewOps, cast<SCEVAddExpr>(S)->getNoWrapFlags());
10068 case scMulExpr:
10069 return getMulExpr(NewOps, cast<SCEVMulExpr>(S)->getNoWrapFlags());
10070 case scUDivExpr:
10071 return getUDivExpr(NewOps[0], NewOps[1]);
10072 case scUMaxExpr:
10073 case scSMaxExpr:
10074 case scUMinExpr:
10075 case scSMinExpr:
10076 return getMinMaxExpr(S->getSCEVType(), NewOps);
10078 return getSequentialMinMaxExpr(S->getSCEVType(), NewOps);
10079 case scConstant:
10080 case scVScale:
10081 case scUnknown:
10082 return S;
10083 case scCouldNotCompute:
10084 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10085 }
10086 llvm_unreachable("Unknown SCEV kind!");
10087}
10088
10089SCEVUse ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
10090 switch (V->getSCEVType()) {
10091 case scConstant:
10092 case scVScale:
10093 return V;
10094 case scAddRecExpr: {
10095 // If this is a loop recurrence for a loop that does not contain L, then we
10096 // are dealing with the final value computed by the loop.
10097 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(V);
10098 // First, attempt to evaluate each operand.
10099 // Avoid performing the look-up in the common case where the specified
10100 // expression has no loop-variant portions.
10101 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
10102 SCEVUse OpAtScope = getSCEVAtScope(AddRec->getOperand(i), L);
10103 if (OpAtScope == AddRec->getOperand(i))
10104 continue;
10105
10106 // Okay, at least one of these operands is loop variant but might be
10107 // foldable. Build a new instance of the folded commutative expression.
10109 NewOps.reserve(AddRec->getNumOperands());
10110 append_range(NewOps, AddRec->operands().take_front(i));
10111 NewOps.push_back(OpAtScope);
10112 for (++i; i != e; ++i)
10113 NewOps.push_back(getSCEVAtScope(AddRec->getOperand(i), L));
10114
10115 const SCEV *FoldedRec = getAddRecExpr(
10116 NewOps, AddRec->getLoop(), AddRec->getNoWrapFlags(SCEV::FlagNW));
10117 AddRec = dyn_cast<SCEVAddRecExpr>(FoldedRec);
10118 // The addrec may be folded to a nonrecurrence, for example, if the
10119 // induction variable is multiplied by zero after constant folding. Go
10120 // ahead and return the folded value.
10121 if (!AddRec)
10122 return FoldedRec;
10123 break;
10124 }
10125
10126 // If the scope is outside the addrec's loop, evaluate it by using the
10127 // loop exit value of the addrec.
10128 if (!AddRec->getLoop()->contains(L)) {
10129 SCEVUse ExitValue = AddRec->getExitValue(*this);
10130 if (isa<SCEVCouldNotCompute>(ExitValue))
10131 return AddRec;
10132 return ExitValue;
10133 }
10134
10135 return AddRec;
10136 }
10137 case scTruncate:
10138 case scZeroExtend:
10139 case scSignExtend:
10140 case scPtrToAddr:
10141 case scAddExpr:
10142 case scMulExpr:
10143 case scUDivExpr:
10144 case scUMaxExpr:
10145 case scSMaxExpr:
10146 case scUMinExpr:
10147 case scSMinExpr:
10148 case scSequentialUMinExpr: {
10149 ArrayRef<SCEVUse> Ops = V->operands();
10150 // Avoid performing the look-up in the common case where the specified
10151 // expression has no loop-variant portions.
10152 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
10153 SCEVUse OpAtScope = getSCEVAtScope(Ops[i].getPointer(), L);
10154 if (OpAtScope != Ops[i].getPointer()) {
10155 // Okay, at least one of these operands is loop variant but might be
10156 // foldable. Build a new instance of the folded commutative expression.
10158 NewOps.reserve(Ops.size());
10159 append_range(NewOps, Ops.take_front(i));
10160 NewOps.push_back(OpAtScope);
10161
10162 for (++i; i != e; ++i) {
10163 OpAtScope = getSCEVAtScope(Ops[i].getPointer(), L);
10164 NewOps.push_back(OpAtScope);
10165 }
10166
10167 return getWithOperands(V, NewOps);
10168 }
10169 }
10170 // If we got here, all operands are loop invariant.
10171 return V;
10172 }
10173 case scUnknown: {
10174 // If this instruction is evolved from a constant-evolving PHI, compute the
10175 // exit value from the loop without using SCEVs.
10176 const SCEVUnknown *SU = cast<SCEVUnknown>(V);
10178 if (!I)
10179 return V; // This is some other type of SCEVUnknown, just return it.
10180
10181 if (PHINode *PN = dyn_cast<PHINode>(I)) {
10182 const Loop *CurrLoop = this->LI[I->getParent()];
10183 // Looking for loop exit value.
10184 if (CurrLoop && CurrLoop->getParentLoop() == L &&
10185 PN->getParent() == CurrLoop->getHeader()) {
10186 // Okay, there is no closed form solution for the PHI node. Check
10187 // to see if the loop that contains it has a known backedge-taken
10188 // count. If so, we may be able to force computation of the exit
10189 // value.
10190 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(CurrLoop);
10191 // This trivial case can show up in some degenerate cases where
10192 // the incoming IR has not yet been fully simplified.
10193 if (BackedgeTakenCount->isZero()) {
10194 Value *InitValue = nullptr;
10195 bool MultipleInitValues = false;
10196 for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) {
10197 if (!CurrLoop->contains(PN->getIncomingBlock(i))) {
10198 if (!InitValue)
10199 InitValue = PN->getIncomingValue(i);
10200 else if (InitValue != PN->getIncomingValue(i)) {
10201 MultipleInitValues = true;
10202 break;
10203 }
10204 }
10205 }
10206 if (!MultipleInitValues && InitValue)
10207 return getSCEV(InitValue);
10208 }
10209 // Do we have a loop invariant value flowing around the backedge
10210 // for a loop which must execute the backedge?
10211 if (!isa<SCEVCouldNotCompute>(BackedgeTakenCount) &&
10212 isKnownNonZero(BackedgeTakenCount) &&
10213 PN->getNumIncomingValues() == 2) {
10214
10215 unsigned InLoopPred =
10216 CurrLoop->contains(PN->getIncomingBlock(0)) ? 0 : 1;
10217 Value *BackedgeVal = PN->getIncomingValue(InLoopPred);
10218 if (CurrLoop->isLoopInvariant(BackedgeVal))
10219 return getSCEV(BackedgeVal);
10220 }
10221 if (auto *BTCC = dyn_cast<SCEVConstant>(BackedgeTakenCount)) {
10222 // Okay, we know how many times the containing loop executes. If
10223 // this is a constant evolving PHI node, get the final value at
10224 // the specified iteration number.
10225 Constant *RV =
10226 getConstantEvolutionLoopExitValue(PN, BTCC->getAPInt(), CurrLoop);
10227 if (RV)
10228 return getSCEV(RV);
10229 }
10230 }
10231 }
10232
10233 // Okay, this is an expression that we cannot symbolically evaluate
10234 // into a SCEV. Check to see if it's possible to symbolically evaluate
10235 // the arguments into constants, and if so, try to constant propagate the
10236 // result. This is particularly useful for computing loop exit values.
10237 if (!canConstantFold(I, &TLI))
10238 return V; // This is some other type of SCEVUnknown, just return it.
10239
10240 SmallVector<Constant *, 4> Operands;
10241 Operands.reserve(I->getNumOperands());
10242 bool MadeImprovement = false;
10243 for (Value *Op : I->operands()) {
10244 if (Constant *C = dyn_cast<Constant>(Op)) {
10245 Operands.push_back(C);
10246 continue;
10247 }
10248
10249 // If any of the operands is non-constant and if they are
10250 // non-integer and non-pointer, don't even try to analyze them
10251 // with scev techniques.
10252 if (!isSCEVable(Op->getType()))
10253 return V;
10254
10255 const SCEV *OrigV = getSCEV(Op);
10256 const SCEV *OpV = getSCEVAtScope(OrigV, L);
10257 MadeImprovement |= OrigV != OpV;
10258
10260 if (!C)
10261 return V;
10262 assert(C->getType() == Op->getType() && "Type mismatch");
10263 Operands.push_back(C);
10264 }
10265
10266 // Check to see if getSCEVAtScope actually made an improvement.
10267 if (!MadeImprovement)
10268 return V; // This is some other type of SCEVUnknown, just return it.
10269
10270 Constant *C = nullptr;
10271 const DataLayout &DL = getDataLayout();
10272 C = ConstantFoldInstOperands(I, Operands, DL, &TLI,
10273 /*AllowNonDeterministic=*/false);
10274 if (!C)
10275 return V;
10276 return getSCEV(C);
10277 }
10278 case scCouldNotCompute:
10279 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10280 }
10281 llvm_unreachable("Unknown SCEV type!");
10282}
10283
10285 return getSCEVAtScope(getSCEV(V), L);
10286}
10287
10288const SCEV *ScalarEvolution::stripInjectiveFunctions(const SCEV *S) const {
10290 return stripInjectiveFunctions(ZExt->getOperand());
10292 return stripInjectiveFunctions(SExt->getOperand());
10293 return S;
10294}
10295
10296/// Finds the minimum unsigned root of the following equation:
10297///
10298/// A * X = B (mod N)
10299///
10300/// where N = 2^BW and BW is the common bit width of A and B. The signedness of
10301/// A and B isn't important.
10302///
10303/// If the equation does not have a solution, SCEVCouldNotCompute is returned.
10304static const SCEV *
10307 ScalarEvolution &SE, const Loop *L) {
10308 uint32_t BW = A.getBitWidth();
10309 assert(BW == SE.getTypeSizeInBits(B->getType()));
10310 assert(A != 0 && "A must be non-zero.");
10311
10312 // 1. D = gcd(A, N)
10313 //
10314 // The gcd of A and N may have only one prime factor: 2. The number of
10315 // trailing zeros in A is its multiplicity
10316 uint32_t Mult2 = A.countr_zero();
10317 // D = 2^Mult2
10318
10319 // 2. Check if B is divisible by D.
10320 //
10321 // B is divisible by D if and only if the multiplicity of prime factor 2 for B
10322 // is not less than multiplicity of this prime factor for D.
10323 unsigned MinTZ = SE.getMinTrailingZeros(B);
10324 // Try again with the terminator of the loop predecessor for context-specific
10325 // result, if MinTZ s too small.
10326 if (MinTZ < Mult2 && L->getLoopPredecessor())
10327 MinTZ = SE.getMinTrailingZeros(B, L->getLoopPredecessor()->getTerminator());
10328 if (MinTZ < Mult2) {
10329 // Check if we can prove there's no remainder using URem.
10330 const SCEV *URem =
10331 SE.getURemExpr(B, SE.getConstant(APInt::getOneBitSet(BW, Mult2)));
10332 const SCEV *Zero = SE.getZero(B->getType());
10333 if (!SE.isKnownPredicate(CmpInst::ICMP_EQ, URem, Zero)) {
10334 // Try to add a predicate ensuring B is a multiple of 1 << Mult2.
10335 if (!Predicates)
10336 return SE.getCouldNotCompute();
10337
10338 // Avoid adding a predicate that is known to be false.
10339 if (SE.isKnownPredicate(CmpInst::ICMP_NE, URem, Zero))
10340 return SE.getCouldNotCompute();
10341 Predicates->push_back(SE.getEqualPredicate(URem, Zero));
10342 }
10343 }
10344
10345 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
10346 // modulo (N / D).
10347 //
10348 // If D == 1, (N / D) == N == 2^BW, so we need one extra bit to represent
10349 // (N / D) in general. The inverse itself always fits into BW bits, though,
10350 // so we immediately truncate it.
10351 APInt AD = A.lshr(Mult2).trunc(BW - Mult2); // AD = A / D
10352 APInt I = AD.multiplicativeInverse().zext(BW);
10353
10354 // 4. Compute the minimum unsigned root of the equation:
10355 // I * (B / D) mod (N / D)
10356 // To simplify the computation, we factor out the divide by D:
10357 // (I * B mod N) / D
10358 const SCEV *D = SE.getConstant(APInt::getOneBitSet(BW, Mult2));
10359 return SE.getUDivExactExpr(SE.getMulExpr(B, SE.getConstant(I)), D);
10360}
10361
10362/// For a given quadratic addrec, generate coefficients of the corresponding
10363/// quadratic equation, multiplied by a common value to ensure that they are
10364/// integers.
10365/// The returned value is a tuple { A, B, C, M, BitWidth }, where
10366/// Ax^2 + Bx + C is the quadratic function, M is the value that A, B and C
10367/// were multiplied by, and BitWidth is the bit width of the original addrec
10368/// coefficients.
10369/// This function returns std::nullopt if the addrec coefficients are not
10370/// compile- time constants.
10371static std::optional<std::tuple<APInt, APInt, APInt, APInt, unsigned>>
10373 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!");
10374 const SCEVConstant *LC = dyn_cast<SCEVConstant>(AddRec->getOperand(0));
10375 const SCEVConstant *MC = dyn_cast<SCEVConstant>(AddRec->getOperand(1));
10376 const SCEVConstant *NC = dyn_cast<SCEVConstant>(AddRec->getOperand(2));
10377 LLVM_DEBUG(dbgs() << __func__ << ": analyzing quadratic addrec: "
10378 << *AddRec << '\n');
10379
10380 // We currently can only solve this if the coefficients are constants.
10381 if (!LC || !MC || !NC) {
10382 LLVM_DEBUG(dbgs() << __func__ << ": coefficients are not constant\n");
10383 return std::nullopt;
10384 }
10385
10386 APInt L = LC->getAPInt();
10387 APInt M = MC->getAPInt();
10388 APInt N = NC->getAPInt();
10389 assert(!N.isZero() && "This is not a quadratic addrec");
10390
10391 unsigned BitWidth = LC->getAPInt().getBitWidth();
10392 unsigned NewWidth = BitWidth + 1;
10393 LLVM_DEBUG(dbgs() << __func__ << ": addrec coeff bw: "
10394 << BitWidth << '\n');
10395 // The sign-extension (as opposed to a zero-extension) here matches the
10396 // extension used in SolveQuadraticEquationWrap (with the same motivation).
10397 N = N.sext(NewWidth);
10398 M = M.sext(NewWidth);
10399 L = L.sext(NewWidth);
10400
10401 // The increments are M, M+N, M+2N, ..., so the accumulated values are
10402 // L+M, (L+M)+(M+N), (L+M)+(M+N)+(M+2N), ..., that is,
10403 // L+M, L+2M+N, L+3M+3N, ...
10404 // After n iterations the accumulated value Acc is L + nM + n(n-1)/2 N.
10405 //
10406 // The equation Acc = 0 is then
10407 // L + nM + n(n-1)/2 N = 0, or 2L + 2M n + n(n-1) N = 0.
10408 // In a quadratic form it becomes:
10409 // N n^2 + (2M-N) n + 2L = 0.
10410
10411 APInt A = N;
10412 APInt B = 2 * M - A;
10413 APInt C = 2 * L;
10414 APInt T = APInt(NewWidth, 2);
10415 LLVM_DEBUG(dbgs() << __func__ << ": equation " << A << "x^2 + " << B
10416 << "x + " << C << ", coeff bw: " << NewWidth
10417 << ", multiplied by " << T << '\n');
10418 return std::make_tuple(A, B, C, T, BitWidth);
10419}
10420
10421/// Helper function to compare optional APInts:
10422/// (a) if X and Y both exist, return min(X, Y),
10423/// (b) if neither X nor Y exist, return std::nullopt,
10424/// (c) if exactly one of X and Y exists, return that value.
10425static std::optional<APInt> MinOptional(std::optional<APInt> X,
10426 std::optional<APInt> Y) {
10427 if (X && Y) {
10428 unsigned W = std::max(X->getBitWidth(), Y->getBitWidth());
10429 APInt XW = X->sext(W);
10430 APInt YW = Y->sext(W);
10431 return XW.slt(YW) ? *X : *Y;
10432 }
10433 if (!X && !Y)
10434 return std::nullopt;
10435 return X ? *X : *Y;
10436}
10437
10438/// Helper function to truncate an optional APInt to a given BitWidth.
10439/// When solving addrec-related equations, it is preferable to return a value
10440/// that has the same bit width as the original addrec's coefficients. If the
10441/// solution fits in the original bit width, truncate it (except for i1).
10442/// Returning a value of a different bit width may inhibit some optimizations.
10443///
10444/// In general, a solution to a quadratic equation generated from an addrec
10445/// may require BW+1 bits, where BW is the bit width of the addrec's
10446/// coefficients. The reason is that the coefficients of the quadratic
10447/// equation are BW+1 bits wide (to avoid truncation when converting from
10448/// the addrec to the equation).
10449static std::optional<APInt> TruncIfPossible(std::optional<APInt> X,
10450 unsigned BitWidth) {
10451 if (!X)
10452 return std::nullopt;
10453 unsigned W = X->getBitWidth();
10455 return X->trunc(BitWidth);
10456 return X;
10457}
10458
10459/// Let c(n) be the value of the quadratic chrec {L,+,M,+,N} after n
10460/// iterations. The values L, M, N are assumed to be signed, and they
10461/// should all have the same bit widths.
10462/// Find the least n >= 0 such that c(n) = 0 in the arithmetic modulo 2^BW,
10463/// where BW is the bit width of the addrec's coefficients.
10464/// If the calculated value is a BW-bit integer (for BW > 1), it will be
10465/// returned as such, otherwise the bit width of the returned value may
10466/// be greater than BW.
10467///
10468/// This function returns std::nullopt if
10469/// (a) the addrec coefficients are not constant, or
10470/// (b) SolveQuadraticEquationWrap was unable to find a solution. For cases
10471/// like x^2 = 5, no integer solutions exist, in other cases an integer
10472/// solution may exist, but SolveQuadraticEquationWrap may fail to find it.
10473static std::optional<APInt>
10475 APInt A, B, C, M;
10476 unsigned BitWidth;
10477 auto T = GetQuadraticEquation(AddRec);
10478 if (!T)
10479 return std::nullopt;
10480
10481 std::tie(A, B, C, M, BitWidth) = *T;
10482 LLVM_DEBUG(dbgs() << __func__ << ": solving for unsigned overflow\n");
10483 std::optional<APInt> X =
10485 if (!X)
10486 return std::nullopt;
10487
10488 ConstantInt *CX = ConstantInt::get(SE.getContext(), *X);
10490 if (!V->isZero())
10491 return std::nullopt;
10492
10493 return TruncIfPossible(X, BitWidth);
10494}
10495
10496/// Let c(n) be the value of the quadratic chrec {0,+,M,+,N} after n
10497/// iterations. The values M, N are assumed to be signed, and they
10498/// should all have the same bit widths.
10499/// Find the least n such that c(n) does not belong to the given range,
10500/// while c(n-1) does.
10501///
10502/// This function returns std::nullopt if
10503/// (a) the addrec coefficients are not constant, or
10504/// (b) SolveQuadraticEquationWrap was unable to find a solution for the
10505/// bounds of the range.
10506static std::optional<APInt>
10508 const ConstantRange &Range, ScalarEvolution &SE) {
10509 assert(AddRec->getOperand(0)->isZero() &&
10510 "Starting value of addrec should be 0");
10511 LLVM_DEBUG(dbgs() << __func__ << ": solving boundary crossing for range "
10512 << Range << ", addrec " << *AddRec << '\n');
10513 // This case is handled in getNumIterationsInRange. Here we can assume that
10514 // we start in the range.
10515 assert(Range.contains(APInt(SE.getTypeSizeInBits(AddRec->getType()), 0)) &&
10516 "Addrec's initial value should be in range");
10517
10518 APInt A, B, C, M;
10519 unsigned BitWidth;
10520 auto T = GetQuadraticEquation(AddRec);
10521 if (!T)
10522 return std::nullopt;
10523
10524 // Be careful about the return value: there can be two reasons for not
10525 // returning an actual number. First, if no solutions to the equations
10526 // were found, and second, if the solutions don't leave the given range.
10527 // The first case means that the actual solution is "unknown", the second
10528 // means that it's known, but not valid. If the solution is unknown, we
10529 // cannot make any conclusions.
10530 // Return a pair: the optional solution and a flag indicating if the
10531 // solution was found.
10532 auto SolveForBoundary =
10533 [&](APInt Bound) -> std::pair<std::optional<APInt>, bool> {
10534 // Solve for signed overflow and unsigned overflow, pick the lower
10535 // solution.
10536 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: checking boundary "
10537 << Bound << " (before multiplying by " << M << ")\n");
10538 Bound *= M; // The quadratic equation multiplier.
10539
10540 std::optional<APInt> SO;
10541 if (BitWidth > 1) {
10542 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10543 "signed overflow\n");
10545 }
10546 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10547 "unsigned overflow\n");
10548 std::optional<APInt> UO =
10550
10551 auto LeavesRange = [&] (const APInt &X) {
10552 ConstantInt *C0 = ConstantInt::get(SE.getContext(), X);
10554 if (Range.contains(V0->getValue()))
10555 return false;
10556 // X should be at least 1, so X-1 is non-negative.
10557 ConstantInt *C1 = ConstantInt::get(SE.getContext(), X-1);
10559 if (Range.contains(V1->getValue()))
10560 return true;
10561 return false;
10562 };
10563
10564 // If SolveQuadraticEquationWrap returns std::nullopt, it means that there
10565 // can be a solution, but the function failed to find it. We cannot treat it
10566 // as "no solution".
10567 if (!SO || !UO)
10568 return {std::nullopt, false};
10569
10570 // Check the smaller value first to see if it leaves the range.
10571 // At this point, both SO and UO must have values.
10572 std::optional<APInt> Min = MinOptional(SO, UO);
10573 if (LeavesRange(*Min))
10574 return { Min, true };
10575 std::optional<APInt> Max = Min == SO ? UO : SO;
10576 if (LeavesRange(*Max))
10577 return { Max, true };
10578
10579 // Solutions were found, but were eliminated, hence the "true".
10580 return {std::nullopt, true};
10581 };
10582
10583 std::tie(A, B, C, M, BitWidth) = *T;
10584 // Lower bound is inclusive, subtract 1 to represent the exiting value.
10585 APInt Lower = Range.getLower().sext(A.getBitWidth()) - 1;
10586 APInt Upper = Range.getUpper().sext(A.getBitWidth());
10587 auto SL = SolveForBoundary(Lower);
10588 auto SU = SolveForBoundary(Upper);
10589 // If any of the solutions was unknown, no meaninigful conclusions can
10590 // be made.
10591 if (!SL.second || !SU.second)
10592 return std::nullopt;
10593
10594 // Claim: The correct solution is not some value between Min and Max.
10595 //
10596 // Justification: Assuming that Min and Max are different values, one of
10597 // them is when the first signed overflow happens, the other is when the
10598 // first unsigned overflow happens. Crossing the range boundary is only
10599 // possible via an overflow (treating 0 as a special case of it, modeling
10600 // an overflow as crossing k*2^W for some k).
10601 //
10602 // The interesting case here is when Min was eliminated as an invalid
10603 // solution, but Max was not. The argument is that if there was another
10604 // overflow between Min and Max, it would also have been eliminated if
10605 // it was considered.
10606 //
10607 // For a given boundary, it is possible to have two overflows of the same
10608 // type (signed/unsigned) without having the other type in between: this
10609 // can happen when the vertex of the parabola is between the iterations
10610 // corresponding to the overflows. This is only possible when the two
10611 // overflows cross k*2^W for the same k. In such case, if the second one
10612 // left the range (and was the first one to do so), the first overflow
10613 // would have to enter the range, which would mean that either we had left
10614 // the range before or that we started outside of it. Both of these cases
10615 // are contradictions.
10616 //
10617 // Claim: In the case where SolveForBoundary returns std::nullopt, the correct
10618 // solution is not some value between the Max for this boundary and the
10619 // Min of the other boundary.
10620 //
10621 // Justification: Assume that we had such Max_A and Min_B corresponding
10622 // to range boundaries A and B and such that Max_A < Min_B. If there was
10623 // a solution between Max_A and Min_B, it would have to be caused by an
10624 // overflow corresponding to either A or B. It cannot correspond to B,
10625 // since Min_B is the first occurrence of such an overflow. If it
10626 // corresponded to A, it would have to be either a signed or an unsigned
10627 // overflow that is larger than both eliminated overflows for A. But
10628 // between the eliminated overflows and this overflow, the values would
10629 // cover the entire value space, thus crossing the other boundary, which
10630 // is a contradiction.
10631
10632 return TruncIfPossible(MinOptional(SL.first, SU.first), BitWidth);
10633}
10634
10635ScalarEvolution::ExitLimit ScalarEvolution::howFarToZero(const SCEV *V,
10636 const Loop *L,
10637 bool ControlsOnlyExit,
10638 bool AllowPredicates) {
10639
10640 // This is only used for loops with a "x != y" exit test. The exit condition
10641 // is now expressed as a single expression, V = x-y. So the exit test is
10642 // effectively V != 0. We know and take advantage of the fact that this
10643 // expression only being used in a comparison by zero context.
10644
10646 // If the value is a constant
10647 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
10648 // If the value is already zero, the branch will execute zero times.
10649 if (C->getValue()->isZero()) return C;
10650 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10651 }
10652
10653 const SCEVAddRecExpr *AddRec =
10654 dyn_cast<SCEVAddRecExpr>(stripInjectiveFunctions(V));
10655
10656 if (!AddRec && AllowPredicates)
10657 // Try to make this an AddRec using runtime tests, in the first X
10658 // iterations of this loop, where X is the SCEV expression found by the
10659 // algorithm below.
10660 AddRec = convertSCEVToAddRecWithPredicates(V, L, Predicates);
10661
10662 if (!AddRec || AddRec->getLoop() != L)
10663 return getCouldNotCompute();
10664
10665 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
10666 // the quadratic equation to solve it.
10667 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
10668 // We can only use this value if the chrec ends up with an exact zero
10669 // value at this index. When solving for "X*X != 5", for example, we
10670 // should not accept a root of 2.
10671 if (auto S = SolveQuadraticAddRecExact(AddRec, *this)) {
10672 const auto *R = cast<SCEVConstant>(getConstant(*S));
10673 return ExitLimit(R, R, R, false, Predicates);
10674 }
10675 return getCouldNotCompute();
10676 }
10677
10678 // Otherwise we can only handle this if it is affine.
10679 if (!AddRec->isAffine())
10680 return getCouldNotCompute();
10681
10682 // If this is an affine expression, the execution count of this branch is
10683 // the minimum unsigned root of the following equation:
10684 //
10685 // Start + Step*N = 0 (mod 2^BW)
10686 //
10687 // equivalent to:
10688 //
10689 // Step*N = -Start (mod 2^BW)
10690 //
10691 // where BW is the common bit width of Start and Step.
10692
10693 // Get the initial value for the loop.
10694 const SCEV *Start = getSCEVAtScope(AddRec->getStart(), L->getParentLoop());
10695 const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1), L->getParentLoop());
10696
10697 if (!isLoopInvariant(Step, L))
10698 return getCouldNotCompute();
10699
10700 LoopGuards Guards = LoopGuards::collectOnDemand(L, *this);
10701 // Specialize step for this loop so we get context sensitive facts below.
10702 const SCEV *StepWLG = applyLoopGuards(Step, Guards);
10703
10704 // For positive steps (counting up until unsigned overflow):
10705 // N = -Start/Step (as unsigned)
10706 // For negative steps (counting down to zero):
10707 // N = Start/-Step
10708 // First compute the unsigned distance from zero in the direction of Step.
10709 bool CountDown = isKnownNegative(StepWLG);
10710 if (!CountDown && !isKnownNonNegative(StepWLG))
10711 return getCouldNotCompute();
10712
10713 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(Start);
10714 // Handle unitary steps, which cannot wraparound.
10715 // 1*N = -Start; -1*N = Start (mod 2^BW), so:
10716 // N = Distance (as unsigned)
10717
10718 if (match(Step, m_CombineOr(m_scev_One(), m_scev_AllOnes()))) {
10719 APInt MaxBECount = getUnsignedRangeMax(applyLoopGuards(Distance, Guards));
10720 MaxBECount = APIntOps::umin(MaxBECount, getUnsignedRangeMax(Distance));
10721
10722 // When a loop like "for (int i = 0; i != n; ++i) { /* body */ }" is rotated,
10723 // we end up with a loop whose backedge-taken count is n - 1. Detect this
10724 // case, and see if we can improve the bound.
10725 //
10726 // Explicitly handling this here is necessary because getUnsignedRange
10727 // isn't context-sensitive; it doesn't know that we only care about the
10728 // range inside the loop.
10729 const SCEV *Zero = getZero(Distance->getType());
10730 const SCEV *One = getOne(Distance->getType());
10731 const SCEV *DistancePlusOne = getAddExpr(Distance, One);
10732 if (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_NE, DistancePlusOne, Zero)) {
10733 // If Distance + 1 doesn't overflow, we can compute the maximum distance
10734 // as "unsigned_max(Distance + 1) - 1". Also apply the loop guards to
10735 // Distance + 1; the range of Distance itself may be a wrapped set even
10736 // when the guards bound Distance + 1 tightly.
10737 APInt Max = APIntOps::umin(
10738 getUnsignedRangeMax(applyLoopGuards(DistancePlusOne, Guards)),
10739 getUnsignedRangeMax(DistancePlusOne));
10740 MaxBECount = APIntOps::umin(MaxBECount, Max - 1);
10741 }
10742 return ExitLimit(Distance, getConstant(MaxBECount), Distance, false,
10743 Predicates);
10744 }
10745
10746 // If the condition controls loop exit (the loop exits only if the expression
10747 // is true) and the addition is no-wrap we can use unsigned divide to
10748 // compute the backedge count. In this case, the step may not divide the
10749 // distance, but we don't care because if the condition is "missed" the loop
10750 // will have undefined behavior due to wrapping.
10751 if (ControlsOnlyExit && AddRec->hasNoSelfWrap() &&
10752 loopHasNoAbnormalExits(AddRec->getLoop())) {
10753
10754 // If the stride is zero and the start is non-zero, the loop must be
10755 // infinite. In C++, most loops are finite by assumption, in which case the
10756 // step being zero implies UB must execute if the loop is entered.
10757 if (!(loopIsFiniteByAssumption(L) && isKnownNonZero(Start)) &&
10758 !isKnownNonZero(StepWLG))
10759 return getCouldNotCompute();
10760
10761 const SCEV *Exact =
10762 getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step);
10763 const SCEV *ConstantMax = getCouldNotCompute();
10764 if (Exact != getCouldNotCompute()) {
10765 APInt MaxInt = getUnsignedRangeMax(applyLoopGuards(Exact, Guards));
10766 ConstantMax =
10768 }
10769 const SCEV *SymbolicMax =
10770 isa<SCEVCouldNotCompute>(Exact) ? ConstantMax : Exact;
10771 return ExitLimit(Exact, ConstantMax, SymbolicMax, false, Predicates);
10772 }
10773
10774 // Solve the general equation.
10775 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Step);
10776 if (!StepC || StepC->getValue()->isZero())
10777 return getCouldNotCompute();
10778 const SCEV *E = SolveLinEquationWithOverflow(
10779 StepC->getAPInt(), getNegativeSCEV(Start),
10780 AllowPredicates ? &Predicates : nullptr, *this, L);
10781
10782 const SCEV *M = E;
10783 if (E != getCouldNotCompute()) {
10784 APInt MaxWithGuards = getUnsignedRangeMax(applyLoopGuards(E, Guards));
10785 M = getConstant(APIntOps::umin(MaxWithGuards, getUnsignedRangeMax(E)));
10786 }
10787 auto *S = isa<SCEVCouldNotCompute>(E) ? M : E;
10788 return ExitLimit(E, M, S, false, Predicates);
10789}
10790
10791ScalarEvolution::ExitLimit
10792ScalarEvolution::howFarToNonZero(const SCEV *V, const Loop *L) {
10793 // Loops that look like: while (X == 0) are very strange indeed. We don't
10794 // handle them yet except for the trivial case. This could be expanded in the
10795 // future as needed.
10796
10797 // If the value is a constant, check to see if it is known to be non-zero
10798 // already. If so, the backedge will execute zero times.
10799 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
10800 if (!C->getValue()->isZero())
10801 return getZero(C->getType());
10802 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10803 }
10804
10805 // We could implement others, but I really doubt anyone writes loops like
10806 // this, and if they did, they would already be constant folded.
10807 return getCouldNotCompute();
10808}
10809
10810std::pair<const BasicBlock *, const BasicBlock *>
10811ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(const BasicBlock *BB)
10812 const {
10813 // If the block has a unique predecessor, then there is no path from the
10814 // predecessor to the block that does not go through the direct edge
10815 // from the predecessor to the block.
10816 if (const BasicBlock *Pred = BB->getSinglePredecessor())
10817 return {Pred, BB};
10818
10819 // A loop's header is defined to be a block that dominates the loop.
10820 // If the header has a unique predecessor outside the loop, it must be
10821 // a block that has exactly one successor that can reach the loop.
10822 if (const Loop *L = LI.getLoopFor(BB))
10823 return {L->getLoopPredecessor(), L->getHeader()};
10824
10825 return {nullptr, BB};
10826}
10827
10828/// SCEV structural equivalence is usually sufficient for testing whether two
10829/// expressions are equal, however for the purposes of looking for a condition
10830/// guarding a loop, it can be useful to be a little more general, since a
10831/// front-end may have replicated the controlling expression.
10832static bool HasSameValue(const SCEV *A, const SCEV *B) {
10833 // Quick check to see if they are the same SCEV.
10834 if (A == B) return true;
10835
10836 auto ComputesEqualValues = [](const Instruction *A, const Instruction *B) {
10837 // Not all instructions that are "identical" compute the same value. For
10838 // instance, two distinct alloca instructions allocating the same type are
10839 // identical and do not read memory; but compute distinct values.
10840 return A->isIdenticalTo(B) && (isa<BinaryOperator>(A) || isa<GetElementPtrInst>(A));
10841 };
10842
10843 // Otherwise, if they're both SCEVUnknown, it's possible that they hold
10844 // two different instructions with the same value. Check for this case.
10845 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(A))
10846 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(B))
10847 if (const Instruction *AI = dyn_cast<Instruction>(AU->getValue()))
10848 if (const Instruction *BI = dyn_cast<Instruction>(BU->getValue()))
10849 if (ComputesEqualValues(AI, BI))
10850 return true;
10851
10852 // Otherwise assume they may have a different value.
10853 return false;
10854}
10855
10856static bool MatchBinarySub(const SCEV *S, SCEVUse &LHS, SCEVUse &RHS) {
10857 const SCEV *Op0, *Op1;
10858 if (!match(S, m_scev_Add(m_SCEV(Op0), m_SCEV(Op1))))
10859 return false;
10860 if (match(Op0, m_scev_Mul(m_scev_AllOnes(), m_SCEV(RHS)))) {
10861 LHS = Op1;
10862 return true;
10863 }
10864 if (match(Op1, m_scev_Mul(m_scev_AllOnes(), m_SCEV(RHS)))) {
10865 LHS = Op0;
10866 return true;
10867 }
10868 return false;
10869}
10870
10872 SCEVUse &RHS, unsigned Depth) {
10873 bool Changed = false;
10874 // Simplifies ICMP to trivial true or false by turning it into '0 == 0' or
10875 // '0 != 0'.
10876 auto TrivialCase = [&](bool TriviallyTrue) {
10878 Pred = TriviallyTrue ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
10879 return true;
10880 };
10881 // If we hit the max recursion limit bail out.
10882 if (Depth >= 3)
10883 return false;
10884
10885 const SCEV *NewLHS, *NewRHS;
10886 if (match(LHS, m_scev_c_Mul(m_SCEV(NewLHS), m_SCEVVScale())) &&
10887 match(RHS, m_scev_c_Mul(m_SCEV(NewRHS), m_SCEVVScale()))) {
10888 const SCEVMulExpr *LMul = cast<SCEVMulExpr>(LHS);
10889 const SCEVMulExpr *RMul = cast<SCEVMulExpr>(RHS);
10890
10891 // (X * vscale) pred (Y * vscale) ==> X pred Y
10892 // when both multiples are NSW.
10893 // (X * vscale) uicmp/eq/ne (Y * vscale) ==> X uicmp/eq/ne Y
10894 // when both multiples are NUW.
10895 if ((LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap()) ||
10896 (LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap() &&
10897 !ICmpInst::isSigned(Pred))) {
10898 LHS = NewLHS;
10899 RHS = NewRHS;
10900 Changed = true;
10901 }
10902 }
10903
10904 // Canonicalize a constant to the right side.
10905 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
10906 // Check for both operands constant.
10907 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
10908 if (!ICmpInst::compare(LHSC->getAPInt(), RHSC->getAPInt(), Pred))
10909 return TrivialCase(false);
10910 return TrivialCase(true);
10911 }
10912 // Otherwise swap the operands to put the constant on the right.
10913 std::swap(LHS, RHS);
10915 Changed = true;
10916 }
10917
10918 // (K + A) pred (K + B) --> A pred B
10919 // For equality, no flags are needed.
10920 // For signed, both adds must be NSW. For unsigned, both must be NUW.
10921 {
10922 const SCEVConstant *C = nullptr;
10923 if (match(LHS, m_scev_Add(m_SCEVConstant(C), m_SCEV(NewLHS))) &&
10924 match(RHS, m_scev_Add(m_scev_Specific(C), m_SCEV(NewRHS)))) {
10925 const auto *LAdd = cast<SCEVAddExpr>(LHS);
10926 const auto *RAdd = cast<SCEVAddExpr>(RHS);
10927 if (ICmpInst::isEquality(Pred) ||
10928 (ICmpInst::isSigned(Pred) && LAdd->hasNoSignedWrap() &&
10929 RAdd->hasNoSignedWrap()) ||
10930 (ICmpInst::isUnsigned(Pred) && LAdd->hasNoUnsignedWrap() &&
10931 RAdd->hasNoUnsignedWrap())) {
10932 LHS = NewLHS;
10933 RHS = NewRHS;
10934 Changed = true;
10935 }
10936 }
10937 }
10938
10939 // (C * A) pred (C * B) --> A pred B
10940 // For equality predicates, both muls must be NUW or both must be NSW
10941 // (either suffices to make multiplication by C injective; C == 0 is
10942 // impossible because SCEV folds 0 * X to 0).
10943 // For signed ordering, C must be positive and both muls must be NSW.
10944 // For unsigned ordering, both muls must be NUW.
10945 {
10946 const SCEVConstant *C = nullptr;
10947 if (match(LHS, m_scev_Mul(m_SCEVConstant(C), m_SCEV(NewLHS))) &&
10948 match(RHS, m_scev_Mul(m_scev_Specific(C), m_SCEV(NewRHS)))) {
10949 const auto *LMul = cast<SCEVMulExpr>(LHS);
10950 const auto *RMul = cast<SCEVMulExpr>(RHS);
10951 bool BothNUW = LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap();
10952 bool BothNSW = LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap();
10953 if ((ICmpInst::isEquality(Pred) && (BothNUW || BothNSW)) ||
10954 (ICmpInst::isSigned(Pred) && BothNSW &&
10955 C->getAPInt().isStrictlyPositive()) ||
10956 (ICmpInst::isUnsigned(Pred) && BothNUW)) {
10957 LHS = NewLHS;
10958 RHS = NewRHS;
10959 Changed = true;
10960 }
10961 }
10962 }
10963
10964 // If we're comparing an addrec with a value which is loop-invariant in the
10965 // addrec's loop, put the addrec on the left. Also make a dominance check,
10966 // as both operands could be addrecs loop-invariant in each other's loop.
10967 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) {
10968 const Loop *L = AR->getLoop();
10969 if (isLoopInvariant(LHS, L) && properlyDominates(LHS, L->getHeader())) {
10970 std::swap(LHS, RHS);
10972 Changed = true;
10973 }
10974 }
10975
10976 // If there's a constant operand, canonicalize comparisons with boundary
10977 // cases, and canonicalize *-or-equal comparisons to regular comparisons.
10978 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS)) {
10979 const APInt &RA = RC->getAPInt();
10980
10981 bool SimplifiedByConstantRange = false;
10982
10983 if (!ICmpInst::isEquality(Pred)) {
10985 if (ExactCR.isFullSet())
10986 return TrivialCase(true);
10987 if (ExactCR.isEmptySet())
10988 return TrivialCase(false);
10989
10990 APInt NewRHS;
10991 CmpInst::Predicate NewPred;
10992 if (ExactCR.getEquivalentICmp(NewPred, NewRHS) &&
10993 ICmpInst::isEquality(NewPred)) {
10994 // We were able to convert an inequality to an equality.
10995 Pred = NewPred;
10996 RHS = getConstant(NewRHS);
10997 Changed = SimplifiedByConstantRange = true;
10998 }
10999 }
11000
11001 if (!SimplifiedByConstantRange) {
11002 switch (Pred) {
11003 default:
11004 break;
11005 case ICmpInst::ICMP_EQ:
11006 case ICmpInst::ICMP_NE:
11007 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b.
11008 if (RA.isZero() && MatchBinarySub(LHS, LHS, RHS))
11009 Changed = true;
11010 break;
11011
11012 // The "Should have been caught earlier!" messages refer to the fact
11013 // that the ExactCR.isFullSet() or ExactCR.isEmptySet() check above
11014 // should have fired on the corresponding cases, and canonicalized the
11015 // check to trivial case.
11016
11017 case ICmpInst::ICMP_UGE:
11018 assert(!RA.isMinValue() && "Should have been caught earlier!");
11019 Pred = ICmpInst::ICMP_UGT;
11020 RHS = getConstant(RA - 1);
11021 Changed = true;
11022 break;
11023 case ICmpInst::ICMP_ULE:
11024 assert(!RA.isMaxValue() && "Should have been caught earlier!");
11025 Pred = ICmpInst::ICMP_ULT;
11026 RHS = getConstant(RA + 1);
11027 Changed = true;
11028 break;
11029 case ICmpInst::ICMP_SGE:
11030 assert(!RA.isMinSignedValue() && "Should have been caught earlier!");
11031 Pred = ICmpInst::ICMP_SGT;
11032 RHS = getConstant(RA - 1);
11033 Changed = true;
11034 break;
11035 case ICmpInst::ICMP_SLE:
11036 assert(!RA.isMaxSignedValue() && "Should have been caught earlier!");
11037 Pred = ICmpInst::ICMP_SLT;
11038 RHS = getConstant(RA + 1);
11039 Changed = true;
11040 break;
11041 }
11042 }
11043 }
11044
11045 // a /u b == 0 => a < b
11046 // a /u b != 0 => a >= b
11047 if (ICmpInst::isEquality(Pred) && RHS->isZero() &&
11048 match(LHS, m_scev_UDiv(m_SCEV(LHS), m_SCEV(RHS)))) {
11050 Changed = true;
11051 }
11052
11053 // Check for obvious equality.
11054 if (HasSameValue(LHS, RHS)) {
11055 if (ICmpInst::isTrueWhenEqual(Pred))
11056 return TrivialCase(true);
11058 return TrivialCase(false);
11059 }
11060
11061 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
11062 // adding or subtracting 1 from one of the operands.
11063 switch (Pred) {
11064 case ICmpInst::ICMP_SLE:
11065 if (!getSignedRangeMax(RHS).isMaxSignedValue()) {
11066 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
11068 Pred = ICmpInst::ICMP_SLT;
11069 Changed = true;
11070 } else if (!getSignedRangeMin(LHS).isMinSignedValue()) {
11071 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS,
11073 Pred = ICmpInst::ICMP_SLT;
11074 Changed = true;
11075 }
11076 break;
11077 case ICmpInst::ICMP_SGE:
11078 if (!getSignedRangeMin(RHS).isMinSignedValue()) {
11079 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS,
11081 Pred = ICmpInst::ICMP_SGT;
11082 Changed = true;
11083 } else if (!getSignedRangeMax(LHS).isMaxSignedValue()) {
11084 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
11086 Pred = ICmpInst::ICMP_SGT;
11087 Changed = true;
11088 }
11089 break;
11090 case ICmpInst::ICMP_ULE:
11091 if (!getUnsignedRangeMax(RHS).isMaxValue()) {
11092 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
11094 Pred = ICmpInst::ICMP_ULT;
11095 Changed = true;
11096 } else if (!getUnsignedRangeMin(LHS).isMinValue()) {
11097 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS);
11098 Pred = ICmpInst::ICMP_ULT;
11099 Changed = true;
11100 }
11101 break;
11102 case ICmpInst::ICMP_UGE:
11103 // If RHS is an op we can fold the -1, try that first.
11104 // Otherwise prefer LHS to preserve the nuw flag.
11105 if ((isa<SCEVConstant>(RHS) ||
11107 isa<SCEVConstant>(cast<SCEVNAryExpr>(RHS)->getOperand(0)))) &&
11108 !getUnsignedRangeMin(RHS).isMinValue()) {
11109 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS);
11110 Pred = ICmpInst::ICMP_UGT;
11111 Changed = true;
11112 } else if (!getUnsignedRangeMax(LHS).isMaxValue()) {
11113 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
11115 Pred = ICmpInst::ICMP_UGT;
11116 Changed = true;
11117 } else if (!getUnsignedRangeMin(RHS).isMinValue()) {
11118 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS);
11119 Pred = ICmpInst::ICMP_UGT;
11120 Changed = true;
11121 }
11122 break;
11123 default:
11124 break;
11125 }
11126
11127 // TODO: More simplifications are possible here.
11128
11129 // Recursively simplify until we either hit a recursion limit or nothing
11130 // changes.
11131 if (Changed)
11132 (void)SimplifyICmpOperands(Pred, LHS, RHS, Depth + 1);
11133
11134 return Changed;
11135}
11136
11138 return getSignedRangeMax(S).isNegative();
11139}
11140
11144
11146 return !getSignedRangeMin(S).isNegative();
11147}
11148
11152
11154 // Query push down for cases where the unsigned range is
11155 // less than sufficient.
11156 if (const auto *SExt = dyn_cast<SCEVSignExtendExpr>(S))
11157 return isKnownNonZero(SExt->getOperand(0));
11158 return getUnsignedRangeMin(S) != 0;
11159}
11160
11162 bool OrNegative) {
11163 auto NonRecursive = [OrNegative](const SCEV *S) {
11164 if (auto *C = dyn_cast<SCEVConstant>(S))
11165 return C->getAPInt().isPowerOf2() ||
11166 (OrNegative && C->getAPInt().isNegatedPowerOf2());
11167
11168 // vscale is a power-of-two.
11169 return isa<SCEVVScale>(S);
11170 };
11171
11172 if (NonRecursive(S))
11173 return true;
11174
11175 auto *Mul = dyn_cast<SCEVMulExpr>(S);
11176 if (!Mul)
11177 return false;
11178 return all_of(Mul->operands(), NonRecursive) && (OrZero || isKnownNonZero(S));
11179}
11180
11182 const SCEV *S, uint64_t M,
11184 if (M == 0)
11185 return false;
11186 if (M == 1)
11187 return true;
11188
11189 // For a constant, check that "S % M == 0".
11190 if (auto *Cst = dyn_cast<SCEVConstant>(S)) {
11191 APInt C = Cst->getAPInt();
11192 return C.urem(M) == 0;
11193 }
11194
11195 // Basic tests have failed.
11196 // Check "S % M == 0" at compile time and record runtime Assumptions.
11197 auto *STy = dyn_cast<IntegerType>(S->getType());
11198 const SCEV *SmodM =
11199 getURemExpr(S, getConstant(ConstantInt::get(STy, M, false)));
11200 const SCEV *Zero = getZero(STy);
11201
11202 // Check whether "S % M == 0" is known at compile time.
11204 return true;
11205
11206 // Check whether "S % M != 0" is known at compile time.
11208 return false;
11209
11210 if (!Predicates)
11211 return false;
11212
11213 // Look through Add and AddRec expressions with nuw to improve the
11214 // precision of added predicates. S is a multiple of M if S starts with a
11215 // multiple of M and at every iteration step S only adds multiples of M.
11218 all_of(S->operands(),
11219 [&](SCEVUse Op) { return isKnownMultipleOf(Op, M, Predicates); }))
11220 return true;
11221
11222 // Similarly, look through Mul with nuw, where any operand being a
11223 // known-multiple is sufficient.
11224 if (auto *Mul = dyn_cast<SCEVMulExpr>(S))
11225 if (Mul->hasNoUnsignedWrap() && any_of(S->operands(), [&](SCEVUse Op) {
11226 return isKnownMultipleOf(Op, M, Predicates);
11227 }))
11228 return true;
11229
11230 // Similarly, look through MinMax, with no wrapping arithmetic to consider.
11231 if (isa<SCEVMinMaxExpr>(S) && all_of(S->operands(), [&](SCEVUse Op) {
11232 return isKnownMultipleOf(Op, M, Predicates);
11233 }))
11234 return true;
11235
11237
11238 // Detect redundant predicates.
11239 for (auto *A : *Predicates)
11240 if (A->implies(P, *this))
11241 return true;
11242
11243 // Only record non-redundant predicates.
11244 Predicates->push_back(P);
11245 return true;
11246}
11247
11249 return ((isKnownNonNegative(S1) && isKnownNonNegative(S2)) ||
11251}
11252
11253std::pair<const SCEV *, const SCEV *>
11255 // Compute SCEV on entry of loop L.
11256 const SCEV *Start = SCEVInitRewriter::rewrite(S, L, *this);
11257 if (Start == getCouldNotCompute())
11258 return { Start, Start };
11259 // Compute post increment SCEV for loop L.
11260 const SCEV *PostInc = SCEVPostIncRewriter::rewrite(S, L, *this);
11261 assert(PostInc != getCouldNotCompute() && "Unexpected could not compute");
11262 return { Start, PostInc };
11263}
11264
11266 SCEVUse RHS) {
11267 // First collect all loops.
11269 getUsedLoops(LHS, LoopsUsed);
11270 getUsedLoops(RHS, LoopsUsed);
11271
11272 if (LoopsUsed.empty())
11273 return false;
11274
11275 // Domination relationship must be a linear order on collected loops.
11276#ifndef NDEBUG
11277 for (const auto *L1 : LoopsUsed)
11278 for (const auto *L2 : LoopsUsed)
11279 assert((DT.dominates(L1->getHeader(), L2->getHeader()) ||
11280 DT.dominates(L2->getHeader(), L1->getHeader())) &&
11281 "Domination relationship is not a linear order");
11282#endif
11283
11284 const Loop *MDL =
11285 *llvm::max_element(LoopsUsed, [&](const Loop *L1, const Loop *L2) {
11286 return DT.properlyDominates(L1->getHeader(), L2->getHeader());
11287 });
11288
11289 // Get init and post increment value for LHS.
11290 auto SplitLHS = SplitIntoInitAndPostInc(MDL, LHS);
11291 // if LHS contains unknown non-invariant SCEV then bail out.
11292 if (SplitLHS.first == getCouldNotCompute())
11293 return false;
11294 assert (SplitLHS.second != getCouldNotCompute() && "Unexpected CNC");
11295 // Get init and post increment value for RHS.
11296 auto SplitRHS = SplitIntoInitAndPostInc(MDL, RHS);
11297 // if RHS contains unknown non-invariant SCEV then bail out.
11298 if (SplitRHS.first == getCouldNotCompute())
11299 return false;
11300 assert (SplitRHS.second != getCouldNotCompute() && "Unexpected CNC");
11301 // It is possible that init SCEV contains an invariant load but it does
11302 // not dominate MDL and is not available at MDL loop entry, so we should
11303 // check it here.
11304 if (!isAvailableAtLoopEntry(SplitLHS.first, MDL) ||
11305 !isAvailableAtLoopEntry(SplitRHS.first, MDL))
11306 return false;
11307
11308 // It seems backedge guard check is faster than entry one so in some cases
11309 // it can speed up whole estimation by short circuit
11310 return isLoopBackedgeGuardedByCond(MDL, Pred, SplitLHS.second,
11311 SplitRHS.second) &&
11312 isLoopEntryGuardedByCond(MDL, Pred, SplitLHS.first, SplitRHS.first);
11313}
11314
11316 SCEVUse RHS) {
11317 // Canonicalize the inputs first.
11318 (void)SimplifyICmpOperands(Pred, LHS, RHS);
11319
11320 return isKnownViaInduction(Pred, LHS, RHS) ||
11321 isKnownPredicateViaSplitting(Pred, LHS, RHS) ||
11322 isKnownViaNonRecursiveReasoning(Pred, LHS, RHS);
11323}
11324
11326 const SCEV *LHS,
11327 const SCEV *RHS) {
11328 if (isKnownPredicate(Pred, LHS, RHS))
11329 return true;
11331 return false;
11332 return std::nullopt;
11333}
11334
11336 const SCEV *RHS,
11337 const Instruction *CtxI) {
11338 // TODO: Analyze guards and assumes from Context's block.
11339 return isKnownPredicate(Pred, LHS, RHS) ||
11340 isBasicBlockEntryGuardedByCond(CtxI->getParent(), Pred, LHS, RHS);
11341}
11342
11343std::optional<bool>
11345 const SCEV *RHS, const Instruction *CtxI) {
11346 std::optional<bool> KnownWithoutContext = evaluatePredicate(Pred, LHS, RHS);
11347 if (KnownWithoutContext)
11348 return KnownWithoutContext;
11349
11350 if (isBasicBlockEntryGuardedByCond(CtxI->getParent(), Pred, LHS, RHS))
11351 return true;
11353 CtxI->getParent(), ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11354 return false;
11355 return std::nullopt;
11356}
11357
11359 const SCEVAddRecExpr *LHS,
11360 const SCEV *RHS) {
11361 const Loop *L = LHS->getLoop();
11362 return isLoopEntryGuardedByCond(L, Pred, LHS->getStart(), RHS) &&
11363 isLoopBackedgeGuardedByCond(L, Pred, LHS->getPostIncExpr(*this), RHS);
11364}
11365
11366std::optional<ScalarEvolution::MonotonicPredicateType>
11368 ICmpInst::Predicate Pred) {
11369 auto Result = getMonotonicPredicateTypeImpl(LHS, Pred);
11370
11371#ifndef NDEBUG
11372 // Verify an invariant: inverting the predicate should turn a monotonically
11373 // increasing change to a monotonically decreasing one, and vice versa.
11374 if (Result) {
11375 auto ResultSwapped =
11376 getMonotonicPredicateTypeImpl(LHS, ICmpInst::getSwappedPredicate(Pred));
11377
11378 assert(*ResultSwapped != *Result &&
11379 "monotonicity should flip as we flip the predicate");
11380 }
11381#endif
11382
11383 return Result;
11384}
11385
11386std::optional<ScalarEvolution::MonotonicPredicateType>
11387ScalarEvolution::getMonotonicPredicateTypeImpl(const SCEVAddRecExpr *LHS,
11388 ICmpInst::Predicate Pred) {
11389 // A zero step value for LHS means the induction variable is essentially a
11390 // loop invariant value. We don't really depend on the predicate actually
11391 // flipping from false to true (for increasing predicates, and the other way
11392 // around for decreasing predicates), all we care about is that *if* the
11393 // predicate changes then it only changes from false to true.
11394 //
11395 // A zero step value in itself is not very useful, but there may be places
11396 // where SCEV can prove X >= 0 but not prove X > 0, so it is helpful to be
11397 // as general as possible.
11398
11399 // Only handle LE/LT/GE/GT predicates.
11400 if (!ICmpInst::isRelational(Pred))
11401 return std::nullopt;
11402
11403 bool IsGreater = ICmpInst::isGE(Pred) || ICmpInst::isGT(Pred);
11404 assert((IsGreater || ICmpInst::isLE(Pred) || ICmpInst::isLT(Pred)) &&
11405 "Should be greater or less!");
11406
11407 // Check that AR does not wrap.
11408 if (ICmpInst::isUnsigned(Pred)) {
11409 if (!LHS->hasNoUnsignedWrap())
11410 return std::nullopt;
11412 }
11414 "Relational predicate is either signed or unsigned!");
11415 if (!LHS->hasNoSignedWrap())
11416 return std::nullopt;
11417
11418 const SCEV *Step = LHS->getStepRecurrence(*this);
11419
11420 if (isKnownNonNegative(Step))
11422
11423 if (isKnownNonPositive(Step))
11425
11426 return std::nullopt;
11427}
11428
11429std::optional<ScalarEvolution::LoopInvariantPredicate>
11431 const SCEV *RHS, const Loop *L,
11432 const Instruction *CtxI) {
11433 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11434 if (!isLoopInvariant(RHS, L)) {
11435 if (!isLoopInvariant(LHS, L))
11436 return std::nullopt;
11437
11438 std::swap(LHS, RHS);
11440 }
11441
11442 const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(LHS);
11443 if (!ArLHS || ArLHS->getLoop() != L)
11444 return std::nullopt;
11445
11446 // Even if both are valid, we need to consistently choose the unsigned or the
11447 // signed predicate for the monotonicity and backedge checks below, not
11448 // mixtures of both. For now, prefer the unsigned predicate.
11449 Pred = Pred.dropSameSign();
11450 auto MonotonicType = getMonotonicPredicateType(ArLHS, Pred);
11451 if (!MonotonicType)
11452 return std::nullopt;
11453 // If the predicate "ArLHS `Pred` RHS" monotonically increases from false to
11454 // true as the loop iterates, and the backedge is control dependent on
11455 // "ArLHS `Pred` RHS" == true then we can reason as follows:
11456 //
11457 // * if the predicate was false in the first iteration then the predicate
11458 // is never evaluated again, since the loop exits without taking the
11459 // backedge.
11460 // * if the predicate was true in the first iteration then it will
11461 // continue to be true for all future iterations since it is
11462 // monotonically increasing.
11463 //
11464 // For both the above possibilities, we can replace the loop varying
11465 // predicate with its value on the first iteration of the loop (which is
11466 // loop invariant).
11467 //
11468 // A similar reasoning applies for a monotonically decreasing predicate, by
11469 // replacing true with false and false with true in the above two bullets.
11471 auto P = Increasing ? Pred : ICmpInst::getInverseCmpPredicate(Pred);
11472
11473 if (isLoopBackedgeGuardedByCond(L, P, LHS, RHS))
11475 RHS);
11476
11477 if (!CtxI)
11478 return std::nullopt;
11479 // Try to prove via context.
11480 // TODO: Support other cases.
11481 switch (Pred) {
11482 default:
11483 break;
11484 case ICmpInst::ICMP_ULE:
11485 case ICmpInst::ICMP_ULT: {
11486 assert(ArLHS->hasNoUnsignedWrap() && "Is a requirement of monotonicity!");
11487 // Given preconditions
11488 // (1) ArLHS does not cross the border of positive and negative parts of
11489 // range because of:
11490 // - Positive step; (TODO: lift this limitation)
11491 // - nuw - does not cross zero boundary;
11492 // - nsw - does not cross SINT_MAX boundary;
11493 // (2) ArLHS <s RHS
11494 // (3) RHS >=s 0
11495 // we can replace the loop variant ArLHS <u RHS condition with loop
11496 // invariant Start(ArLHS) <u RHS.
11497 //
11498 // Because of (1) there are two options:
11499 // - ArLHS is always negative. It means that ArLHS <u RHS is always false;
11500 // - ArLHS is always non-negative. Because of (3) RHS is also non-negative.
11501 // It means that ArLHS <s RHS <=> ArLHS <u RHS.
11502 // Because of (2) ArLHS <u RHS is trivially true.
11503 // All together it means that ArLHS <u RHS <=> Start(ArLHS) >=s 0.
11504 // We can strengthen this to Start(ArLHS) <u RHS.
11505 auto SignFlippedPred = ICmpInst::getFlippedSignednessPredicate(Pred);
11506 if (ArLHS->hasNoSignedWrap() && ArLHS->isAffine() &&
11507 isKnownPositive(ArLHS->getStepRecurrence(*this)) &&
11508 isKnownNonNegative(RHS) &&
11509 isKnownPredicateAt(SignFlippedPred, ArLHS, RHS, CtxI))
11511 RHS);
11512 }
11513 }
11514
11515 return std::nullopt;
11516}
11517
11518std::optional<ScalarEvolution::LoopInvariantPredicate>
11520 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11521 const Instruction *CtxI, const SCEV *MaxIter) {
11523 Pred, LHS, RHS, L, CtxI, MaxIter))
11524 return LIP;
11525 if (auto *UMin = dyn_cast<SCEVUMinExpr>(MaxIter))
11526 // Number of iterations expressed as UMIN isn't always great for expressing
11527 // the value on the last iteration. If the straightforward approach didn't
11528 // work, try the following trick: if the a predicate is invariant for X, it
11529 // is also invariant for umin(X, ...). So try to find something that works
11530 // among subexpressions of MaxIter expressed as umin.
11531 for (SCEVUse Op : UMin->operands())
11533 Pred, LHS, RHS, L, CtxI, Op))
11534 return LIP;
11535 return std::nullopt;
11536}
11537
11538std::optional<ScalarEvolution::LoopInvariantPredicate>
11540 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11541 const Instruction *CtxI, const SCEV *MaxIter) {
11542 // Try to prove the following set of facts:
11543 // - The predicate is monotonic in the iteration space.
11544 // - If the check does not fail on the 1st iteration:
11545 // - No overflow will happen during first MaxIter iterations;
11546 // - It will not fail on the MaxIter'th iteration.
11547 // If the check does fail on the 1st iteration, we leave the loop and no
11548 // other checks matter.
11549
11550 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11551 if (!isLoopInvariant(RHS, L)) {
11552 if (!isLoopInvariant(LHS, L))
11553 return std::nullopt;
11554
11555 std::swap(LHS, RHS);
11557 }
11558
11559 auto *AR = dyn_cast<SCEVAddRecExpr>(LHS);
11560 if (!AR || AR->getLoop() != L)
11561 return std::nullopt;
11562
11563 // Even if both are valid, we need to consistently chose the unsigned or the
11564 // signed predicate below, not mixtures of both. For now, prefer the unsigned
11565 // predicate.
11566 Pred = Pred.dropSameSign();
11567
11568 // The predicate must be relational (i.e. <, <=, >=, >).
11569 if (!ICmpInst::isRelational(Pred))
11570 return std::nullopt;
11571
11572 // TODO: Support steps other than +/- 1.
11573 const SCEV *Step = AR->getStepRecurrence(*this);
11574 auto *One = getOne(Step->getType());
11575 auto *MinusOne = getNegativeSCEV(One);
11576 if (Step != One && Step != MinusOne)
11577 return std::nullopt;
11578
11579 // Type mismatch here means that MaxIter is potentially larger than max
11580 // unsigned value in start type, which mean we cannot prove no wrap for the
11581 // indvar.
11582 if (AR->getType() != MaxIter->getType())
11583 return std::nullopt;
11584
11585 // Value of IV on suggested last iteration.
11586 const SCEV *Last = AR->evaluateAtIteration(MaxIter, *this);
11587 // Does it still meet the requirement?
11588 if (!isLoopBackedgeGuardedByCond(L, Pred, Last, RHS))
11589 return std::nullopt;
11590 // Because step is +/- 1 and MaxIter has same type as Start (i.e. it does
11591 // not exceed max unsigned value of this type), this effectively proves
11592 // that there is no wrap during the iteration. To prove that there is no
11593 // signed/unsigned wrap, we need to check that
11594 // Start <= Last for step = 1 or Start >= Last for step = -1.
11595 ICmpInst::Predicate NoOverflowPred =
11597 if (Step == MinusOne)
11598 NoOverflowPred = ICmpInst::getSwappedPredicate(NoOverflowPred);
11599 const SCEV *Start = AR->getStart();
11600 if (!isKnownPredicateAt(NoOverflowPred, Start, Last, CtxI))
11601 return std::nullopt;
11602
11603 // Everything is fine.
11604 return ScalarEvolution::LoopInvariantPredicate(Pred, Start, RHS);
11605}
11606
11607bool ScalarEvolution::isKnownPredicateViaConstantRanges(CmpPredicate Pred,
11608 SCEVUse LHS,
11609 SCEVUse RHS) {
11610 if (HasSameValue(LHS, RHS))
11611 return ICmpInst::isTrueWhenEqual(Pred);
11612
11613 auto CheckRange = [&](bool IsSigned) {
11614 auto RangeLHS = IsSigned ? getSignedRange(LHS) : getUnsignedRange(LHS);
11615 auto RangeRHS = IsSigned ? getSignedRange(RHS) : getUnsignedRange(RHS);
11616 return RangeLHS.icmp(Pred, RangeRHS);
11617 };
11618
11619 // The check at the top of the function catches the case where the values are
11620 // known to be equal.
11621 if (Pred == CmpInst::ICMP_EQ)
11622 return false;
11623
11624 if (Pred == CmpInst::ICMP_NE) {
11625 if (CheckRange(true) || CheckRange(false))
11626 return true;
11627 auto *Diff = getMinusSCEV(LHS, RHS);
11628 return !isa<SCEVCouldNotCompute>(Diff) && isKnownNonZero(Diff);
11629 }
11630
11631 return CheckRange(CmpInst::isSigned(Pred));
11632}
11633
11634bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred,
11636 // Match X to (A + C1)<ExpectedFlags> and Y to (A + C2)<ExpectedFlags>, where
11637 // C1 and C2 are constant integers. If either X or Y are not add expressions,
11638 // consider them as X + 0 and Y + 0 respectively. C1 and C2 are returned via
11639 // OutC1 and OutC2.
11640 auto MatchBinaryAddToConst = [this](SCEVUse X, SCEVUse Y, APInt &OutC1,
11641 APInt &OutC2, SCEVFlags ExpectedFlags) {
11642 SCEVUse XNonConstOp, XConstOp;
11643 SCEVUse YNonConstOp, YConstOp;
11644 SCEVFlags XFlagsPresent;
11645 SCEVFlags YFlagsPresent;
11646
11647 // Use nullptr for 0, to avoid creating a zero SCEV.
11648 if (!splitBinaryAdd(X, XConstOp, XNonConstOp, XFlagsPresent)) {
11649 XConstOp = nullptr;
11650 XNonConstOp = X;
11651 XFlagsPresent = ExpectedFlags;
11652 }
11653 if (XConstOp && !isa<SCEVConstant>(XConstOp))
11654 return false;
11655
11656 if (!splitBinaryAdd(Y, YConstOp, YNonConstOp, YFlagsPresent)) {
11657 YConstOp = nullptr;
11658 YNonConstOp = Y;
11659 YFlagsPresent = ExpectedFlags;
11660 }
11661
11662 if (YNonConstOp != XNonConstOp)
11663 return false;
11664
11665 if (YConstOp && !isa<SCEVConstant>(YConstOp))
11666 return false;
11667
11668 // When matching ADDs with NUW flags (and unsigned predicates), only the
11669 // second ADD (with the larger constant) requires NUW.
11670 if ((YFlagsPresent & ExpectedFlags) != ExpectedFlags)
11671 return false;
11673 (XFlagsPresent & ExpectedFlags) != ExpectedFlags) {
11674 return false;
11675 }
11676
11677 OutC1 = XConstOp ? cast<SCEVConstant>(XConstOp)->getAPInt()
11678 : APInt::getZero(getTypeSizeInBits(X->getType()));
11679 OutC2 = YConstOp ? cast<SCEVConstant>(YConstOp)->getAPInt()
11680 : APInt::getZero(getTypeSizeInBits(Y->getType()));
11681
11682 return true;
11683 };
11684
11685 APInt C1;
11686 APInt C2;
11687
11688 switch (Pred) {
11689 default:
11690 break;
11691
11692 case ICmpInst::ICMP_SGE:
11693 std::swap(LHS, RHS);
11694 [[fallthrough]];
11695 case ICmpInst::ICMP_SLE:
11696 // (X + C1)<nsw> s<= (X + C2)<nsw> if C1 s<= C2.
11697 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.sle(C2))
11698 return true;
11699
11700 break;
11701
11702 case ICmpInst::ICMP_SGT:
11703 std::swap(LHS, RHS);
11704 [[fallthrough]];
11705 case ICmpInst::ICMP_SLT:
11706 // (X + C1)<nsw> s< (X + C2)<nsw> if C1 s< C2.
11707 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.slt(C2))
11708 return true;
11709
11710 break;
11711
11712 case ICmpInst::ICMP_UGE:
11713 std::swap(LHS, RHS);
11714 [[fallthrough]];
11715 case ICmpInst::ICMP_ULE:
11716 // (X + C1) u<= (X + C2)<nuw> for C1 u<= C2.
11717 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ule(C2))
11718 return true;
11719
11720 break;
11721
11722 case ICmpInst::ICMP_UGT:
11723 std::swap(LHS, RHS);
11724 [[fallthrough]];
11725 case ICmpInst::ICMP_ULT:
11726 // (X + C1) u< (X + C2)<nuw> if C1 u< C2.
11727 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ult(C2))
11728 return true;
11729 break;
11730 }
11731
11732 return false;
11733}
11734
11735bool ScalarEvolution::isKnownPredicateViaSplitting(CmpPredicate Pred,
11737 if (Pred != ICmpInst::ICMP_ULT || ProvingSplitPredicate)
11738 return false;
11739
11740 // Allowing arbitrary number of activations of isKnownPredicateViaSplitting on
11741 // the stack can result in exponential time complexity.
11742 SaveAndRestore Restore(ProvingSplitPredicate, true);
11743
11744 // If L >= 0 then I `ult` L <=> I >= 0 && I `slt` L
11745 //
11746 // To prove L >= 0 we use isKnownNonNegative whereas to prove I >= 0 we use
11747 // isKnownPredicate. isKnownPredicate is more powerful, but also more
11748 // expensive; and using isKnownNonNegative(RHS) is sufficient for most of the
11749 // interesting cases seen in practice. We can consider "upgrading" L >= 0 to
11750 // use isKnownPredicate later if needed.
11751 return isKnownNonNegative(RHS) &&
11754}
11755
11756bool ScalarEvolution::isImpliedViaGuard(const BasicBlock *BB, CmpPredicate Pred,
11757 const SCEV *LHS, const SCEV *RHS) {
11758 // No need to even try if we know the module has no guards.
11759 if (!HasGuards)
11760 return false;
11761
11762 return any_of(*BB, [&](const Instruction &I) {
11763 using namespace llvm::PatternMatch;
11764
11765 Value *Condition;
11767 m_Value(Condition))) &&
11768 isImpliedCond(Pred, LHS, RHS, Condition, false);
11769 });
11770}
11771
11772/// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
11773/// protected by a conditional between LHS and RHS. This is used to
11774/// to eliminate casts.
11776 CmpPredicate Pred,
11777 const SCEV *LHS,
11778 const SCEV *RHS) {
11779 // Interpret a null as meaning no loop, where there is obviously no guard
11780 // (interprocedural conditions notwithstanding). Do not bother about
11781 // unreachable loops.
11782 if (!L || !DT.isReachableFromEntry(L->getHeader()))
11783 return true;
11784
11785 if (Opts.scev_verify_ir)
11786 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()) &&
11787 "This cannot be done on broken IR!");
11788
11789
11790 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11791 return true;
11792
11793 BasicBlock *Latch = L->getLoopLatch();
11794 if (!Latch)
11795 return false;
11796
11797 CondBrInst *LoopContinuePredicate =
11799 if (LoopContinuePredicate &&
11800 isImpliedCond(Pred, LHS, RHS, LoopContinuePredicate->getCondition(),
11801 LoopContinuePredicate->getSuccessor(0) != L->getHeader()))
11802 return true;
11803
11804 // We don't want more than one activation of the following loops on the stack
11805 // -- that can lead to O(n!) time complexity.
11806 if (WalkingBEDominatingConds)
11807 return false;
11808
11809 SaveAndRestore ClearOnExit(WalkingBEDominatingConds, true);
11810
11811 // See if we can exploit a trip count to prove the predicate.
11812 const auto &BETakenInfo = getBackedgeTakenInfo(L);
11813 const SCEV *LatchBECount = BETakenInfo.getExact(Latch, this);
11814 if (LatchBECount != getCouldNotCompute()) {
11815 // We know that Latch branches back to the loop header exactly
11816 // LatchBECount times. This means the backdege condition at Latch is
11817 // equivalent to "{0,+,1} u< LatchBECount".
11818 Type *Ty = LatchBECount->getType();
11819 auto NoWrapFlags = SCEVFlags(SCEV::FlagNUW | SCEV::FlagNW);
11820 const SCEV *LoopCounter =
11821 getAddRecExpr(getZero(Ty), getOne(Ty), L, NoWrapFlags);
11822 if (isImpliedCond(Pred, LHS, RHS, ICmpInst::ICMP_ULT, LoopCounter,
11823 LatchBECount))
11824 return true;
11825 }
11826
11827 // Check conditions due to any @llvm.assume intrinsics.
11828 for (auto &AssumeVH : AC.assumptions()) {
11829 if (!AssumeVH)
11830 continue;
11831 auto *CI = cast<CallInst>(AssumeVH);
11832 if (!DT.dominates(CI, Latch->getTerminator()))
11833 continue;
11834
11835 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
11836 return true;
11837 }
11838
11839 if (isImpliedViaGuard(Latch, Pred, LHS, RHS))
11840 return true;
11841
11842 for (DomTreeNode *DTN = DT[Latch], *HeaderDTN = DT[L->getHeader()];
11843 DTN != HeaderDTN; DTN = DTN->getIDom()) {
11844 assert(DTN && "should reach the loop header before reaching the root!");
11845
11846 BasicBlock *BB = DTN->getBlock();
11847 if (isImpliedViaGuard(BB, Pred, LHS, RHS))
11848 return true;
11849
11850 BasicBlock *PBB = BB->getSinglePredecessor();
11851 if (!PBB)
11852 continue;
11853
11855 if (!ContBr || ContBr->getSuccessor(0) == ContBr->getSuccessor(1))
11856 continue;
11857
11858 // If we have an edge `E` within the loop body that dominates the only
11859 // latch, the condition guarding `E` also guards the backedge. This
11860 // reasoning works only for loops with a single latch.
11861 // We're constructively (and conservatively) enumerating edges within the
11862 // loop body that dominate the latch. The dominator tree better agree
11863 // with us on this:
11864 assert(DT.dominates(BasicBlockEdge(PBB, BB), Latch) && "should be!");
11865 if (isImpliedCond(Pred, LHS, RHS, ContBr->getCondition(),
11866 BB != ContBr->getSuccessor(0)))
11867 return true;
11868 }
11869
11870 return false;
11871}
11872
11874 CmpPredicate Pred,
11875 const SCEV *LHS,
11876 const SCEV *RHS) {
11877 // Do not bother proving facts for unreachable code.
11878 if (!DT.isReachableFromEntry(BB))
11879 return true;
11880 if (Opts.scev_verify_ir)
11881 assert(!verifyFunction(*BB->getParent(), &dbgs()) &&
11882 "This cannot be done on broken IR!");
11883
11884 // If we cannot prove strict comparison (e.g. a > b), maybe we can prove
11885 // the facts (a >= b && a != b) separately. A typical situation is when the
11886 // non-strict comparison is known from ranges and non-equality is known from
11887 // dominating predicates. If we are proving strict comparison, we always try
11888 // to prove non-equality and non-strict comparison separately.
11889 CmpPredicate NonStrictPredicate = ICmpInst::getNonStrictCmpPredicate(Pred);
11890 const bool ProvingStrictComparison =
11891 Pred != NonStrictPredicate.dropSameSign();
11892 const Instruction *CtxI = &BB->front();
11894
11895 // Try to prove (Pred, LHS, RHS) using isImpliedCond.
11896 auto ProveViaCond = [&](const Value *Condition, bool Inverse) {
11897 Conds.emplace_back(Condition, Inverse);
11898 return isImpliedCond(Pred, LHS, RHS, Condition, Inverse, CtxI);
11899 };
11900
11901 // Starting at the block's predecessor, climb up the predecessor chain, as long
11902 // as there are predecessors that can be found that have unique successors
11903 // leading to the original block.
11904 const Loop *ContainingLoop = LI.getLoopFor(BB);
11905 const BasicBlock *PredBB;
11906 if (ContainingLoop && ContainingLoop->getHeader() == BB)
11907 PredBB = ContainingLoop->getLoopPredecessor();
11908 else
11909 PredBB = BB->getSinglePredecessor();
11910 for (std::pair<const BasicBlock *, const BasicBlock *> Pair(PredBB, BB);
11911 Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
11912 const CondBrInst *BlockEntryPredicate =
11913 dyn_cast<CondBrInst>(Pair.first->getTerminator());
11914 if (!BlockEntryPredicate)
11915 continue;
11916
11917 if (ProveViaCond(BlockEntryPredicate->getCondition(),
11918 BlockEntryPredicate->getSuccessor(0) != Pair.second))
11919 return true;
11920 }
11921
11922 // Check conditions due to any @llvm.assume intrinsics.
11923 for (auto &AssumeVH : AC.assumptions()) {
11924 if (!AssumeVH)
11925 continue;
11926 auto *CI = cast<CallInst>(AssumeVH);
11927 if (!DT.dominates(CI, BB))
11928 continue;
11929
11930 if (ProveViaCond(CI->getArgOperand(0), false))
11931 return true;
11932 }
11933
11934 // Check conditions due to any @llvm.experimental.guard intrinsics.
11935 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
11936 F.getParent(), Intrinsic::experimental_guard);
11937 if (GuardDecl)
11938 for (const auto *GU : GuardDecl->users())
11939 if (const auto *Guard = dyn_cast<IntrinsicInst>(GU))
11940 if (Guard->getFunction() == BB->getParent() && DT.dominates(Guard, BB))
11941 if (ProveViaCond(Guard->getArgOperand(0), false))
11942 return true;
11943
11944 // First try to prove conditions directly, then fall back to proving via the
11945 // non-strict predicate and not-equal.
11946 if (!ProvingStrictComparison)
11947 return false;
11948 auto ProveViaAll = [&](CmpPredicate P) {
11949 return isKnownViaNonRecursiveReasoning(P, LHS, RHS) ||
11950 any_of(Conds, [&](const auto &C) {
11951 return isImpliedCond(P, LHS, RHS, C.first, C.second, CtxI);
11952 });
11953 };
11954 return ProveViaAll(NonStrictPredicate) && ProveViaAll(ICmpInst::ICMP_NE);
11955}
11956
11958 const SCEV *LHS,
11959 const SCEV *RHS) {
11960 // Interpret a null as meaning no loop, where there is obviously no guard
11961 // (interprocedural conditions notwithstanding).
11962 if (!L)
11963 return false;
11964
11965 // Both LHS and RHS must be available at loop entry.
11967 "LHS is not available at Loop Entry");
11969 "RHS is not available at Loop Entry");
11970
11971 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11972 return true;
11973
11974 return isBasicBlockEntryGuardedByCond(L->getHeader(), Pred, LHS, RHS);
11975}
11976
11977bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
11978 const SCEV *RHS,
11979 const Value *FoundCondValue, bool Inverse,
11980 const Instruction *CtxI) {
11981 // False conditions implies anything. Do not bother analyzing it further.
11982 if (FoundCondValue ==
11983 ConstantInt::getBool(FoundCondValue->getContext(), Inverse))
11984 return true;
11985
11986 if (!PendingLoopPredicates.insert(FoundCondValue).second)
11987 return false;
11988
11989 llvm::scope_exit ClearOnExit(
11990 [&]() { PendingLoopPredicates.erase(FoundCondValue); });
11991
11992 // Recursively handle And and Or conditions.
11993 const Value *Op0, *Op1;
11994 if (match(FoundCondValue, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) {
11995 if (!Inverse)
11996 return isImpliedCond(Pred, LHS, RHS, Op0, Inverse, CtxI) ||
11997 isImpliedCond(Pred, LHS, RHS, Op1, Inverse, CtxI);
11998 } else if (match(FoundCondValue, m_LogicalOr(m_Value(Op0), m_Value(Op1)))) {
11999 if (Inverse)
12000 return isImpliedCond(Pred, LHS, RHS, Op0, Inverse, CtxI) ||
12001 isImpliedCond(Pred, LHS, RHS, Op1, Inverse, CtxI);
12002 }
12003
12004 const ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue);
12005 if (!ICI) return false;
12006
12007 // Now that we found a conditional branch that dominates the loop or controls
12008 // the loop latch. Check to see if it is the comparison we are looking for.
12009 CmpPredicate FoundPred;
12010 if (Inverse)
12011 FoundPred = ICI->getInverseCmpPredicate();
12012 else
12013 FoundPred = ICI->getCmpPredicate();
12014
12015 const SCEV *FoundLHS = getSCEV(ICI->getOperand(0));
12016 const SCEV *FoundRHS = getSCEV(ICI->getOperand(1));
12017
12018 return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS, CtxI);
12019}
12020
12021bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
12022 const SCEV *RHS, CmpPredicate FoundPred,
12023 const SCEV *FoundLHS, const SCEV *FoundRHS,
12024 const Instruction *CtxI) {
12025 // Balance the types.
12026 if (getTypeSizeInBits(LHS->getType()) <
12027 getTypeSizeInBits(FoundLHS->getType())) {
12028 // For unsigned and equality predicates, try to prove that both found
12029 // operands fit into narrow unsigned range. If so, try to prove facts in
12030 // narrow types.
12031 if (!CmpInst::isSigned(FoundPred) && !FoundLHS->getType()->isPointerTy() &&
12032 !FoundRHS->getType()->isPointerTy()) {
12033 auto *NarrowType = LHS->getType();
12034 auto *WideType = FoundLHS->getType();
12035 auto BitWidth = getTypeSizeInBits(NarrowType);
12036 const SCEV *MaxValue = getZeroExtendExpr(
12038 if (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_ULE, FoundLHS,
12039 MaxValue) &&
12040 isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_ULE, FoundRHS,
12041 MaxValue)) {
12042 const SCEV *TruncFoundLHS = getTruncateExpr(FoundLHS, NarrowType);
12043 const SCEV *TruncFoundRHS = getTruncateExpr(FoundRHS, NarrowType);
12044 // We cannot preserve samesign after truncation.
12045 if (isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred.dropSameSign(),
12046 TruncFoundLHS, TruncFoundRHS, CtxI))
12047 return true;
12048 }
12049 }
12050
12051 if (LHS->getType()->isPointerTy() || RHS->getType()->isPointerTy())
12052 return false;
12053 if (CmpInst::isSigned(Pred)) {
12054 LHS = getSignExtendExpr(LHS, FoundLHS->getType());
12055 RHS = getSignExtendExpr(RHS, FoundLHS->getType());
12056 } else {
12057 LHS = getZeroExtendExpr(LHS, FoundLHS->getType());
12058 RHS = getZeroExtendExpr(RHS, FoundLHS->getType());
12059 }
12060 } else if (getTypeSizeInBits(LHS->getType()) >
12061 getTypeSizeInBits(FoundLHS->getType())) {
12062 if (FoundLHS->getType()->isPointerTy() || FoundRHS->getType()->isPointerTy())
12063 return false;
12064 if (CmpInst::isSigned(FoundPred)) {
12065 FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType());
12066 FoundRHS = getSignExtendExpr(FoundRHS, LHS->getType());
12067 } else {
12068 FoundLHS = getZeroExtendExpr(FoundLHS, LHS->getType());
12069 FoundRHS = getZeroExtendExpr(FoundRHS, LHS->getType());
12070 }
12071 }
12072 return isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred, FoundLHS,
12073 FoundRHS, CtxI);
12074}
12075
12076bool ScalarEvolution::isImpliedCondBalancedTypes(
12077 CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS, CmpPredicate FoundPred,
12078 SCEVUse FoundLHS, SCEVUse FoundRHS, const Instruction *CtxI) {
12080 getTypeSizeInBits(FoundLHS->getType()) &&
12081 "Types should be balanced!");
12082 // Canonicalize the query to match the way instcombine will have
12083 // canonicalized the comparison.
12085 if (LHS == RHS)
12087 if (SimplifyICmpOperands(FoundPred, FoundLHS, FoundRHS))
12088 if (FoundLHS == FoundRHS)
12089 return CmpInst::isFalseWhenEqual(FoundPred);
12090
12091 // Check to see if we can make the LHS or RHS match.
12092 if (LHS == FoundRHS || RHS == FoundLHS) {
12093 if (isa<SCEVConstant>(RHS)) {
12094 std::swap(FoundLHS, FoundRHS);
12095 FoundPred = ICmpInst::getSwappedCmpPredicate(FoundPred);
12096 } else {
12097 std::swap(LHS, RHS);
12099 }
12100 }
12101
12102 // Check whether the found predicate is the same as the desired predicate.
12103 if (auto P = CmpPredicate::getMatching(FoundPred, Pred))
12104 return isImpliedCondOperands(*P, LHS, RHS, FoundLHS, FoundRHS, CtxI);
12105
12106 // Check whether swapping the found predicate makes it the same as the
12107 // desired predicate.
12108 if (auto P = CmpPredicate::getMatching(
12110 // We can write the implication
12111 // 0. LHS Pred RHS <- FoundLHS SwapPred FoundRHS
12112 // using one of the following ways:
12113 // 1. LHS Pred RHS <- FoundRHS Pred FoundLHS
12114 // 2. RHS SwapPred LHS <- FoundLHS SwapPred FoundRHS
12115 // Both require swapping the operands of one condition. Don't do this if it
12116 // would break canonical constant/addrec ordering.
12118 return isImpliedCondOperands(ICmpInst::getSwappedCmpPredicate(*P), RHS,
12119 LHS, FoundLHS, FoundRHS, CtxI);
12120 if (!isa<SCEVConstant>(FoundRHS) && !isa<SCEVAddRecExpr>(FoundLHS))
12121 return isImpliedCondOperands(*P, LHS, RHS, FoundRHS, FoundLHS, CtxI);
12122
12123 return false;
12124 }
12125
12126 // Try the query as is and with its operands swapped. E.g. "x >s -1" is
12127 // tried as "-1 <s x" too, to match the found condition "x <u y".
12128 auto IsSignFlippedPredicate = [](CmpInst::Predicate P1,
12130 return CmpInst::isRelational(P2) &&
12132 };
12133 if (IsSignFlippedPredicate(Pred, FoundPred) &&
12134 isImpliedCondViaSignFlippedPred(Pred, LHS, RHS, FoundLHS, FoundRHS, CtxI))
12135 return true;
12136 CmpPredicate SwappedPred = ICmpInst::getSwappedCmpPredicate(Pred);
12137 if (IsSignFlippedPredicate(SwappedPred, FoundPred) &&
12138 isImpliedCondViaSignFlippedPred(SwappedPred, RHS, LHS, FoundLHS, FoundRHS,
12139 CtxI))
12140 return true;
12141
12142 // Check if we can make progress by sharpening ranges.
12143 if (FoundPred == ICmpInst::ICMP_NE &&
12144 (isa<SCEVConstant>(FoundLHS) || isa<SCEVConstant>(FoundRHS))) {
12145
12146 const SCEVConstant *C = nullptr;
12147 const SCEV *V = nullptr;
12148
12149 if (isa<SCEVConstant>(FoundLHS)) {
12150 C = cast<SCEVConstant>(FoundLHS);
12151 V = FoundRHS;
12152 } else {
12153 C = cast<SCEVConstant>(FoundRHS);
12154 V = FoundLHS;
12155 }
12156
12157 // The guarding predicate tells us that C != V. If the known range
12158 // of V is [C, t), we can sharpen the range to [C + 1, t). The
12159 // range we consider has to correspond to same signedness as the
12160 // predicate we're interested in folding.
12161
12162 APInt Min = ICmpInst::isSigned(Pred) ?
12164
12165 if (Min == C->getAPInt()) {
12166 // Given (V >= Min && V != Min) we conclude V >= (Min + 1).
12167 // This is true even if (Min + 1) wraps around -- in case of
12168 // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)).
12169
12170 APInt SharperMin = Min + 1;
12171
12172 switch (Pred) {
12173 case ICmpInst::ICMP_SGE:
12174 case ICmpInst::ICMP_UGE:
12175 // We know V `Pred` SharperMin. If this implies LHS `Pred`
12176 // RHS, we're done.
12177 if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(SharperMin),
12178 CtxI))
12179 return true;
12180 [[fallthrough]];
12181
12182 case ICmpInst::ICMP_SGT:
12183 case ICmpInst::ICMP_UGT:
12184 // We know from the range information that (V `Pred` Min ||
12185 // V == Min). We know from the guarding condition that !(V
12186 // == Min). This gives us
12187 //
12188 // V `Pred` Min || V == Min && !(V == Min)
12189 // => V `Pred` Min
12190 //
12191 // If V `Pred` Min implies LHS `Pred` RHS, we're done.
12192
12193 if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(Min), CtxI))
12194 return true;
12195 break;
12196
12197 // `LHS < RHS` and `LHS <= RHS` are handled in the same way as `RHS > LHS` and `RHS >= LHS` respectively.
12198 case ICmpInst::ICMP_SLE:
12199 case ICmpInst::ICMP_ULE:
12200 if (isImpliedCondOperands(ICmpInst::getSwappedCmpPredicate(Pred), RHS,
12201 LHS, V, getConstant(SharperMin), CtxI))
12202 return true;
12203 [[fallthrough]];
12204
12205 case ICmpInst::ICMP_SLT:
12206 case ICmpInst::ICMP_ULT:
12207 if (isImpliedCondOperands(ICmpInst::getSwappedCmpPredicate(Pred), RHS,
12208 LHS, V, getConstant(Min), CtxI))
12209 return true;
12210 break;
12211
12212 default:
12213 // No change
12214 break;
12215 }
12216 }
12217 }
12218
12219 // Check whether the actual condition is beyond sufficient.
12220 if (FoundPred == ICmpInst::ICMP_EQ)
12222 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, CtxI))
12223 return true;
12224 if (Pred == ICmpInst::ICMP_NE)
12225 if (!ICmpInst::isTrueWhenEqual(FoundPred))
12226 if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS, CtxI))
12227 return true;
12228
12229 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS))
12230 return true;
12231
12232 // Otherwise assume the worst.
12233 return false;
12234}
12235
12236bool ScalarEvolution::isImpliedCondViaSignFlippedPred(
12237 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12238 const SCEV *FoundRHS, const Instruction *CtxI) {
12239 CmpPredicate FoundPred = ICmpInst::getFlippedSignednessPredicate(Pred);
12240 // Unsigned comparison is the same as signed comparison when both the
12241 // operands are non-negative or negative.
12242 if (haveSameSign(FoundLHS, FoundRHS))
12243 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, CtxI);
12244 // Canonicalize both conditions to "le/lt".
12247 FoundPred = ICmpInst::getSwappedCmpPredicate(FoundPred);
12248 std::swap(LHS, RHS);
12249 std::swap(FoundLHS, FoundRHS);
12250 }
12251 assert((ICmpInst::isLT(Pred) || ICmpInst::isLE(Pred)) && "Must be!");
12252 assert((ICmpInst::isLT(FoundPred) || ICmpInst::isLE(FoundPred)) &&
12253 "Must be!");
12254 if (ICmpInst::isSigned(Pred)) {
12255 // Use implication:
12256 // x <u y && y <=u 2^(n-1) --> x >=s 0,
12257 // x <=u y && y <u 2^(n-1) --> x >=s 0,
12258 // where n is the bit width of x and y.
12259 // In both cases x <u 2^(n-1), so x >=s 0.
12260 // y <=u 2^(n-1) means y >=s 0 or y == INT_MIN.
12261 // If the implication's left part holds, try proving the query from x >=s
12262 // 0.
12263 Type *Ty = FoundLHS->getType();
12264 unsigned BitWidth = getTypeSizeInBits(Ty);
12265 const SCEV *SignedMin = getConstant(APInt::getSignedMinValue(BitWidth));
12266 CmpPredicate BoundPred =
12268 if (isKnownViaNonRecursiveReasoning(BoundPred, FoundRHS, SignedMin)) {
12269 // x >=s 0, written as 0 <=s x, or as -1 <s x to match a strict query.
12270 const SCEV *Bound = ICmpInst::isLE(Pred) ? getZero(Ty) : getMinusOne(Ty);
12271 if (isImpliedCondOperands(Pred, LHS, RHS, Bound, FoundLHS, CtxI))
12272 return true;
12273 }
12274 }
12276 // Use implication:
12277 // x <u y && y >=s 0 --> x <s y.
12278 // If we can prove the left part, the right part is also proven.
12279 return isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS);
12280 }
12282 // Use implication:
12283 // x <s y && y <s 0 --> x <u y.
12284 // If we can prove the left part, the right part is also proven.
12285 return isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS);
12286 }
12287 return false;
12288}
12289
12290bool ScalarEvolution::splitBinaryAdd(SCEVUse Expr, SCEVUse &L, SCEVUse &R,
12291 SCEVFlags &Flags) {
12292 if (!match(Expr, m_scev_Add(m_SCEV(L), m_SCEV(R))))
12293 return false;
12294
12295 Flags = cast<SCEVAddExpr>(Expr)->getNoWrapFlags();
12296 return true;
12297}
12298
12299std::optional<APInt>
12301 // We avoid subtracting expressions here because this function is usually
12302 // fairly deep in the call stack (i.e. is called many times).
12303
12304 unsigned BW = getTypeSizeInBits(More->getType());
12305 APInt Diff(BW, 0);
12306 APInt DiffMul(BW, 1);
12307 // Try various simplifications to reduce the difference to a constant. Limit
12308 // the number of allowed simplifications to keep compile-time low.
12309 for (unsigned I = 0; I < 8; ++I) {
12310 if (More == Less)
12311 return Diff;
12312
12313 // Reduce addrecs with identical steps to their start value.
12315 const auto *LAR = cast<SCEVAddRecExpr>(Less);
12316 const auto *MAR = cast<SCEVAddRecExpr>(More);
12317
12318 if (LAR->getLoop() != MAR->getLoop())
12319 return std::nullopt;
12320
12321 // We look at affine expressions only; not for correctness but to keep
12322 // getStepRecurrence cheap.
12323 if (!LAR->isAffine() || !MAR->isAffine())
12324 return std::nullopt;
12325
12326 if (LAR->getStepRecurrence(*this) != MAR->getStepRecurrence(*this))
12327 return std::nullopt;
12328
12329 Less = LAR->getStart();
12330 More = MAR->getStart();
12331 continue;
12332 }
12333
12334 // Try to match a common constant multiply.
12335 auto MatchConstMul =
12336 [](const SCEV *S) -> std::optional<std::pair<const SCEV *, APInt>> {
12337 const APInt *C;
12338 const SCEV *Op;
12339 if (match(S, m_scev_Mul(m_scev_APInt(C), m_SCEV(Op))))
12340 return {{Op, *C}};
12341 return std::nullopt;
12342 };
12343 if (auto MatchedMore = MatchConstMul(More)) {
12344 if (auto MatchedLess = MatchConstMul(Less)) {
12345 if (MatchedMore->second == MatchedLess->second) {
12346 More = MatchedMore->first;
12347 Less = MatchedLess->first;
12348 DiffMul *= MatchedMore->second;
12349 continue;
12350 }
12351 }
12352 }
12353
12354 // Try to cancel out common factors in two add expressions.
12356 auto Add = [&](const SCEV *S, int Mul) {
12357 if (auto *C = dyn_cast<SCEVConstant>(S)) {
12358 if (Mul == 1) {
12359 Diff += C->getAPInt() * DiffMul;
12360 } else {
12361 assert(Mul == -1);
12362 Diff -= C->getAPInt() * DiffMul;
12363 }
12364 } else
12365 Multiplicity[S] += Mul;
12366 };
12367 auto Decompose = [&](const SCEV *S, int Mul) {
12368 if (isa<SCEVAddExpr>(S)) {
12369 for (const SCEV *Op : S->operands())
12370 Add(Op, Mul);
12371 } else
12372 Add(S, Mul);
12373 };
12374 Decompose(More, 1);
12375 Decompose(Less, -1);
12376
12377 // Check whether all the non-constants cancel out, or reduce to new
12378 // More/Less values.
12379 const SCEV *NewMore = nullptr, *NewLess = nullptr;
12380 for (const auto &[S, Mul] : Multiplicity) {
12381 if (Mul == 0)
12382 continue;
12383 if (Mul == 1) {
12384 if (NewMore)
12385 return std::nullopt;
12386 NewMore = S;
12387 } else if (Mul == -1) {
12388 if (NewLess)
12389 return std::nullopt;
12390 NewLess = S;
12391 } else
12392 return std::nullopt;
12393 }
12394
12395 // Values stayed the same, no point in trying further.
12396 if (NewMore == More || NewLess == Less)
12397 return std::nullopt;
12398
12399 More = NewMore;
12400 Less = NewLess;
12401
12402 // Reduced to constant.
12403 if (!More && !Less)
12404 return Diff;
12405
12406 // Left with variable on only one side, bail out.
12407 if (!More || !Less)
12408 return std::nullopt;
12409 }
12410
12411 // Did not reduce to constant.
12412 return std::nullopt;
12413}
12414
12415bool ScalarEvolution::isImpliedCondOperandsViaAddRecStart(
12416 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12417 const SCEV *FoundRHS, const Instruction *CtxI) {
12418 // Try to recognize the following pattern:
12419 //
12420 // FoundRHS = ...
12421 // ...
12422 // loop:
12423 // FoundLHS = {Start,+,W}
12424 // context_bb: // Basic block from the same loop
12425 // known(Pred, FoundLHS, FoundRHS)
12426 //
12427 // If some predicate is known in the context of a loop, it is also known on
12428 // each iteration of this loop, including the first iteration. Therefore, in
12429 // this case, `FoundLHS Pred FoundRHS` implies `Start Pred FoundRHS`. Try to
12430 // prove the original pred using this fact.
12431 if (!CtxI)
12432 return false;
12433 const BasicBlock *ContextBB = CtxI->getParent();
12434 // Make sure AR varies in the context block.
12435 if (auto *AR = dyn_cast<SCEVAddRecExpr>(FoundLHS)) {
12436 const Loop *L = AR->getLoop();
12437 const auto *Latch = L->getLoopLatch();
12438 // Make sure that context belongs to the loop and executes on 1st iteration
12439 // (if it ever executes at all).
12440 if (!L->contains(ContextBB) || !Latch || !DT.dominates(ContextBB, Latch))
12441 return false;
12442 if (!isAvailableAtLoopEntry(FoundRHS, AR->getLoop()))
12443 return false;
12444 return isImpliedCondOperands(Pred, LHS, RHS, AR->getStart(), FoundRHS);
12445 }
12446
12447 if (auto *AR = dyn_cast<SCEVAddRecExpr>(FoundRHS)) {
12448 const Loop *L = AR->getLoop();
12449 const auto *Latch = L->getLoopLatch();
12450 // Make sure that context belongs to the loop and executes on 1st iteration
12451 // (if it ever executes at all).
12452 if (!L->contains(ContextBB) || !Latch || !DT.dominates(ContextBB, Latch))
12453 return false;
12454 if (!isAvailableAtLoopEntry(FoundLHS, AR->getLoop()))
12455 return false;
12456 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, AR->getStart());
12457 }
12458
12459 return false;
12460}
12461
12462bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow(CmpPredicate Pred,
12463 const SCEV *LHS,
12464 const SCEV *RHS,
12465 const SCEV *FoundLHS,
12466 const SCEV *FoundRHS) {
12468 return false;
12469
12470 const auto *AddRecLHS = dyn_cast<SCEVAddRecExpr>(LHS);
12471 if (!AddRecLHS)
12472 return false;
12473
12474 const auto *AddRecFoundLHS = dyn_cast<SCEVAddRecExpr>(FoundLHS);
12475 if (!AddRecFoundLHS)
12476 return false;
12477
12478 // We'd like to let SCEV reason about control dependencies, so we constrain
12479 // both the inequalities to be about add recurrences on the same loop. This
12480 // way we can use isLoopEntryGuardedByCond later.
12481
12482 const Loop *L = AddRecFoundLHS->getLoop();
12483 if (L != AddRecLHS->getLoop())
12484 return false;
12485
12486 // FoundLHS u< FoundRHS u< -C => (FoundLHS + C) u< (FoundRHS + C) ... (1)
12487 //
12488 // FoundLHS s< FoundRHS s< INT_MIN - C => (FoundLHS + C) s< (FoundRHS + C)
12489 // ... (2)
12490 //
12491 // Informal proof for (2), assuming (1) [*]:
12492 //
12493 // We'll also assume (A s< B) <=> ((A + INT_MIN) u< (B + INT_MIN)) ... (3)[**]
12494 //
12495 // Then
12496 //
12497 // FoundLHS s< FoundRHS s< INT_MIN - C
12498 // <=> (FoundLHS + INT_MIN) u< (FoundRHS + INT_MIN) u< -C [ using (3) ]
12499 // <=> (FoundLHS + INT_MIN + C) u< (FoundRHS + INT_MIN + C) [ using (1) ]
12500 // <=> (FoundLHS + INT_MIN + C + INT_MIN) s<
12501 // (FoundRHS + INT_MIN + C + INT_MIN) [ using (3) ]
12502 // <=> FoundLHS + C s< FoundRHS + C
12503 //
12504 // [*]: (1) can be proved by ruling out overflow.
12505 //
12506 // [**]: This can be proved by analyzing all the four possibilities:
12507 // (A s< 0, B s< 0), (A s< 0, B s>= 0), (A s>= 0, B s< 0) and
12508 // (A s>= 0, B s>= 0).
12509 //
12510 // Note:
12511 // Despite (2), "FoundRHS s< INT_MIN - C" does not mean that "FoundRHS + C"
12512 // will not sign underflow. For instance, say FoundLHS = (i8 -128), FoundRHS
12513 // = (i8 -127) and C = (i8 -100). Then INT_MIN - C = (i8 -28), and FoundRHS
12514 // s< (INT_MIN - C). Lack of sign overflow / underflow in "FoundRHS + C" is
12515 // neither necessary nor sufficient to prove "(FoundLHS + C) s< (FoundRHS +
12516 // C)".
12517
12518 std::optional<APInt> LDiff = computeConstantDifference(LHS, FoundLHS);
12519 if (!LDiff)
12520 return false;
12521 std::optional<APInt> RDiff = computeConstantDifference(RHS, FoundRHS);
12522 if (!RDiff || *LDiff != *RDiff)
12523 return false;
12524
12525 if (LDiff->isMinValue())
12526 return true;
12527
12528 APInt FoundRHSLimit;
12529
12530 if (Pred == CmpInst::ICMP_ULT) {
12531 FoundRHSLimit = -(*RDiff);
12532 } else {
12533 assert(Pred == CmpInst::ICMP_SLT && "Checked above!");
12534 FoundRHSLimit = APInt::getSignedMinValue(getTypeSizeInBits(RHS->getType())) - *RDiff;
12535 }
12536
12537 // Try to prove (1) or (2), as needed.
12538 return isAvailableAtLoopEntry(FoundRHS, L) &&
12539 isLoopEntryGuardedByCond(L, Pred, FoundRHS,
12540 getConstant(FoundRHSLimit));
12541}
12542
12543bool ScalarEvolution::isImpliedViaMerge(CmpPredicate Pred, const SCEV *LHS,
12544 const SCEV *RHS, const SCEV *FoundLHS,
12545 const SCEV *FoundRHS, unsigned Depth) {
12546 const PHINode *LPhi = nullptr, *RPhi = nullptr;
12547
12548 llvm::scope_exit ClearOnExit([&]() {
12549 if (LPhi) {
12550 bool Erased = PendingMerges.erase(LPhi);
12551 assert(Erased && "Failed to erase LPhi!");
12552 (void)Erased;
12553 }
12554 if (RPhi) {
12555 bool Erased = PendingMerges.erase(RPhi);
12556 assert(Erased && "Failed to erase RPhi!");
12557 (void)Erased;
12558 }
12559 });
12560
12561 // Find respective Phis and check that they are not being pending.
12562 if (const SCEVUnknown *LU = dyn_cast<SCEVUnknown>(LHS))
12563 if (auto *Phi = dyn_cast<PHINode>(LU->getValue())) {
12564 if (!PendingMerges.insert(Phi).second)
12565 return false;
12566 LPhi = Phi;
12567 }
12568 if (const SCEVUnknown *RU = dyn_cast<SCEVUnknown>(RHS))
12569 if (auto *Phi = dyn_cast<PHINode>(RU->getValue())) {
12570 // If we detect a loop of Phi nodes being processed by this method, for
12571 // example:
12572 //
12573 // %a = phi i32 [ %some1, %preheader ], [ %b, %latch ]
12574 // %b = phi i32 [ %some2, %preheader ], [ %a, %latch ]
12575 //
12576 // we don't want to deal with a case that complex, so return conservative
12577 // answer false.
12578 if (!PendingMerges.insert(Phi).second)
12579 return false;
12580 RPhi = Phi;
12581 }
12582
12583 // If none of LHS, RHS is a Phi, nothing to do here.
12584 if (!LPhi && !RPhi)
12585 return false;
12586
12587 // If there is a SCEVUnknown Phi we are interested in, make it left.
12588 if (!LPhi) {
12589 std::swap(LHS, RHS);
12590 std::swap(FoundLHS, FoundRHS);
12591 std::swap(LPhi, RPhi);
12593 }
12594
12595 assert(LPhi && "LPhi should definitely be a SCEVUnknown Phi!");
12596 const BasicBlock *LBB = LPhi->getParent();
12597 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(RHS);
12598
12599 auto ProvedEasily = [&](const SCEV *S1, const SCEV *S2) {
12600 return isKnownViaNonRecursiveReasoning(Pred, S1, S2) ||
12601 isImpliedCondOperandsViaRanges(Pred, S1, S2, Pred, FoundLHS, FoundRHS) ||
12602 isImpliedViaOperations(Pred, S1, S2, FoundLHS, FoundRHS, Depth);
12603 };
12604
12605 if (RPhi && RPhi->getParent() == LBB) {
12606 // Case one: RHS is also a SCEVUnknown Phi from the same basic block.
12607 // If we compare two Phis from the same block, and for each entry block
12608 // the predicate is true for incoming values from this block, then the
12609 // predicate is also true for the Phis.
12610 for (const BasicBlock *IncBB : predecessors(LBB)) {
12611 const SCEV *L = getSCEV(LPhi->getIncomingValueForBlock(IncBB));
12612 const SCEV *R = getSCEV(RPhi->getIncomingValueForBlock(IncBB));
12613 if (!ProvedEasily(L, R))
12614 return false;
12615 }
12616 } else if (RAR && RAR->getLoop()->getHeader() == LBB) {
12617 // Case two: RHS is also a Phi from the same basic block, and it is an
12618 // AddRec. It means that there is a loop which has both AddRec and Unknown
12619 // PHIs, for it we can compare incoming values of AddRec from above the loop
12620 // and latch with their respective incoming values of LPhi.
12621 // TODO: Generalize to handle loops with many inputs in a header.
12622 if (LPhi->getNumIncomingValues() != 2) return false;
12623
12624 auto *RLoop = RAR->getLoop();
12625 auto *Predecessor = RLoop->getLoopPredecessor();
12626 assert(Predecessor && "Loop with AddRec with no predecessor?");
12627 const SCEV *L1 = getSCEV(LPhi->getIncomingValueForBlock(Predecessor));
12628 if (!ProvedEasily(L1, RAR->getStart()))
12629 return false;
12630 auto *Latch = RLoop->getLoopLatch();
12631 assert(Latch && "Loop with AddRec with no latch?");
12632 const SCEV *L2 = getSCEV(LPhi->getIncomingValueForBlock(Latch));
12633 if (!ProvedEasily(L2, RAR->getPostIncExpr(*this)))
12634 return false;
12635 } else {
12636 // In all other cases go over inputs of LHS and compare each of them to RHS,
12637 // the predicate is true for (LHS, RHS) if it is true for all such pairs.
12638 // At this point RHS is either a non-Phi, or it is a Phi from some block
12639 // different from LBB.
12640 for (const BasicBlock *IncBB : predecessors(LBB)) {
12641 // Check that RHS is available in this block.
12642 if (!dominates(RHS, IncBB))
12643 return false;
12644 const SCEV *L = getSCEV(LPhi->getIncomingValueForBlock(IncBB));
12645 // Make sure L does not refer to a value from a potentially previous
12646 // iteration of a loop.
12647 if (!properlyDominates(L, LBB))
12648 return false;
12649 // Addrecs are considered to properly dominate their loop, so are missed
12650 // by the previous check. Discard any values that have computable
12651 // evolution in this loop.
12652 if (auto *Loop = LI.getLoopFor(LBB))
12654 return false;
12655 if (!ProvedEasily(L, RHS))
12656 return false;
12657 }
12658 }
12659 return true;
12660}
12661
12662bool ScalarEvolution::isImpliedCondOperandsViaShift(CmpPredicate Pred,
12663 const SCEV *LHS,
12664 const SCEV *RHS,
12665 const SCEV *FoundLHS,
12666 const SCEV *FoundRHS) {
12667 // We want to imply LHS < RHS from LHS < (RHS >> shiftvalue). First, make
12668 // sure that we are dealing with same LHS.
12669 if (RHS == FoundRHS) {
12670 std::swap(LHS, RHS);
12671 std::swap(FoundLHS, FoundRHS);
12673 }
12674 if (LHS != FoundLHS)
12675 return false;
12676
12677 auto *SUFoundRHS = dyn_cast<SCEVUnknown>(FoundRHS);
12678 if (!SUFoundRHS)
12679 return false;
12680
12681 Value *Shiftee, *ShiftValue;
12682
12683 using namespace PatternMatch;
12684 if (match(SUFoundRHS->getValue(),
12685 m_LShr(m_Value(Shiftee), m_Value(ShiftValue)))) {
12686 auto *ShifteeS = getSCEV(Shiftee);
12687 // Prove one of the following:
12688 // LHS <u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <u RHS
12689 // LHS <=u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <=u RHS
12690 // LHS <s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12691 // ---> LHS <s RHS
12692 // LHS <=s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12693 // ---> LHS <=s RHS
12695 return isKnownPredicate(ICmpInst::ICMP_ULE, ShifteeS, RHS);
12697 if (isKnownNonNegative(ShifteeS))
12698 return isKnownPredicate(ICmpInst::ICMP_SLE, ShifteeS, RHS);
12699 }
12700
12701 return false;
12702}
12703
12704bool ScalarEvolution::isImpliedCondOperandsViaMatchingDiff(
12705 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12706 const SCEV *FoundRHS) {
12707 // Only valid for equality predicates: (A == B) implies (C == D) when
12708 // the SCEV difference A - B equals C - D (they check the same
12709 // underlying relationship at every iteration).
12711 return false;
12712
12713 // Restrict to cases involving loop recurrences - that's where this
12714 // pattern arises (correlated IV comparisons). This avoids calling
12715 // getMinusSCEV on arbitrary non-loop expressions.
12717 (!isa<SCEVAddRecExpr>(FoundLHS) && !isa<SCEVAddRecExpr>(FoundRHS)))
12718 return false;
12719
12720 // AddRecs from different loops can never produce matching differences.
12721 const SCEVAddRecExpr *QueryAddRec = dyn_cast<SCEVAddRecExpr>(LHS);
12722 if (!QueryAddRec)
12723 QueryAddRec = cast<SCEVAddRecExpr>(RHS);
12724 const SCEVAddRecExpr *FoundAddRec = dyn_cast<SCEVAddRecExpr>(FoundLHS);
12725 if (!FoundAddRec)
12726 FoundAddRec = cast<SCEVAddRecExpr>(FoundRHS);
12727 if (QueryAddRec->getLoop() != FoundAddRec->getLoop())
12728 return false;
12729
12730 // If the strides differ, the differences can never match.
12731 if (QueryAddRec->getStepRecurrence(*this) !=
12732 FoundAddRec->getStepRecurrence(*this))
12733 return false;
12734
12735 // Compute differences. For pointer-typed operands sharing the same base,
12736 // getMinusSCEV strips the common base and returns an integer SCEV.
12737 // For example, {base,+,8} - (base+8*n) = {-8n,+,8}
12738 const SCEV *FoundDiff = getMinusSCEV(FoundLHS, FoundRHS);
12739 if (isa<SCEVCouldNotCompute>(FoundDiff))
12740 return false;
12741
12742 const SCEV *Diff = getMinusSCEV(LHS, RHS);
12743 if (isa<SCEVCouldNotCompute>(Diff))
12744 return false;
12745
12746 return Diff == FoundDiff;
12747}
12748
12749bool ScalarEvolution::isImpliedCondOperands(CmpPredicate Pred, const SCEV *LHS,
12750 const SCEV *RHS,
12751 const SCEV *FoundLHS,
12752 const SCEV *FoundRHS,
12753 const Instruction *CtxI) {
12754 return isImpliedCondOperandsViaRanges(Pred, LHS, RHS, Pred, FoundLHS,
12755 FoundRHS) ||
12756 isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS,
12757 FoundRHS) ||
12758 isImpliedCondOperandsViaShift(Pred, LHS, RHS, FoundLHS, FoundRHS) ||
12759 isImpliedCondOperandsViaAddRecStart(Pred, LHS, RHS, FoundLHS, FoundRHS,
12760 CtxI) ||
12761 isImpliedCondOperandsViaMatchingDiff(Pred, LHS, RHS, FoundLHS,
12762 FoundRHS) ||
12763 isImpliedCondOperandsHelper(Pred, LHS, RHS, FoundLHS, FoundRHS);
12764}
12765
12766/// Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
12767template <typename MinMaxExprType>
12768static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr,
12769 const SCEV *Candidate) {
12770 const MinMaxExprType *MinMaxExpr = dyn_cast<MinMaxExprType>(MaybeMinMaxExpr);
12771 if (!MinMaxExpr)
12772 return false;
12773
12774 return is_contained(MinMaxExpr->operands(), Candidate);
12775}
12776
12778 CmpPredicate Pred, const SCEV *LHS,
12779 const SCEV *RHS) {
12780 // If both sides are affine addrecs for the same loop, with equal
12781 // steps, and we know the recurrences don't wrap, then we only
12782 // need to check the predicate on the starting values.
12783
12785 return false;
12786
12787 const SCEV *LStart, *RStart, *Step;
12788 const Loop *L;
12789 if (!match(LHS,
12790 m_scev_AffineAddRec(m_SCEV(LStart), m_SCEV(Step), m_Loop(L))) ||
12792 m_SpecificLoop(L))))
12793 return false;
12797 if (!LAR->getNoWrapFlags(NW) || !RAR->getNoWrapFlags(NW))
12798 return false;
12799
12800 return SE.isKnownPredicate(Pred, LStart, RStart);
12801}
12802
12803/// Is LHS `Pred` RHS true because one of them is an AddRec that is known not to
12804/// go below its own start value?
12807 const SCEV *LHS,
12808 const SCEV *RHS) {
12809 // Normalize to (AddRec Pred Start).
12812 std::swap(LHS, RHS);
12813 }
12814
12815 // The recurrence is equal to Start in the first iteration, so only the
12816 // non-strict predicate holds.
12818 return false;
12819
12820 const auto *AR = dyn_cast<SCEVAddRecExpr>(LHS);
12821 if (!AR || AR->getStart() != RHS)
12822 return false;
12823
12824 return SE.getMonotonicPredicateType(AR, Pred) ==
12826}
12827
12828/// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max
12829/// expression?
12831 const SCEV *LHS, const SCEV *RHS) {
12832 switch (Pred) {
12833 default:
12834 return false;
12835
12836 case ICmpInst::ICMP_SGE:
12837 std::swap(LHS, RHS);
12838 [[fallthrough]];
12839 case ICmpInst::ICMP_SLE:
12840 return
12841 // min(A, ...) <= A
12843 // A <= max(A, ...)
12845
12846 case ICmpInst::ICMP_UGE:
12847 std::swap(LHS, RHS);
12848 [[fallthrough]];
12849 case ICmpInst::ICMP_ULE:
12850 return
12851 // min(A, ...) <= A
12852 // FIXME: what about umin_seq?
12854 // A <= max(A, ...)
12856
12857 case ICmpInst::ICMP_UGT:
12858 std::swap(LHS, RHS);
12859 [[fallthrough]];
12860 case ICmpInst::ICMP_ULT:
12861 // umin(Ops) u<= each Op, so proving Op u< RHS for any Op proves
12862 // umin(Ops) u< RHS.
12863 //
12864 // Use computeConstantDifference instead of the more powerful
12865 // isKnownPredicate to keep this check cheap: isKnownPredicateViaMinOrMax
12866 // is called from isKnownViaNonRecursiveReasoning, so recursing into
12867 // the full predicate prover would be expensive.
12868 if (const auto *Min = dyn_cast<SCEVUMinExpr>(LHS)) {
12869 for (SCEVUse Op : Min->operands()) {
12870 std::optional<APInt> Diff = SE.computeConstantDifference(RHS, Op);
12871 // When Op and RHS share a common base differing by a
12872 // constant offset D (RHS - Op = D), Op u< RHS holds iff D != 0 and
12873 // RHS >= D (unsigned), i.e. the subtraction doesn't underflow.
12874 if (Diff && !Diff->isZero() && SE.getUnsignedRangeMin(RHS).uge(*Diff))
12875 return true;
12876 }
12877 }
12878 return false;
12879 }
12880
12881 llvm_unreachable("covered switch fell through?!");
12882}
12883
12884bool ScalarEvolution::isImpliedViaOperations(CmpPredicate Pred, const SCEV *LHS,
12885 const SCEV *RHS,
12886 const SCEV *FoundLHS,
12887 const SCEV *FoundRHS,
12888 unsigned Depth) {
12891 "LHS and RHS have different sizes?");
12892 assert(getTypeSizeInBits(FoundLHS->getType()) ==
12893 getTypeSizeInBits(FoundRHS->getType()) &&
12894 "FoundLHS and FoundRHS have different sizes?");
12895 // We want to avoid hurting the compile time with analysis of too big trees.
12896 if (Depth > Opts.scalar_evolution_max_scev_operations_implication_depth)
12897 return false;
12898
12899 // We only want to work with GT comparison so far.
12900 if (ICmpInst::isLT(Pred)) {
12902 std::swap(LHS, RHS);
12903 std::swap(FoundLHS, FoundRHS);
12904 }
12905
12906 CmpInst::Predicate P = Pred.getPreferredSignedPredicate();
12907
12908 // For unsigned, try to reduce it to corresponding signed comparison.
12909 if (P == ICmpInst::ICMP_UGT)
12910 // We can replace unsigned predicate with its signed counterpart if all
12911 // involved values are non-negative.
12912 // TODO: We could have better support for unsigned.
12913 if (isKnownNonNegative(FoundLHS) && isKnownNonNegative(FoundRHS)) {
12914 // Knowing that both FoundLHS and FoundRHS are non-negative, and knowing
12915 // FoundLHS >u FoundRHS, we also know that FoundLHS >s FoundRHS. Let us
12916 // use this fact to prove that LHS and RHS are non-negative.
12917 const SCEV *MinusOne = getMinusOne(LHS->getType());
12918 if (isImpliedCondOperands(ICmpInst::ICMP_SGT, LHS, MinusOne, FoundLHS,
12919 FoundRHS) &&
12920 isImpliedCondOperands(ICmpInst::ICMP_SGT, RHS, MinusOne, FoundLHS,
12921 FoundRHS))
12923 }
12924
12925 if (P != ICmpInst::ICMP_SGT)
12926 return false;
12927
12928 auto GetOpFromSExt = [&](const SCEV *S) -> const SCEV * {
12929 if (auto *Ext = dyn_cast<SCEVSignExtendExpr>(S))
12930 return Ext->getOperand();
12931 // TODO: If S is a SCEVConstant then you can cheaply "strip" the sext off
12932 // the constant in some cases.
12933 return S;
12934 };
12935
12936 // Acquire values from extensions.
12937 auto *OrigLHS = LHS;
12938 auto *OrigFoundLHS = FoundLHS;
12939 LHS = GetOpFromSExt(LHS);
12940 FoundLHS = GetOpFromSExt(FoundLHS);
12941
12942 // Is the SGT predicate can be proved trivially or using the found context.
12943 auto IsSGTViaContext = [&](const SCEV *S1, const SCEV *S2) {
12944 return isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_SGT, S1, S2) ||
12945 isImpliedViaOperations(ICmpInst::ICMP_SGT, S1, S2, OrigFoundLHS,
12946 FoundRHS, Depth + 1);
12947 };
12948
12949 SCEVUse LL;
12950 SCEVUse LR;
12951 if (match(LHS, m_scev_NSWAdd(m_SCEV(LL), m_SCEV(LR)))) {
12952 // We want to avoid creation of any new non-constant SCEV. Since we are
12953 // going to compare the operands to RHS, we should be certain that we don't
12954 // need any size extensions for this. So let's decline all cases when the
12955 // sizes of types of LHS and RHS do not match.
12956 // TODO: Maybe try to get RHS from sext to catch more cases?
12958 return false;
12959
12960 auto *MinusOne = getMinusOne(RHS->getType());
12961
12962 // Checks that S1 >= 0 && S2 > RHS, trivially or using the found context.
12963 auto IsSumGreaterThanRHS = [&](const SCEV *S1, const SCEV *S2) {
12964 return IsSGTViaContext(S1, MinusOne) && IsSGTViaContext(S2, RHS);
12965 };
12966 // Try to prove the following rule:
12967 // (LHS = LL + LR) && (LL >= 0) && (LR > RHS) => (LHS > RHS).
12968 // (LHS = LL + LR) && (LR >= 0) && (LL > RHS) => (LHS > RHS).
12969 if (IsSumGreaterThanRHS(LL, LR) || IsSumGreaterThanRHS(LR, LL))
12970 return true;
12971 } else if (auto *LHSUnknownExpr = dyn_cast<SCEVUnknown>(LHS)) {
12972 Value *LL, *LR;
12973 // FIXME: Once we have SDiv implemented, we can get rid of this matching.
12974
12975 using namespace llvm::PatternMatch;
12976
12977 if (match(LHSUnknownExpr->getValue(), m_SDiv(m_Value(LL), m_Value(LR)))) {
12978 // Rules for division.
12979 // We are going to perform some comparisons with Denominator and its
12980 // derivative expressions. In general case, creating a SCEV for it may
12981 // lead to a complex analysis of the entire graph, and in particular it
12982 // can request trip count recalculation for the same loop. This would
12983 // cache as SCEVCouldNotCompute to avoid the infinite recursion. To avoid
12984 // this, we only want to create SCEVs that are constants in this section.
12985 // So we bail if Denominator is not a constant.
12986 if (!isa<ConstantInt>(LR))
12987 return false;
12988
12989 auto *Denominator = cast<SCEVConstant>(getSCEV(LR));
12990
12991 // We want to make sure that LHS = FoundLHS / Denominator. If it is so,
12992 // then a SCEV for the numerator already exists and matches with FoundLHS.
12993 auto *Numerator = getExistingSCEV(LL);
12994 if (!Numerator || Numerator->getType() != FoundLHS->getType())
12995 return false;
12996
12997 // Make sure that the numerator matches with FoundLHS and the denominator
12998 // is positive.
12999 if (!HasSameValue(Numerator, FoundLHS) || !isKnownPositive(Denominator))
13000 return false;
13001
13002 auto *DTy = Denominator->getType();
13003 auto *FRHSTy = FoundRHS->getType();
13004 if (DTy->isPointerTy() != FRHSTy->isPointerTy())
13005 // One of types is a pointer and another one is not. We cannot extend
13006 // them properly to a wider type, so let us just reject this case.
13007 // TODO: Usage of getEffectiveSCEVType for DTy, FRHSTy etc should help
13008 // to avoid this check.
13009 return false;
13010
13011 // Given that:
13012 // FoundLHS > FoundRHS, LHS = FoundLHS / Denominator, Denominator > 0.
13013 auto *WTy = getWiderType(DTy, FRHSTy);
13014 auto *DenominatorExt = getNoopOrSignExtend(Denominator, WTy);
13015 auto *FoundRHSExt = getNoopOrSignExtend(FoundRHS, WTy);
13016
13017 // Try to prove the following rule:
13018 // (FoundRHS > Denominator - 2) && (RHS <= 0) => (LHS > RHS).
13019 // For example, given that FoundLHS > 2. It means that FoundLHS is at
13020 // least 3. If we divide it by Denominator < 4, we will have at least 1.
13021 auto *DenomMinusTwo = getMinusSCEV(DenominatorExt, getConstant(WTy, 2));
13022 if (isKnownNonPositive(RHS) &&
13023 IsSGTViaContext(FoundRHSExt, DenomMinusTwo))
13024 return true;
13025
13026 // Try to prove the following rule:
13027 // (FoundRHS > -1 - Denominator) && (RHS < 0) => (LHS > RHS).
13028 // For example, given that FoundLHS > -3. Then FoundLHS is at least -2.
13029 // If we divide it by Denominator > 2, then:
13030 // 1. If FoundLHS is negative, then the result is 0.
13031 // 2. If FoundLHS is non-negative, then the result is non-negative.
13032 // Anyways, the result is non-negative.
13033 auto *MinusOne = getMinusOne(WTy);
13034 auto *NegDenomMinusOne = getMinusSCEV(MinusOne, DenominatorExt);
13035 if (isKnownNegative(RHS) &&
13036 IsSGTViaContext(FoundRHSExt, NegDenomMinusOne))
13037 return true;
13038 }
13039 }
13040
13041 // If our expression contained SCEVUnknown Phis, and we split it down and now
13042 // need to prove something for them, try to prove the predicate for every
13043 // possible incoming values of those Phis.
13044 if (isImpliedViaMerge(Pred, OrigLHS, RHS, OrigFoundLHS, FoundRHS, Depth + 1))
13045 return true;
13046
13047 return false;
13048}
13049
13051 const SCEV *RHS) {
13052 // zext x u<= sext x, sext x s<= zext x
13053 const SCEV *Op;
13054 switch (Pred) {
13055 case ICmpInst::ICMP_SGE:
13056 std::swap(LHS, RHS);
13057 [[fallthrough]];
13058 case ICmpInst::ICMP_SLE: {
13059 // If operand >=s 0 then ZExt == SExt. If operand <s 0 then SExt <s ZExt.
13060 return match(LHS, m_scev_SExt(m_SCEV(Op))) &&
13062 }
13063 case ICmpInst::ICMP_UGE:
13064 std::swap(LHS, RHS);
13065 [[fallthrough]];
13066 case ICmpInst::ICMP_ULE: {
13067 // If operand >=u 0 then ZExt == SExt. If operand <u 0 then ZExt <u SExt.
13068 return match(LHS, m_scev_ZExt(m_SCEV(Op))) &&
13070 }
13071 default:
13072 return false;
13073 };
13074 llvm_unreachable("unhandled case");
13075}
13076
13077bool ScalarEvolution::isKnownViaNonRecursiveReasoning(CmpPredicate Pred,
13078 SCEVUse LHS,
13079 SCEVUse RHS) {
13081 isKnownPredicateViaConstantRanges(Pred, LHS, RHS) ||
13085 isKnownPredicateViaNoOverflow(Pred, LHS, RHS);
13086}
13087
13088bool ScalarEvolution::isImpliedCondOperandsHelper(CmpPredicate Pred,
13089 const SCEV *LHS,
13090 const SCEV *RHS,
13091 const SCEV *FoundLHS,
13092 const SCEV *FoundRHS) {
13094 if (HasSameValue(LHS, FoundLHS) && HasSameValue(RHS, FoundRHS))
13095 return true;
13096 } else {
13098 if (isKnownViaNonRecursiveReasoning(NonStrictPred, LHS, FoundLHS) &&
13099 isKnownViaNonRecursiveReasoning(
13100 ICmpInst::getSwappedPredicate(NonStrictPred), RHS, FoundRHS))
13101 return true;
13102 }
13103
13104 // Maybe it can be proved via operations?
13105 return isImpliedViaOperations(Pred, LHS, RHS, FoundLHS, FoundRHS);
13106}
13107
13108bool ScalarEvolution::isImpliedCondOperandsViaRanges(
13109 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, CmpPredicate FoundPred,
13110 const SCEV *FoundLHS, const SCEV *FoundRHS) {
13111 if (!isa<SCEVConstant>(RHS) || !isa<SCEVConstant>(FoundRHS))
13112 // The restriction on `FoundRHS` be lifted easily -- it exists only to
13113 // reduce the compile time impact of this optimization.
13114 return false;
13115
13116 std::optional<APInt> Addend = computeConstantDifference(LHS, FoundLHS);
13117 if (!Addend)
13118 return false;
13119
13120 const APInt &ConstFoundRHS = cast<SCEVConstant>(FoundRHS)->getAPInt();
13121
13122 // `FoundLHSRange` is the range we know `FoundLHS` to be in by virtue of the
13123 // antecedent "`FoundLHS` `FoundPred` `FoundRHS`".
13124 ConstantRange FoundLHSRange =
13125 ConstantRange::makeExactICmpRegion(FoundPred, ConstFoundRHS);
13126
13127 // Since `LHS` is `FoundLHS` + `Addend`, we can compute a range for `LHS`:
13128 ConstantRange LHSRange = FoundLHSRange.add(ConstantRange(*Addend));
13129
13130 // We can also compute the range of values for `LHS` that satisfy the
13131 // consequent, "`LHS` `Pred` `RHS`":
13132 const APInt &ConstRHS = cast<SCEVConstant>(RHS)->getAPInt();
13133 // The antecedent implies the consequent if every value of `LHS` that
13134 // satisfies the antecedent also satisfies the consequent.
13135 return LHSRange.icmp(Pred, ConstRHS);
13136}
13137
13138bool ScalarEvolution::canIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride,
13139 bool IsSigned, bool Invert) {
13140 assert(isKnownPositive(Stride) && "Positive stride expected!");
13141
13142 unsigned BitWidth = getTypeSizeInBits(RHS->getType());
13143 const SCEV *One = getOne(Stride->getType());
13144
13145 if (IsSigned) {
13146 APInt MaxRHS = getRangeMax(RHS, /*IsSigned=*/true, Invert);
13147 APInt MaxValue = APInt::getSignedMaxValue(BitWidth);
13148 APInt MaxStrideMinusOne = getSignedRangeMax(getMinusSCEV(Stride, One));
13149
13150 // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow!
13151 return (std::move(MaxValue) - MaxStrideMinusOne).slt(MaxRHS);
13152 }
13153
13154 APInt MaxRHS = getRangeMax(RHS, /*IsSigned=*/false, Invert);
13155 APInt MaxValue = APInt::getMaxValue(BitWidth);
13156 APInt MaxStrideMinusOne = getUnsignedRangeMax(getMinusSCEV(Stride, One));
13157
13158 // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow!
13159 return (std::move(MaxValue) - MaxStrideMinusOne).ult(MaxRHS);
13160}
13161
13163 // umin(N, 1) + floor((N - umin(N, 1)) / D)
13164 // This is equivalent to "1 + floor((N - 1) / D)" for N != 0. The umin
13165 // expression fixes the case of N=0.
13166 const SCEV *MinNOne = getUMinExpr(N, getOne(N->getType()));
13167 const SCEV *NMinusOne = getMinusSCEV(N, MinNOne);
13168 return getAddExpr(MinNOne, getUDivExpr(NMinusOne, D));
13169}
13170
13171const SCEV *
13172ScalarEvolution::computeMaxBECountForLT(const SCEV *Start, const SCEV *Stride,
13173 const SCEV *End, unsigned BitWidth,
13174 bool IsSigned, bool Invert) {
13175 // The logic in this function assumes we can represent a positive stride.
13176 // If we can't, the backedge-taken count must be zero.
13177 if (IsSigned && BitWidth == 1)
13178 return getZero(Stride->getType());
13179
13180 // This code below only been closely audited for negative strides in the
13181 // unsigned comparison case, it may be correct for signed comparison, but
13182 // that needs to be established.
13183 if (IsSigned && isKnownNegative(Stride))
13184 return getCouldNotCompute();
13185
13186 // Calculate the maximum backedge count based on the range of values
13187 // permitted by Start, End, and Stride. If Invert is true, both Start and End
13188 // need inverting. Stride was already negated by the caller.
13189 APInt MinStart = getRangeMin(Start, IsSigned, Invert);
13190
13191 APInt MinStride =
13192 IsSigned ? getSignedRangeMin(Stride) : getUnsignedRangeMin(Stride);
13193
13194 // We assume either the stride is positive, or the backedge-taken count
13195 // is zero. So force StrideForMaxBECount to be at least one.
13196 APInt One(BitWidth, 1);
13197 APInt StrideForMaxBECount = IsSigned ? APIntOps::smax(One, MinStride)
13198 : APIntOps::umax(One, MinStride);
13199
13200 APInt MaxValue = IsSigned ? APInt::getSignedMaxValue(BitWidth)
13201 : APInt::getMaxValue(BitWidth);
13202 APInt Limit = MaxValue - (StrideForMaxBECount - 1);
13203
13204 // Although End can be a MAX expression we estimate MaxEnd considering only
13205 // the case End = RHS of the loop termination condition. This is safe because
13206 // in the other case (End - Start) is zero, leading to a zero maximum backedge
13207 // taken count.
13208 APInt MaxEnd = getRangeMax(End, IsSigned, Invert);
13209 MaxEnd =
13210 IsSigned ? APIntOps::smin(MaxEnd, Limit) : APIntOps::umin(MaxEnd, Limit);
13211
13212 // MaxBECount = ceil((max(MaxEnd, MinStart) - MinStart) / Stride)
13213 MaxEnd = IsSigned ? APIntOps::smax(MaxEnd, MinStart)
13214 : APIntOps::umax(MaxEnd, MinStart);
13215
13216 APInt Delta = MaxEnd - MinStart;
13217
13218 // Try to refine Delta in case End - Start (or Start - End if Invert) gives a
13219 // tighter bound after folding.
13220 const SCEV *DeltaExpr =
13221 Invert ? getMinusSCEV(Start, End) : getMinusSCEV(End, Start);
13222 Delta = APIntOps::umin(Delta, getUnsignedRangeMax(DeltaExpr));
13223
13224 return getUDivCeilSCEV(getConstant(Delta), getConstant(StrideForMaxBECount));
13225}
13226
13228ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
13229 const Loop *L, bool IsSigned, bool Invert,
13230 bool ControlsOnlyExit, bool AllowPredicates) {
13232
13233 LoopGuards Guards = LoopGuards::collectOnDemand(L, *this);
13234
13235 // FIXME: Extend the non-invariant RHS analysis to greater-than comparisons.
13236 if (Invert && !isLoopInvariant(RHS, L))
13237 return getCouldNotCompute();
13238
13240 bool PredicatedIV = false;
13241 // FIXME: Generalize the NUW inference below to decreasing IVs.
13242 if (!IV && !Invert) {
13243 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(LHS)) {
13244 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(ZExt->getOperand());
13245 if (AR && AR->getLoop() == L && AR->isAffine()) {
13246 auto canProveNUW = [&]() {
13247 // We can use the comparison to infer no-wrap flags only if it fully
13248 // controls the loop exit.
13249 if (!ControlsOnlyExit)
13250 return false;
13251
13252 if (!isLoopInvariant(RHS, L))
13253 return false;
13254
13255 if (!isKnownNonZero(AR->getStepRecurrence(*this)))
13256 // We need the sequence defined by AR to strictly increase in the
13257 // unsigned integer domain for the logic below to hold.
13258 return false;
13259
13260 const unsigned InnerBitWidth = getTypeSizeInBits(AR->getType());
13261 const unsigned OuterBitWidth = getTypeSizeInBits(RHS->getType());
13262 // If RHS <=u Limit, then there must exist a value V in the sequence
13263 // defined by AR (e.g. {Start,+,Step}) such that V >u RHS, and
13264 // V <=u UINT_MAX. Thus, we must exit the loop before unsigned
13265 // overflow occurs. This limit also implies that a signed comparison
13266 // (in the wide bitwidth) is equivalent to an unsigned comparison as
13267 // the high bits on both sides must be zero.
13268 APInt StrideMax = getUnsignedRangeMax(AR->getStepRecurrence(*this));
13269 APInt Limit = APInt::getMaxValue(InnerBitWidth) - (StrideMax - 1);
13270 Limit = Limit.zext(OuterBitWidth);
13271 return getUnsignedRangeMax(applyLoopGuards(RHS, Guards)).ule(Limit);
13272 };
13273 auto Flags = AR->getNoWrapFlags();
13274 if (!hasFlags(Flags, SCEV::FlagNUW) && canProveNUW())
13275 Flags = setFlags(Flags, SCEV::FlagNUW);
13276
13277 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), Flags);
13278 if (AR->hasNoUnsignedWrap()) {
13279 // Emulate what getZeroExtendExpr would have done during construction
13280 // if we'd been able to infer the fact just above at that time.
13281 const SCEV *Step = AR->getStepRecurrence(*this);
13282 Type *Ty = ZExt->getType();
13283 const SCEV *S = getAddRecExpr(
13285 getZeroExtendExpr(Step, Ty, 0), L, AR->getNoWrapFlags());
13287 }
13288 }
13289 }
13290 }
13291
13292 if (!IV && AllowPredicates) {
13293 // Try to make this an AddRec using runtime tests, in the first X
13294 // iterations of this loop, where X is the SCEV expression found by the
13295 // algorithm below.
13296 IV = convertSCEVToAddRecWithPredicates(LHS, L, Predicates);
13297 PredicatedIV = true;
13298 }
13299
13300 // Avoid weird loops
13301 if (!IV || IV->getLoop() != L || !IV->isAffine())
13302 return getCouldNotCompute();
13303
13304 // A precondition of this method is that the condition being analyzed
13305 // reaches an exiting branch which dominates the latch. Given that, we can
13306 // assume that an increment which violates the nowrap specification and
13307 // produces poison must cause undefined behavior when the resulting poison
13308 // value is branched upon and thus we can conclude that the backedge is
13309 // taken no more often than would be required to produce that poison value.
13310 // Note that a well defined loop can exit on the iteration which violates
13311 // the nowrap specification if there is another exit (either explicit or
13312 // implicit/exceptional) which causes the loop to execute before the
13313 // exiting instruction we're analyzing would trigger UB.
13314 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13315 bool NoWrap = ControlsOnlyExit && any(IV->getNoWrapFlags(WrapType));
13316 // Reverse the ordering for greater-than comparisons.
13318 if (Invert)
13320
13321 // The step of ~IV is the negated step of IV.
13322 const SCEV *Stride = IV->getStepRecurrence(*this);
13323 if (Invert)
13324 Stride = getNegativeSCEV(Stride);
13325 const SCEV *GuardedStride = Stride;
13326
13327 // Whether the IV may reach the maximum (or minimum if inverted) value
13328 // before the exit is taken.
13329 bool IVMayOverflow = true;
13330
13331 bool PositiveStride = isKnownPositive(Stride);
13332 // A dominating guard may prove the stride positive.
13333 if (!PositiveStride) {
13334 const SCEV *LoopGuardedStride = applyLoopGuards(Stride, Guards);
13335 if (isKnownPositive(LoopGuardedStride)) {
13336 GuardedStride = LoopGuardedStride;
13337 PositiveStride = true;
13338 // Encode the context-sensitive stride > 0 fact into the expression
13339 Stride = getUMaxExpr(Stride, getOne(Stride->getType()));
13340 }
13341 }
13342
13343 // Avoid negative or zero stride values.
13344 if (!PositiveStride) {
13345 // FIXME: Generalize the unknown-stride analysis to decreasing IVs.
13346 if (Invert)
13347 return getCouldNotCompute();
13348
13349 // We can compute the correct backedge taken count for loops with unknown
13350 // strides if we can prove that the loop is not an infinite loop with side
13351 // effects. Here's the loop structure we are trying to handle -
13352 //
13353 // i = start
13354 // do {
13355 // A[i] = i;
13356 // i += s;
13357 // } while (i < end);
13358 //
13359 // The backedge taken count for such loops is evaluated as -
13360 // (max(end, start + stride) - start - 1) /u stride
13361 //
13362 // The additional preconditions that we need to check to prove correctness
13363 // of the above formula is as follows -
13364 //
13365 // a) IV is either nuw or nsw depending upon signedness (indicated by the
13366 // NoWrap flag).
13367 // b) the loop is guaranteed to be finite (e.g. is mustprogress and has
13368 // b) the loop is guaranteed to be finite (e.g. is mustprogress and has
13369 // no side effects within the loop) or a predicate is added to ensure
13370 // stride is positive.
13371 // c) loop has a single static exit (with no abnormal exits)
13372 //
13373 // Precondition a) implies that if the stride is negative, this is a single
13374 // trip loop. The backedge taken count formula reduces to zero in this case.
13375 //
13376 // Precondition b) and c) combine to imply that if rhs is invariant in L,
13377 // then a zero stride means the backedge can't be taken without executing
13378 // undefined behavior.
13379 //
13380 // The positive stride case is the same as isKnownPositive(Stride) returning
13381 // true (original behavior of the function).
13382 //
13383 if (PredicatedIV || !NoWrap || !loopHasNoAbnormalExits(L))
13384 return getCouldNotCompute();
13385
13386 if (!loopIsFiniteByAssumption(L)) {
13387 // If the loop may be infinite, add a predicate ensuring Stride is
13388 // positive, to guarantee forward progress.
13389 if (!AllowPredicates || !isLoopInvariant(Stride, L))
13390 return getCouldNotCompute();
13391
13392 const SCEV *Zero = getZero(Stride->getType());
13393 const SCEVPredicate *P =
13395 Predicates.push_back(P);
13396 // When the predicate holds (Stride > 0), umax(Stride, 1) == Stride,
13397 // so the result is unchanged. To prevent div by zero.
13398 Stride = getUMaxExpr(Stride, getOne(Stride->getType()));
13399 } else if (!isKnownNonZero(Stride)) {
13400 // If we have a step of zero, and RHS isn't invariant in L, we don't know
13401 // if it might eventually be greater than start and if so, on which
13402 // iteration. We can't even produce a useful upper bound.
13403 if (!isLoopInvariant(RHS, L))
13404 return getCouldNotCompute();
13405
13406 // We allow a potentially zero stride, but we need to divide by stride
13407 // below. Since the loop can't be infinite and this check must control
13408 // the sole exit, we can infer the exit must be taken on the first
13409 // iteration (e.g. backedge count = 0) if the stride is zero. Given that,
13410 // we know the numerator in the divides below must be zero, so we can
13411 // pick an arbitrary non-zero value for the denominator (e.g. stride)
13412 // and produce the right result.
13413 // FIXME: Handle the case where Stride is poison?
13414 auto wouldZeroStrideBeUB = [&]() {
13415 // Proof by contradiction. Suppose the stride were zero. If we can
13416 // prove that the backedge *is* taken on the first iteration, then since
13417 // we know this condition controls the sole exit, we must have an
13418 // infinite loop. We can't have a (well defined) infinite loop per
13419 // check just above.
13420 // Note: The (Start - Stride) term is used to get the start' term from
13421 // (start' + stride,+,stride). Remember that we only care about the
13422 // result of this expression when stride == 0 at runtime.
13423 auto *StartIfZero = getMinusSCEV(IV->getStart(), Stride);
13424 return isLoopEntryGuardedByCond(L, Cond, StartIfZero, RHS);
13425 };
13426 if (!wouldZeroStrideBeUB()) {
13427 Stride = getUMaxExpr(Stride, getOne(Stride->getType()));
13428 }
13429 }
13430 } else {
13431 // Avoid proven overflow cases: this will ensure that the backedge taken
13432 // count will not generate any unsigned overflow.
13433 IVMayOverflow = canIVOverflowOnLT(RHS, GuardedStride, IsSigned, Invert) &&
13434 canIVOverflowOnLT(applyLoopGuards(RHS, Guards),
13435 GuardedStride, IsSigned, Invert);
13436 if (IVMayOverflow && !NoWrap)
13437 return getCouldNotCompute();
13438 }
13439
13440 // On all paths just preceeding, we established the following invariant:
13441 // IV can be assumed not to overflow up to and including the exiting
13442 // iteration. We proved this in one of two ways:
13443 // 1) We can show overflow doesn't occur before the exiting iteration
13444 // 1a) canIVOverflowOnLT, and b) step of one
13445 // 2) We can show that if overflow occurs, the loop must execute UB
13446 // before any possible exit.
13447 // Note that we have not yet proved RHS invariant (in general).
13448
13449 const SCEV *Start = IV->getStart();
13450
13451 // Preserve pointer-typed Start/RHS to pass to isLoopEntryGuardedByCond.
13452 // If we convert to integers, isLoopEntryGuardedByCond will miss some cases.
13453 // Use integer-typed versions for actual computation; we can't subtract
13454 // pointers in general.
13455 const SCEV *OrigStart = Start;
13456 const SCEV *OrigRHS = RHS;
13457 if (Start->getType()->isPointerTy()) {
13458 Start = getPtrToAddrExpr(Start);
13459 if (isa<SCEVCouldNotCompute>(Start))
13460 return Start;
13461 }
13462 if (RHS->getType()->isPointerTy()) {
13465 return RHS;
13466 }
13467
13468 const SCEV *End = nullptr, *BECount = getCouldNotCompute(),
13469 *BECountIfBackedgeTaken = getCouldNotCompute();
13470 if (!isLoopInvariant(RHS, L)) {
13471 assert(!Invert && "RHS must be loop-invariant for Invert");
13472 const auto *RHSAddRec = dyn_cast<SCEVAddRecExpr>(RHS);
13473 if (PositiveStride && RHSAddRec != nullptr && RHSAddRec->getLoop() == L &&
13474 any(RHSAddRec->getNoWrapFlags())) {
13475 // The structure of loop we are trying to calculate backedge count of:
13476 //
13477 // left = left_start
13478 // right = right_start
13479 //
13480 // while(left < right){
13481 // ... do something here ...
13482 // left += s1; // stride of left is s1 (s1 > 0)
13483 // right += s2; // stride of right is s2 (s2 < 0)
13484 // }
13485 //
13486
13487 const SCEV *RHSStart = RHSAddRec->getStart();
13488 const SCEV *RHSStride = RHSAddRec->getStepRecurrence(*this);
13489
13490 // If Stride - RHSStride is positive and does not overflow, we can write
13491 // backedge count as ->
13492 // ceil((End - Start) /u (Stride - RHSStride))
13493 // Where, End = max(RHSStart, Start)
13494
13495 // Check if RHSStride < 0 and Stride - RHSStride will not overflow.
13496 if (isKnownNegative(RHSStride) &&
13497 willNotOverflow(Instruction::Sub, /*Signed=*/true, Stride,
13498 RHSStride)) {
13499
13500 const SCEV *Denominator = getMinusSCEV(Stride, RHSStride);
13501 if (isKnownPositive(Denominator)) {
13502 End = IsSigned ? getSMaxExpr(RHSStart, Start)
13503 : getUMaxExpr(RHSStart, Start);
13504
13505 // We can do this because End >= Start, as End = max(RHSStart, Start)
13506 const SCEV *Delta = getMinusSCEV(End, Start);
13507
13508 BECount = getUDivCeilSCEV(Delta, Denominator);
13509 BECountIfBackedgeTaken =
13510 getUDivCeilSCEV(getMinusSCEV(RHSStart, Start), Denominator);
13511 }
13512 }
13513 }
13514 } else {
13515 // Let End = max(RHS,Start). We use the expression (End-Start)/Stride to
13516 // describe the backedge count: if the backedge is taken at least once then
13517 // End is RHS, and if not End is Start so we get a backedge count of zero.
13518 // Inverted, End is min(RHS, Start).
13519 //
13520 // AddingStrideMinusOneMayOverflow has the following preconditions:
13521 //
13522 // 1. Start <= End, signed if IsSigned (inverted: End <= Start)
13523 // 2. The index variable doesn't overflow.
13524 //
13525 // Therefore, we know N exists such that
13526 // (Start + Stride * N) >= End, and computing "(Start + Stride * N)"
13527 // doesn't overflow.
13528 //
13529 // Using this information, try to prove whether the addition in
13530 // "(End - Start) + (Stride - 1)" has unsigned overflow.
13531 //
13532 // If the IV cannot overflow, RHS is at least Stride - 1 below the maximum
13533 // value, so the distance End - Start is at most UMAX - (Stride - 1) and
13534 // the (Stride - 1) addition below cannot overflow.
13535 const SCEV *One = getOne(Stride->getType());
13536 bool AddingStrideMinusOneMayOverflow = IVMayOverflow && [&] {
13537 if (isKnownToBeAPowerOfTwo(Stride)) {
13538 // Suppose Stride is a power of two, and Start/End are unsigned
13539 // integers. Let UMAX be the largest representable unsigned
13540 // integer.
13541 //
13542 // By the preconditions of this function, we know
13543 // "(Start + Stride * N) >= End", and this doesn't overflow.
13544 // As a formula:
13545 //
13546 // End <= (Start + Stride * N) <= UMAX
13547 //
13548 // Subtracting Start from all the terms:
13549 //
13550 // End - Start <= Stride * N <= UMAX - Start
13551 //
13552 // Since Start is unsigned, UMAX - Start <= UMAX. Therefore:
13553 //
13554 // End - Start <= Stride * N <= UMAX
13555 //
13556 // Stride * N is a multiple of Stride. Therefore,
13557 //
13558 // End - Start <= Stride * N <= UMAX - (UMAX mod Stride)
13559 //
13560 // Since Stride is a power of two, UMAX + 1 is divisible by
13561 // Stride. Therefore, UMAX mod Stride == Stride - 1. So we can
13562 // write:
13563 //
13564 // End - Start <= Stride * N <= UMAX - Stride - 1
13565 //
13566 // Dropping the middle term:
13567 //
13568 // End - Start <= UMAX - Stride - 1
13569 //
13570 // Adding Stride - 1 to both sides:
13571 //
13572 // (End - Start) + (Stride - 1) <= UMAX
13573 //
13574 // In other words, the addition doesn't have unsigned overflow.
13575 //
13576 // A similar proof works if we treat Start/End as signed values.
13577 // Just rewrite steps before "End - Start <= Stride * N <= UMAX"
13578 // to use signed max instead of unsigned max. Note that we're
13579 // trying to prove a lack of unsigned overflow in either case.
13580 // Inverted: "Start - End <= Stride * N <= Start - MIN <= UMAX", same.
13581 return false;
13582 }
13583 if (!Invert && (Start == Stride || Start == getMinusSCEV(Stride, One))) {
13584 // If Start is equal to Stride, (End - Start) + (Stride - 1) == End
13585 // - 1. If !IsSigned, 0 <u Stride == Start <=u End; so 0 <u End - 1
13586 // <u End. If IsSigned, 0 <s Stride == Start <=s End; so 0 <s End -
13587 // 1 <s End.
13588 //
13589 // If Start is equal to Stride - 1, (End - Start) + Stride - 1 ==
13590 // End.
13591 //
13592 // Both need Start to be the smaller value, so neither applies inverted.
13593 return false;
13594 }
13595 return true;
13596 }();
13597
13598 // If inverted, the analyzed values are complements: "~V - Offset" is "~(V +
13599 // Offset)" and "~To - ~From" is "From - To".
13600 auto StepBack = [&](const SCEV *V, const SCEV *Offset) -> const SCEV * {
13601 if (Invert)
13602 return getAddExpr(V, Offset);
13603 return getMinusSCEV(V, Offset);
13604 };
13605 auto Distance = [&](const SCEV *From, const SCEV *To) {
13606 return Invert ? getMinusSCEV(From, To) : getMinusSCEV(To, From);
13607 };
13608
13609 const SCEV *OrigPrevStart = StepBack(OrigStart, Stride);
13610 assert(isAvailableAtLoopEntry(OrigPrevStart, L) && "Must be!");
13611 assert(isAvailableAtLoopEntry(OrigStart, L) && "Must be!");
13612 assert(isAvailableAtLoopEntry(OrigRHS, L) && "Must be!");
13613 // Can we prove Start - Stride < RHS, and either Start - Stride < Start or
13614 // (via !AddingStrideMinusOneMayOverflow) that (RHS - Start) + (Stride - 1)
13615 // does not overflow?
13616 if ((!AddingStrideMinusOneMayOverflow ||
13617 isLoopEntryGuardedByCond(L, Cond, OrigPrevStart, OrigStart)) &&
13618 isLoopEntryGuardedByCond(L, Cond, OrigPrevStart, OrigRHS)) {
13619 // In this case, we can use a refined formula for computing backedge
13620 // taken count. The general formula remains:
13621 // "End-Start /uceiling Stride"
13622 // We want to use the alternate formula:
13623 // "((RHS - 1) - (Start - Stride)) /u Stride"
13624 // Let's do a quick case analysis to show these are equivalent under
13625 // our preconditions. When inverted, the proof uses complemented Start,
13626 // RHS and End; Stride remains positive.
13627 // * For RHS <= Start (End is Start), the backedge-taken count must be
13628 // zero. Together with the precondition "Start - Stride < RHS", we have
13629 // "Start - Stride < RHS <= Start". Subtracting Start - Stride from
13630 // all sides we get "0 < RHS - (Start - Stride) <= Stride".
13631 // Subtracting 1 we get "0 <= (RHS - 1) - (Start - Stride) < Stride".
13632 // So dividing that by Stride gives zero.
13633 //
13634 // * For RHS > Start (End is RHS), the backedge count must be
13635 // "RHS-Start /uceil Stride", so it is sufficient to show that the
13636 // numerator "((RHS - 1) - (Start - Stride))" does not overflow.
13637 //
13638 // If "Start - Stride < Start" holds, we have
13639 // "RHS > Start > Start - Stride". As such
13640 // "RHS - (Start - Stride) - 1" does not overflow, which is the
13641 // reassociated numerator.
13642 //
13643 // Otherwise !AddingStrideMinusOneMayOverflow guarantees that
13644 // "(End - Start) + (Stride - 1)" does not overflow unsigned. Here
13645 // "End" is "RHS", as "RHS > Start", so this is the reassociated
13646 // numerator. Neither sub-term wraps unsigned: "RHS - Start"
13647 // due to "RHS > Start", and "Stride - 1", as Stride is non-zero.
13648 const SCEV *Numerator =
13649 getMinusSCEV(Distance(StepBack(Start, Stride), RHS), One);
13650 BECount = getUDivExpr(Numerator, Stride);
13651 }
13652
13653 if (isa<SCEVCouldNotCompute>(BECount)) {
13654 auto canProveRHSIsAtOrBeyondStart = [&]() {
13655 // Inverted, the claim is "Start >= RHS". Reverse the comparisons below
13656 // by swapping their operands rather than their predicates:
13657 // isLoopEntryGuardedByCond is sensitive to operand order and loses the
13658 // proof if the IV bound moves to the other side.
13659 auto SwapIfInverted = [&](const SCEV *A, const SCEV *B) {
13660 return Invert ? std::pair(B, A) : std::pair(A, B);
13661 };
13662
13663 auto CondGE = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13664 const SCEV *GuardedRHS = applyLoopGuards(OrigRHS, Guards);
13665 const SCEV *GuardedStart = applyLoopGuards(OrigStart, Guards);
13666 if (Invert)
13667 std::swap(GuardedRHS, GuardedStart);
13668
13669 auto [GELHS, GERHS] = SwapIfInverted(OrigRHS, OrigStart);
13670 if (isLoopEntryGuardedByCond(L, CondGE, GELHS, GERHS) ||
13671 isKnownPredicate(CondGE, GuardedRHS, GuardedStart))
13672 return true;
13673
13674 if ((GuardedRHS != GELHS || GuardedStart != GERHS) &&
13675 isKnownPredicate(CondGE, GELHS, GERHS))
13676 return true;
13677
13678 // (RHS > Start - 1) implies RHS >= Start.
13679 // * "RHS >= Start" is trivially equivalent to "RHS > Start - 1" if
13680 // "Start - 1" doesn't overflow.
13681 // * For signed comparison, if Start - 1 does overflow, it's equal
13682 // to INT_MAX, and "RHS >s INT_MAX" is trivially false.
13683 // * For unsigned comparison, if Start - 1 does overflow, it's equal
13684 // to UINT_MAX, and "RHS >u UINT_MAX" is trivially false.
13685 //
13686 // FIXME: Should isLoopEntryGuardedByCond do this for us?
13687 auto CondGT = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13688 auto [GTLHS, GTRHS] = SwapIfInverted(OrigRHS, StepBack(OrigStart, One));
13689 return isLoopEntryGuardedByCond(L, CondGT, GTLHS, GTRHS);
13690 };
13691
13692 // If we know that RHS >= Start in the context of loop, then we know
13693 // that max(RHS, Start) = RHS at this point.
13694 if (canProveRHSIsAtOrBeyondStart()) {
13695 End = RHS;
13696 } else {
13697 // If RHS < Start, the backedge will be taken zero times. So in
13698 // general, we can write the backedge-taken count as:
13699 //
13700 // RHS >= Start ? ceil(RHS - Start) / Stride : 0
13701 //
13702 // We convert it to the following to make it more convenient for SCEV:
13703 //
13704 // ceil(max(RHS, Start) - Start) / Stride
13705 //
13706 // Inverted, this is ceil(Start - min(RHS, Start)) / Stride.
13707 if (Invert)
13708 End = IsSigned ? getSMinExpr(RHS, Start) : getUMinExpr(RHS, Start);
13709 else
13710 End = IsSigned ? getSMaxExpr(RHS, Start) : getUMaxExpr(RHS, Start);
13711
13712 // See what would happen if we assume the backedge is taken. This is
13713 // used to compute MaxBECount.
13714 BECountIfBackedgeTaken = getUDivCeilSCEV(Distance(Start, RHS), Stride);
13715 }
13716
13717 const SCEV *Delta = Distance(Start, End);
13718 if (!AddingStrideMinusOneMayOverflow) {
13719 // floor((D + (S - 1)) / S)
13720 // We prefer this formulation if it's legal because it's fewer
13721 // operations.
13722 BECount =
13723 getUDivExpr(getAddExpr(Delta, getMinusSCEV(Stride, One)), Stride);
13724 } else {
13725 BECount = getUDivCeilSCEV(Delta, Stride);
13726 }
13727 }
13728 }
13729
13730 const SCEV *ConstantMaxBECount;
13731 bool MaxOrZero = false;
13732 if (isa<SCEVConstant>(BECount)) {
13733 ConstantMaxBECount = BECount;
13734 } else {
13735 ConstantMaxBECount = computeMaxBECountForLT(
13736 Start, Stride, RHS, getTypeSizeInBits(LHS->getType()), IsSigned,
13737 Invert);
13738 // If we know exactly how many times the backedge will be taken if it's
13739 // taken at least once, then the backedge count will either be that or
13740 // zero. If that count exceeds the range-based bound, the backedge can
13741 // never be taken.
13742 const APInt *IfTaken, *RangeMax;
13743 if (match(BECountIfBackedgeTaken, m_scev_APInt(IfTaken))) {
13744 if (match(ConstantMaxBECount, m_scev_APInt(RangeMax)) &&
13745 IfTaken->ugt(*RangeMax)) {
13746 ConstantMaxBECount = getZero(BECountIfBackedgeTaken->getType());
13747 } else {
13748 ConstantMaxBECount = BECountIfBackedgeTaken;
13749 MaxOrZero = true;
13750 }
13751 }
13752 }
13753
13754 if (isa<SCEVCouldNotCompute>(ConstantMaxBECount) &&
13755 !isa<SCEVCouldNotCompute>(BECount))
13756 ConstantMaxBECount = getConstant(getUnsignedRangeMax(BECount));
13757
13758 const SCEV *SymbolicMaxBECount =
13759 isa<SCEVCouldNotCompute>(BECount) ? ConstantMaxBECount : BECount;
13760 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, MaxOrZero,
13761 Predicates);
13762}
13763
13765 ScalarEvolution &SE) const {
13766 if (Range.isFullSet()) // Infinite loop.
13767 return SE.getCouldNotCompute();
13768
13769 // If the start is a non-zero constant, shift the range to simplify things.
13770 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(getStart()))
13771 if (!SC->getValue()->isZero()) {
13773 Operands[0] = SE.getZero(SC->getType());
13774 const SCEV *Shifted = SE.getAddRecExpr(Operands, getLoop(),
13776 if (const auto *ShiftedAddRec = dyn_cast<SCEVAddRecExpr>(Shifted))
13777 return ShiftedAddRec->getNumIterationsInRange(
13778 Range.subtract(SC->getAPInt()), SE);
13779 // This is strange and shouldn't happen.
13780 return SE.getCouldNotCompute();
13781 }
13782
13783 // The only time we can solve this is when we have all constant indices.
13784 // Otherwise, we cannot determine the overflow conditions.
13786 return SE.getCouldNotCompute();
13787
13788 // Okay at this point we know that all elements of the chrec are constants and
13789 // that the start element is zero.
13790
13791 // First check to see if the range contains zero. If not, the first
13792 // iteration exits.
13793 unsigned BitWidth = SE.getTypeSizeInBits(getType());
13794 if (!Range.contains(APInt(BitWidth, 0)))
13795 return SE.getZero(getType());
13796
13797 if (isAffine()) {
13798 // If this is an affine expression then we have this situation:
13799 // Solve {0,+,A} in Range === Ax in Range
13800
13801 // We know that zero is in the range. If A is positive then we know that
13802 // the upper value of the range must be the first possible exit value.
13803 // If A is negative then the lower of the range is the last possible loop
13804 // value. Also note that we already checked for a full range.
13805 APInt A = cast<SCEVConstant>(getOperand(1))->getAPInt();
13806 APInt End = A.sge(1) ? (Range.getUpper() - 1) : Range.getLower();
13807
13808 // The exit value should be (End+A)/A.
13809 APInt ExitVal = (End + A).udiv(A);
13810 ConstantInt *ExitValue = ConstantInt::get(SE.getContext(), ExitVal);
13811
13812 // Evaluate at the exit value. If we really did fall out of the valid
13813 // range, then we computed our trip count, otherwise wrap around or other
13814 // things must have happened.
13815 ConstantInt *Val = EvaluateConstantChrecAtConstant(this, ExitValue, SE);
13816 if (Range.contains(Val->getValue()))
13817 return SE.getCouldNotCompute(); // Something strange happened
13818
13819 // Ensure that the previous value is in the range.
13820 assert(Range.contains(
13822 ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) &&
13823 "Linear scev computation is off in a bad way!");
13824 return SE.getConstant(ExitValue);
13825 }
13826
13827 if (isQuadratic()) {
13828 if (auto S = SolveQuadraticAddRecRange(this, Range, SE))
13829 return SE.getConstant(*S);
13830 }
13831
13832 return SE.getCouldNotCompute();
13833}
13834
13835const SCEVAddRecExpr *
13837 assert(getNumOperands() > 1 && "AddRec with zero step?");
13838 // There is a temptation to just call getAddExpr(this, getStepRecurrence(SE)),
13839 // but in this case we cannot guarantee that the value returned will be an
13840 // AddRec because SCEV does not have a fixed point where it stops
13841 // simplification: it is legal to return ({rec1} + {rec2}). For example, it
13842 // may happen if we reach arithmetic depth limit while simplifying. So we
13843 // construct the returned value explicitly.
13845 // If this is {A,+,B,+,C,...,+,N}, then its step is {B,+,C,+,...,+,N}, and
13846 // (this + Step) is {A+B,+,B+C,+...,+,N}.
13847 for (unsigned i = 0, e = getNumOperands() - 1; i < e; ++i)
13848 Ops.push_back(SE.getAddExpr(getOperand(i), getOperand(i + 1)));
13849 // We know that the last operand is not a constant zero (otherwise it would
13850 // have been popped out earlier). This guarantees us that if the result has
13851 // the same last operand, then it will also not be popped out, meaning that
13852 // the returned value will be an AddRec.
13853 const SCEV *Last = getOperand(getNumOperands() - 1);
13854 assert(!Last->isZero() && "Recurrency with zero step?");
13855 Ops.push_back(Last);
13857}
13858
13859// Return true when S contains at least an undef value.
13861 return SCEVExprContains(
13862 S, [](const SCEV *S) { return match(S, m_scev_UndefOrPoison()); });
13863}
13864
13865// Return true when S contains a value that is a nullptr.
13867 return SCEVExprContains(S, [](const SCEV *S) {
13868 if (const auto *SU = dyn_cast<SCEVUnknown>(S))
13869 return SU->getValue() == nullptr;
13870 return false;
13871 });
13872}
13873
13874/// Return the size of an element read or written by Inst.
13876 if (!isa<LoadInst, StoreInst>(Inst))
13877 return nullptr;
13879 return getSizeOfExpr(ETy, getLoadStoreType(Inst));
13880}
13881
13882//===----------------------------------------------------------------------===//
13883// SCEVCallbackVH Class Implementation
13884//===----------------------------------------------------------------------===//
13885
13887 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
13888 if (PHINode *PN = dyn_cast<PHINode>(getValPtr()))
13889 SE->ConstantEvolutionLoopExitValue.erase(PN);
13890 SE->eraseValueFromMap(getValPtr());
13891 // this now dangles!
13892}
13893
13894void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) {
13895 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
13896
13897 // Forget all the expressions associated with users of the old value,
13898 // so that future queries will recompute the expressions using the new
13899 // value.
13900 SE->forgetValue(getValPtr());
13901 // this now dangles!
13902}
13903
13904ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se)
13905 : CallbackVH(V), SE(se) {}
13906
13907//===----------------------------------------------------------------------===//
13908// ScalarEvolution Class Implementation
13909//===----------------------------------------------------------------------===//
13910
13913 LoopInfo &LI)
13914 : Opts(getAnalysisOptions(F)), F(F), DL(F.getDataLayout()), TLI(TLI),
13915 AC(AC), DT(DT), LI(LI), CouldNotCompute(new SCEVCouldNotCompute()),
13916 ValuesAtScopes(64), LoopDispositions(64), BlockDispositions(64) {
13917 // To use guards for proving predicates, we need to scan every instruction in
13918 // relevant basic blocks, and not just terminators. Doing this is a waste of
13919 // time if the IR does not actually contain any calls to
13920 // @llvm.experimental.guard, so do a quick check and remember this beforehand.
13921 //
13922 // This pessimizes the case where a pass that preserves ScalarEvolution wants
13923 // to _add_ guards to the module when there weren't any before, and wants
13924 // ScalarEvolution to optimize based on those guards. For now we prefer to be
13925 // efficient in lieu of being smart in that rather obscure case.
13926
13927 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
13928 F.getParent(), Intrinsic::experimental_guard);
13929 HasGuards = GuardDecl && !GuardDecl->use_empty();
13930}
13931
13933 : Opts(Arg.Opts), F(Arg.F), DL(Arg.DL), HasGuards(Arg.HasGuards),
13934 TLI(Arg.TLI), AC(Arg.AC), DT(Arg.DT), LI(Arg.LI),
13935 CouldNotCompute(std::move(Arg.CouldNotCompute)),
13936 ValueExprMap(std::move(Arg.ValueExprMap)),
13937 PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)),
13938 PendingMerges(std::move(Arg.PendingMerges)),
13939 ConstantMultipleCache(std::move(Arg.ConstantMultipleCache)),
13940 BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)),
13941 PredicatedBackedgeTakenCounts(
13942 std::move(Arg.PredicatedBackedgeTakenCounts)),
13943 BECountUsers(std::move(Arg.BECountUsers)),
13944 ConstantEvolutionLoopExitValue(
13945 std::move(Arg.ConstantEvolutionLoopExitValue)),
13946 ValuesAtScopes(std::move(Arg.ValuesAtScopes)),
13947 ValuesAtScopesUsers(std::move(Arg.ValuesAtScopesUsers)),
13948 LoopDispositions(std::move(Arg.LoopDispositions)),
13949 LoopPropertiesCache(std::move(Arg.LoopPropertiesCache)),
13950 BlockDispositions(std::move(Arg.BlockDispositions)),
13951 SCEVUsers(std::move(Arg.SCEVUsers)),
13952 UnsignedRanges(std::move(Arg.UnsignedRanges)),
13953 SignedRanges(std::move(Arg.SignedRanges)),
13954 UniqueSCEVs(std::move(Arg.UniqueSCEVs)),
13955 UniquePreds(std::move(Arg.UniquePreds)),
13956 SCEVAllocator(std::move(Arg.SCEVAllocator)),
13957 ConstantSCEVs(std::move(Arg.ConstantSCEVs)),
13958 LoopUsers(std::move(Arg.LoopUsers)),
13959 PredicatedSCEVRewrites(std::move(Arg.PredicatedSCEVRewrites)),
13960 FirstUnknown(Arg.FirstUnknown),
13961 NextUnknownOrderID(Arg.NextUnknownOrderID) {
13962 Arg.FirstUnknown = nullptr;
13963}
13964
13966 // Iterate through all the SCEVUnknown instances and call their
13967 // destructors, so that they release their references to their values.
13968 for (SCEVUnknown *U = FirstUnknown; U;) {
13969 SCEVUnknown *Tmp = U;
13970 U = U->Next;
13971 Tmp->~SCEVUnknown();
13972 }
13973 FirstUnknown = nullptr;
13974
13975 ExprValueMap.clear();
13976 ValueExprMap.clear();
13977 HasRecMap.clear();
13978 BackedgeTakenCounts.clear();
13979 PredicatedBackedgeTakenCounts.clear();
13980
13981 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage");
13982 assert(PendingMerges.empty() && "isImpliedViaMerge garbage");
13983 assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!");
13984 assert(!ProvingSplitPredicate && "ProvingSplitPredicate garbage!");
13985}
13986
13990
13991/// When printing a top-level SCEV for trip counts, it's helpful to include
13992/// a type for constants which are otherwise hard to disambiguate.
13993static void PrintSCEVWithTypeHint(raw_ostream &OS, const SCEV* S) {
13994 if (isa<SCEVConstant>(S))
13995 OS << *S->getType() << " ";
13996 OS << *S;
13997}
13998
14000 const Loop *L) {
14001 // Print all inner loops first
14002 for (Loop *I : *L)
14003 PrintLoopInfo(OS, SE, I);
14004
14005 OS << "Loop ";
14006 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
14007 OS << ": ";
14008
14009 SmallVector<BasicBlock *, 8> ExitingBlocks;
14010 L->getExitingBlocks(ExitingBlocks);
14011 if (ExitingBlocks.size() != 1)
14012 OS << "<multiple exits> ";
14013
14014 auto *BTC = SE->getBackedgeTakenCount(L);
14015 if (!isa<SCEVCouldNotCompute>(BTC)) {
14016 OS << "backedge-taken count is ";
14017 PrintSCEVWithTypeHint(OS, BTC);
14018 } else
14019 OS << "Unpredictable backedge-taken count.";
14020 OS << "\n";
14021
14022 if (ExitingBlocks.size() > 1)
14023 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14024 OS << " exit count for " << ExitingBlock->getName() << ": ";
14025 const SCEV *EC = SE->getExitCount(L, ExitingBlock);
14026 PrintSCEVWithTypeHint(OS, EC);
14027 if (isa<SCEVCouldNotCompute>(EC)) {
14028 // Retry with predicates.
14030 EC = SE->getPredicatedExitCount(L, ExitingBlock, &Predicates);
14031 if (!isa<SCEVCouldNotCompute>(EC)) {
14032 OS << "\n predicated exit count for " << ExitingBlock->getName()
14033 << ": ";
14034 PrintSCEVWithTypeHint(OS, EC);
14035 OS << "\n Predicates:\n";
14036 for (const auto *P : Predicates)
14037 P->print(OS, 4);
14038 }
14039 }
14040 OS << "\n";
14041 }
14042
14043 OS << "Loop ";
14044 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
14045 OS << ": ";
14046
14047 auto *ConstantBTC = SE->getConstantMaxBackedgeTakenCount(L);
14048 if (!isa<SCEVCouldNotCompute>(ConstantBTC)) {
14049 OS << "constant max backedge-taken count is ";
14050 PrintSCEVWithTypeHint(OS, ConstantBTC);
14052 OS << ", actual taken count either this or zero.";
14053 } else {
14054 OS << "Unpredictable constant max backedge-taken count. ";
14055 }
14056
14057 OS << "\n"
14058 "Loop ";
14059 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
14060 OS << ": ";
14061
14062 auto *SymbolicBTC = SE->getSymbolicMaxBackedgeTakenCount(L);
14063 if (!isa<SCEVCouldNotCompute>(SymbolicBTC)) {
14064 OS << "symbolic max backedge-taken count is ";
14065 PrintSCEVWithTypeHint(OS, SymbolicBTC);
14067 OS << ", actual taken count either this or zero.";
14068 } else {
14069 OS << "Unpredictable symbolic max backedge-taken count. ";
14070 }
14071 OS << "\n";
14072
14073 if (ExitingBlocks.size() > 1)
14074 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14075 OS << " symbolic max exit count for " << ExitingBlock->getName() << ": ";
14076 auto *ExitBTC = SE->getExitCount(L, ExitingBlock,
14078 PrintSCEVWithTypeHint(OS, ExitBTC);
14079 if (isa<SCEVCouldNotCompute>(ExitBTC)) {
14080 // Retry with predicates.
14082 ExitBTC = SE->getPredicatedExitCount(L, ExitingBlock, &Predicates,
14084 if (!isa<SCEVCouldNotCompute>(ExitBTC)) {
14085 OS << "\n predicated symbolic max exit count for "
14086 << ExitingBlock->getName() << ": ";
14087 PrintSCEVWithTypeHint(OS, ExitBTC);
14088 OS << "\n Predicates:\n";
14089 for (const auto *P : Predicates)
14090 P->print(OS, 4);
14091 }
14092 }
14093 OS << "\n";
14094 }
14095
14097 auto *PBT = SE->getPredicatedBackedgeTakenCount(L, Preds);
14098 if (PBT != BTC) {
14099 OS << "Loop ";
14100 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
14101 OS << ": ";
14102 if (!isa<SCEVCouldNotCompute>(PBT)) {
14103 OS << "Predicated backedge-taken count is ";
14104 PrintSCEVWithTypeHint(OS, PBT);
14105 } else
14106 OS << "Unpredictable predicated backedge-taken count.";
14107 OS << "\n";
14108 OS << " Predicates:\n";
14109 for (const auto *P : Preds)
14110 P->print(OS, 4);
14111 }
14112 Preds.clear();
14113
14114 auto *PredConstantMax =
14116 if (PredConstantMax != ConstantBTC) {
14117 OS << "Loop ";
14118 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
14119 OS << ": ";
14120 if (!isa<SCEVCouldNotCompute>(PredConstantMax)) {
14121 OS << "Predicated constant max backedge-taken count is ";
14122 PrintSCEVWithTypeHint(OS, PredConstantMax);
14123 } else
14124 OS << "Unpredictable predicated constant max backedge-taken count.";
14125 OS << "\n";
14126 OS << " Predicates:\n";
14127 for (const auto *P : Preds)
14128 P->print(OS, 4);
14129 }
14130 Preds.clear();
14131
14132 auto *PredSymbolicMax =
14134 if (SymbolicBTC != PredSymbolicMax) {
14135 OS << "Loop ";
14136 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
14137 OS << ": ";
14138 if (!isa<SCEVCouldNotCompute>(PredSymbolicMax)) {
14139 OS << "Predicated symbolic max backedge-taken count is ";
14140 PrintSCEVWithTypeHint(OS, PredSymbolicMax);
14141 } else
14142 OS << "Unpredictable predicated symbolic max backedge-taken count.";
14143 OS << "\n";
14144 OS << " Predicates:\n";
14145 for (const auto *P : Preds)
14146 P->print(OS, 4);
14147 }
14148
14150 OS << "Loop ";
14151 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
14152 OS << ": ";
14153 OS << "Trip multiple is " << SE->getSmallConstantTripMultiple(L) << "\n";
14154 }
14155}
14156
14157namespace llvm {
14158// Note: these overloaded operators need to be in the llvm namespace for them
14159// to be resolved correctly. If we put them outside the llvm namespace, the
14160//
14161// OS << ": " << SE.getLoopDisposition(SV, InnerL);
14162//
14163// code below "breaks" and start printing raw enum values as opposed to the
14164// string values.
14167 switch (LD) {
14169 OS << "Variant";
14170 break;
14172 OS << "Invariant";
14173 break;
14175 OS << "Uniform";
14176 break;
14178 OS << "Computable";
14179 break;
14180 }
14181 return OS;
14182}
14183
14186 switch (BD) {
14188 OS << "DoesNotDominate";
14189 break;
14191 OS << "Dominates";
14192 break;
14194 OS << "ProperlyDominates";
14195 break;
14196 }
14197 return OS;
14198}
14199} // namespace llvm
14200
14202 // ScalarEvolution's implementation of the print method is to print
14203 // out SCEV values of all instructions that are interesting. Doing
14204 // this potentially causes it to create new SCEV objects though,
14205 // which technically conflicts with the const qualifier. This isn't
14206 // observable from outside the class though, so casting away the
14207 // const isn't dangerous.
14208 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14209
14210 if (Opts.scalar_evolution_classify_expressions) {
14211 OS << "Classifying expressions for: ";
14212 F.printAsOperand(OS, /*PrintType=*/false);
14213 OS << "\n";
14214 for (Instruction &I : instructions(F))
14215 if (isSCEVable(I.getType()) && !isa<CmpInst>(I)) {
14216 OS << I << '\n';
14217 OS << " --> ";
14218 const SCEV *SV = SE.getSCEV(&I);
14219 SV->print(OS);
14220 if (!isa<SCEVCouldNotCompute>(SV)) {
14221 OS << " U: ";
14222 SE.getUnsignedRange(SV).print(OS);
14223 OS << " S: ";
14224 SE.getSignedRange(SV).print(OS);
14225 }
14226
14227 const Loop *L = LI.getLoopFor(I.getParent());
14228
14229 SCEVUse AtUse = SE.getSCEVAtScope(SV, L);
14230 if (AtUse != SV) {
14231 OS << " --> ";
14232 OS << AtUse;
14233 if (!isa<SCEVCouldNotCompute>(AtUse)) {
14234 OS << " U: ";
14235 SE.getUnsignedRange(AtUse).print(OS);
14236 OS << " S: ";
14237 SE.getSignedRange(AtUse).print(OS);
14238 }
14239 }
14240
14241 if (L) {
14242 OS << "\t\t" "Exits: ";
14243 SCEVUse ExitValue = SE.getSCEVAtScope(SV, L->getParentLoop());
14244 if (!SE.isLoopInvariant(ExitValue, L)) {
14245 OS << "<<Unknown>>";
14246 } else {
14247 OS << ExitValue;
14248 }
14249
14250 ListSeparator LS(", ", "\t\tLoopDispositions: { ");
14251 for (const auto *Iter = L; Iter; Iter = Iter->getParentLoop()) {
14252 OS << LS;
14253 Iter->getHeader()->printAsOperand(OS, /*PrintType=*/false);
14254 OS << ": " << SE.getLoopDisposition(SV, Iter);
14255 }
14256
14257 for (const auto *InnerL : depth_first(L)) {
14258 if (InnerL == L)
14259 continue;
14260 OS << LS;
14261 InnerL->getHeader()->printAsOperand(OS, /*PrintType=*/false);
14262 OS << ": " << SE.getLoopDisposition(SV, InnerL);
14263 }
14264
14265 OS << " }";
14266 }
14267
14268 OS << "\n";
14269 }
14270 }
14271
14272 OS << "Determining loop execution counts for: ";
14273 F.printAsOperand(OS, /*PrintType=*/false);
14274 OS << "\n";
14275 for (Loop *I : LI)
14276 PrintLoopInfo(OS, &SE, I);
14277}
14278
14281 if (isa<SCEVConstant>(S))
14282 return LoopInvariant;
14283
14284 auto &Values = LoopDispositions[S];
14285 for (auto &V : Values) {
14286 if (V.getPointer() == L)
14287 return V.getInt();
14288 }
14289 Values.emplace_back(L, LoopVariant);
14290 LoopDisposition D = computeLoopDisposition(S, L);
14291 auto &Values2 = LoopDispositions[S];
14292 for (auto &V : llvm::reverse(Values2)) {
14293 if (V.getPointer() == L) {
14294 V.setInt(D);
14295 break;
14296 }
14297 }
14298 return D;
14299}
14300
14302ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) {
14303 switch (S->getSCEVType()) {
14304 case scConstant:
14305 case scVScale:
14306 return LoopInvariant;
14307 case scAddRecExpr: {
14308 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S);
14309
14310 // If L is the addrec's loop, it's computable.
14311 if (AR->getLoop() == L)
14312 return LoopComputable;
14313
14314 // Add recurrences are never invariant in the function-body (null loop).
14315 if (!L)
14316 return LoopVariant;
14317
14318 // Everything that is not defined at loop entry is variant.
14319 if (DT.dominates(L->getHeader(), AR->getLoop()->getHeader())) {
14320 if (L->contains(AR->getLoop()) &&
14321 llvm::all_of(AR->operands(),
14322 [&](const SCEV *Op) { return isLoopUniform(Op, L); }))
14323 return LoopUniform;
14324
14325 return LoopVariant;
14326 }
14327 assert(!L->contains(AR->getLoop()) && "Containing loop's header does not"
14328 " dominate the contained loop's header?");
14329
14330 // This recurrence is invariant w.r.t. L if AR's loop contains L.
14331 if (AR->getLoop()->contains(L))
14332 return LoopInvariant;
14333
14334 // This recurrence is variant w.r.t. L if any of its operands
14335 // are variant.
14336 for (SCEVUse Op : AR->operands())
14337 if (!isLoopInvariant(Op, L))
14338 return LoopVariant;
14339
14340 // Otherwise it's loop-invariant.
14341 return LoopInvariant;
14342 }
14343 case scTruncate:
14344 case scZeroExtend:
14345 case scSignExtend:
14346 case scPtrToAddr:
14347 case scAddExpr:
14348 case scMulExpr:
14349 case scUDivExpr:
14350 case scUMaxExpr:
14351 case scSMaxExpr:
14352 case scUMinExpr:
14353 case scSMinExpr:
14354 case scSequentialUMinExpr: {
14355 bool HasVarying = false;
14356 bool HasUniform = false;
14357 for (SCEVUse Op : S->operands()) {
14359 if (D == LoopVariant)
14360 return LoopVariant;
14361 if (D == LoopComputable)
14362 HasVarying = true;
14363 if (D == LoopUniform)
14364 HasUniform = true;
14365 }
14366 return HasVarying ? (HasUniform ? LoopVariant : LoopComputable)
14367 : (HasUniform ? LoopUniform : LoopInvariant);
14368 }
14369 case scUnknown:
14370 // All non-instruction values are loop invariant. All instructions are loop
14371 // invariant if they are not contained in the specified loop.
14372 // Instructions are never considered invariant in the function body
14373 // (null loop) because they are defined within the "loop".
14375 return (L && !L->contains(I)) ? LoopInvariant : LoopVariant;
14376 return LoopInvariant;
14377 case scCouldNotCompute:
14378 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14379 }
14380 llvm_unreachable("Unknown SCEV kind!");
14381}
14382
14383bool ScalarEvolution::isLoopUniform(const SCEV *S, const Loop *L) {
14385 return D == LoopUniform || D == LoopInvariant;
14386}
14387
14389 return getLoopDisposition(S, L) == LoopInvariant;
14390}
14391
14393 return getLoopDisposition(S, L) == LoopComputable;
14394}
14395
14398 if (isa<SCEVConstant>(S))
14400
14401 auto &Values = BlockDispositions[S];
14402 for (auto &V : Values) {
14403 if (V.getPointer() == BB)
14404 return V.getInt();
14405 }
14406 Values.emplace_back(BB, DoesNotDominateBlock);
14407 BlockDisposition D = computeBlockDisposition(S, BB);
14408 auto &Values2 = BlockDispositions[S];
14409 for (auto &V : llvm::reverse(Values2)) {
14410 if (V.getPointer() == BB) {
14411 V.setInt(D);
14412 break;
14413 }
14414 }
14415 return D;
14416}
14417
14419ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14420 switch (S->getSCEVType()) {
14421 case scConstant:
14422 case scVScale:
14424 case scAddRecExpr: {
14425 // This uses a "dominates" query instead of "properly dominates" query
14426 // to test for proper dominance too, because the instruction which
14427 // produces the addrec's value is a PHI, and a PHI effectively properly
14428 // dominates its entire containing block.
14429 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S);
14430 if (!DT.dominates(AR->getLoop()->getHeader(), BB))
14431 return DoesNotDominateBlock;
14432
14433 // Fall through into SCEVNAryExpr handling.
14434 [[fallthrough]];
14435 }
14436 case scTruncate:
14437 case scZeroExtend:
14438 case scSignExtend:
14439 case scPtrToAddr:
14440 case scAddExpr:
14441 case scMulExpr:
14442 case scUDivExpr:
14443 case scUMaxExpr:
14444 case scSMaxExpr:
14445 case scUMinExpr:
14446 case scSMinExpr:
14447 case scSequentialUMinExpr: {
14448 bool Proper = true;
14449 for (const SCEV *NAryOp : S->operands()) {
14451 if (D == DoesNotDominateBlock)
14452 return DoesNotDominateBlock;
14453 if (D == DominatesBlock)
14454 Proper = false;
14455 }
14456 return Proper ? ProperlyDominatesBlock : DominatesBlock;
14457 }
14458 case scUnknown:
14459 if (Instruction *I =
14461 if (I->getParent() == BB)
14462 return DominatesBlock;
14463 if (DT.properlyDominates(I->getParent(), BB))
14465 return DoesNotDominateBlock;
14466 }
14468 case scCouldNotCompute:
14469 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14470 }
14471 llvm_unreachable("Unknown SCEV kind!");
14472}
14473
14474bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) {
14475 return getBlockDisposition(S, BB) >= DominatesBlock;
14476}
14477
14480}
14481
14482void ScalarEvolution::forgetBackedgeTakenCounts(const Loop *L,
14483 bool Predicated) {
14484 auto &BECounts =
14485 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14486 auto It = BECounts.find(L);
14487 if (It != BECounts.end()) {
14488 for (const ExitNotTakenInfo &ENT : It->second.ExitNotTaken) {
14489 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14490 if (!isa<SCEVConstant>(S)) {
14491 auto UserIt = BECountUsers.find(S);
14492 assert(UserIt != BECountUsers.end());
14493 UserIt->second.erase({L, Predicated});
14494 }
14495 }
14496 }
14497 BECounts.erase(It);
14498 }
14499}
14500
14501void ScalarEvolution::forgetMemoizedResults(ArrayRef<SCEVUse> SCEVs) {
14502 SmallPtrSet<const SCEV *, 8> ToForget(llvm::from_range, SCEVs);
14503 SmallVector<SCEVUse, 8> Worklist(ToForget.begin(), ToForget.end());
14504
14505 while (!Worklist.empty()) {
14506 const SCEV *Curr = Worklist.pop_back_val();
14507 auto Users = SCEVUsers.find(Curr);
14508 if (Users != SCEVUsers.end())
14509 for (const auto *User : Users->second)
14510 if (ToForget.insert(User).second)
14511 Worklist.push_back(User);
14512 }
14513
14514 for (const auto *S : ToForget)
14515 forgetMemoizedResultsImpl(S);
14516
14517 PredicatedSCEVRewrites.remove_if(
14518 [&](const auto &Entry) { return ToForget.count(Entry.first.first); });
14519}
14520
14521void ScalarEvolution::forgetMemoizedResultsImpl(const SCEV *S) {
14522 LoopDispositions.erase(S);
14523 BlockDispositions.erase(S);
14524 UnsignedRanges.erase(S);
14525 SignedRanges.erase(S);
14526 HasRecMap.erase(S);
14527 ConstantMultipleCache.erase(S);
14528
14529 if (auto *AR = dyn_cast<SCEVAddRecExpr>(S)) {
14530 UnsignedWrapViaInductionTried.erase(AR);
14531 SignedWrapViaInductionTried.erase(AR);
14532 }
14533
14534 auto ExprIt = ExprValueMap.find(S);
14535 if (ExprIt != ExprValueMap.end()) {
14536 for (Value *V : ExprIt->second) {
14537 auto ValueIt = ValueExprMap.find_as(V);
14538 if (ValueIt != ValueExprMap.end())
14539 ValueExprMap.erase(ValueIt);
14540 }
14541 ExprValueMap.erase(ExprIt);
14542 }
14543
14544 auto ScopeIt = ValuesAtScopes.find(S);
14545 if (ScopeIt != ValuesAtScopes.end()) {
14546 for (const auto &Pair : ScopeIt->second)
14547 if (!isa_and_nonnull<SCEVConstant>(Pair.second))
14548 llvm::erase(ValuesAtScopesUsers[Pair.second.getPointer()],
14549 std::make_pair(Pair.first, S));
14550 ValuesAtScopes.erase(ScopeIt);
14551 }
14552
14553 auto ScopeUserIt = ValuesAtScopesUsers.find(S);
14554 if (ScopeUserIt != ValuesAtScopesUsers.end()) {
14555 for (const auto &Pair : ScopeUserIt->second)
14556 // The recorded value at scope is a use of S, which may carry no-wrap
14557 // flags that are not part of this key.
14558 llvm::erase_if(ValuesAtScopes[Pair.second], [&](const auto &LS) {
14559 return LS.first == Pair.first && LS.second.getPointer() == S;
14560 });
14561 ValuesAtScopesUsers.erase(ScopeUserIt);
14562 }
14563
14564 auto BEUsersIt = BECountUsers.find(S);
14565 if (BEUsersIt != BECountUsers.end()) {
14566 // Work on a copy, as forgetBackedgeTakenCounts() will modify the original.
14567 auto Copy = BEUsersIt->second;
14568 for (const auto &Pair : Copy)
14569 forgetBackedgeTakenCounts(Pair.getPointer(), Pair.getInt());
14570 BECountUsers.erase(BEUsersIt);
14571 }
14572
14573 auto FoldUser = FoldCacheUser.find(S);
14574 if (FoldUser != FoldCacheUser.end()) {
14575 for (auto &KV : FoldUser->second)
14576 FoldCache.erase(KV);
14577 FoldCacheUser.erase(FoldUser);
14578 }
14579}
14580
14581void
14582ScalarEvolution::getUsedLoops(const SCEV *S,
14583 SmallPtrSetImpl<const Loop *> &LoopsUsed) {
14584 struct FindUsedLoops {
14585 FindUsedLoops(SmallPtrSetImpl<const Loop *> &LoopsUsed)
14586 : LoopsUsed(LoopsUsed) {}
14587 SmallPtrSetImpl<const Loop *> &LoopsUsed;
14588 bool follow(const SCEV *S) {
14589 if (auto *AR = dyn_cast<SCEVAddRecExpr>(S))
14590 LoopsUsed.insert(AR->getLoop());
14591 return true;
14592 }
14593
14594 bool isDone() const { return false; }
14595 };
14596
14597 FindUsedLoops F(LoopsUsed);
14598 SCEVTraversal<FindUsedLoops>(F).visitAll(S);
14599}
14600
14601void ScalarEvolution::getReachableBlocks(
14604 Worklist.push_back(&F.getEntryBlock());
14605 while (!Worklist.empty()) {
14606 BasicBlock *BB = Worklist.pop_back_val();
14607 if (!Reachable.insert(BB).second)
14608 continue;
14609
14610 Value *Cond;
14611 BasicBlock *TrueBB, *FalseBB;
14612 if (match(BB->getTerminator(), m_Br(m_Value(Cond), m_BasicBlock(TrueBB),
14613 m_BasicBlock(FalseBB)))) {
14614 if (auto *C = dyn_cast<ConstantInt>(Cond)) {
14615 Worklist.push_back(C->isOne() ? TrueBB : FalseBB);
14616 continue;
14617 }
14618
14619 if (auto *Cmp = dyn_cast<ICmpInst>(Cond)) {
14620 const SCEV *L = getSCEV(Cmp->getOperand(0));
14621 const SCEV *R = getSCEV(Cmp->getOperand(1));
14622 if (isKnownPredicateViaConstantRanges(Cmp->getCmpPredicate(), L, R)) {
14623 Worklist.push_back(TrueBB);
14624 continue;
14625 }
14626 if (isKnownPredicateViaConstantRanges(Cmp->getInverseCmpPredicate(), L,
14627 R)) {
14628 Worklist.push_back(FalseBB);
14629 continue;
14630 }
14631 }
14632 }
14633
14634 append_range(Worklist, successors(BB));
14635 }
14636}
14637
14639 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14640 ScalarEvolution SE2(F, TLI, AC, DT, LI);
14641
14642 SmallVector<Loop *, 8> LoopStack(LI.begin(), LI.end());
14643
14644 // Map's SCEV expressions from one ScalarEvolution "universe" to another.
14645 struct SCEVMapper : public SCEVRewriteVisitor<SCEVMapper> {
14646 SCEVMapper(ScalarEvolution &SE) : SCEVRewriteVisitor<SCEVMapper>(SE) {}
14647
14648 const SCEV *visitConstant(const SCEVConstant *Constant) {
14649 return SE.getConstant(Constant->getAPInt());
14650 }
14651
14652 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
14653 return SE.getUnknown(Expr->getValue());
14654 }
14655
14656 const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) {
14657 return SE.getCouldNotCompute();
14658 }
14659 };
14660
14661 SCEVMapper SCM(SE2);
14662 SmallPtrSet<BasicBlock *, 16> ReachableBlocks;
14663 SE2.getReachableBlocks(ReachableBlocks, F);
14664
14665 auto GetDelta = [&](const SCEV *Old, const SCEV *New) -> const SCEV * {
14666 if (containsUndefs(Old) || containsUndefs(New)) {
14667 // SCEV treats "undef" as an unknown but consistent value (i.e. it does
14668 // not propagate undef aggressively). This means we can (and do) fail
14669 // verification in cases where a transform makes a value go from "undef"
14670 // to "undef+1" (say). The transform is fine, since in both cases the
14671 // result is "undef", but SCEV thinks the value increased by 1.
14672 return nullptr;
14673 }
14674
14675 // Unless -verify-scev-strict is set, we only compare constant deltas.
14676 const SCEV *Delta = SE2.getMinusSCEV(Old, New);
14677 if (!Opts.verify_scev_strict && !isa<SCEVConstant>(Delta))
14678 return nullptr;
14679
14680 return Delta;
14681 };
14682
14683 while (!LoopStack.empty()) {
14684 auto *L = LoopStack.pop_back_val();
14685 llvm::append_range(LoopStack, *L);
14686
14687 // Only verify BECounts in reachable loops. For an unreachable loop,
14688 // any BECount is legal.
14689 if (!ReachableBlocks.contains(L->getHeader()))
14690 continue;
14691
14692 // Only verify cached BECounts. Computing new BECounts may change the
14693 // results of subsequent SCEV uses.
14694 auto It = BackedgeTakenCounts.find(L);
14695 if (It == BackedgeTakenCounts.end())
14696 continue;
14697
14698 auto *CurBECount =
14699 SCM.visit(It->second.getExact(L, const_cast<ScalarEvolution *>(this)));
14700 auto *NewBECount = SE2.getBackedgeTakenCount(L);
14701
14702 if (CurBECount == SE2.getCouldNotCompute() ||
14703 NewBECount == SE2.getCouldNotCompute()) {
14704 // NB! This situation is legal, but is very suspicious -- whatever pass
14705 // change the loop to make a trip count go from could not compute to
14706 // computable or vice-versa *should have* invalidated SCEV. However, we
14707 // choose not to assert here (for now) since we don't want false
14708 // positives.
14709 continue;
14710 }
14711
14712 if (SE.getTypeSizeInBits(CurBECount->getType()) >
14713 SE.getTypeSizeInBits(NewBECount->getType()))
14714 NewBECount = SE2.getZeroExtendExpr(NewBECount, CurBECount->getType());
14715 else if (SE.getTypeSizeInBits(CurBECount->getType()) <
14716 SE.getTypeSizeInBits(NewBECount->getType()))
14717 CurBECount = SE2.getZeroExtendExpr(CurBECount, NewBECount->getType());
14718
14719 const SCEV *Delta = GetDelta(CurBECount, NewBECount);
14720 if (Delta && !Delta->isZero()) {
14721 dbgs() << "Trip Count for " << *L << " Changed!\n";
14722 dbgs() << "Old: " << *CurBECount << "\n";
14723 dbgs() << "New: " << *NewBECount << "\n";
14724 dbgs() << "Delta: " << *Delta << "\n";
14725 std::abort();
14726 }
14727 }
14728
14729 // Collect all valid loops currently in LoopInfo.
14730 SmallPtrSet<Loop *, 32> ValidLoops;
14731 SmallVector<Loop *, 32> Worklist(LI.begin(), LI.end());
14732 while (!Worklist.empty()) {
14733 Loop *L = Worklist.pop_back_val();
14734 if (ValidLoops.insert(L).second)
14735 Worklist.append(L->begin(), L->end());
14736 }
14737 for (const auto &KV : ValueExprMap) {
14738#ifndef NDEBUG
14739 // Check for SCEV expressions referencing invalid/deleted loops.
14740 if (auto *AR = dyn_cast<SCEVAddRecExpr>(KV.second)) {
14741 assert(ValidLoops.contains(AR->getLoop()) &&
14742 "AddRec references invalid loop");
14743 }
14744#endif
14745
14746 // Check that the value is also part of the reverse map.
14747 auto It = ExprValueMap.find(KV.second);
14748 if (It == ExprValueMap.end() || !is_contained(It->second, KV.first)) {
14749 dbgs() << "Value " << *KV.first
14750 << " is in ValueExprMap but not in ExprValueMap\n";
14751 std::abort();
14752 }
14753
14754 if (auto *I = dyn_cast<Instruction>(&*KV.first)) {
14755 if (!ReachableBlocks.contains(I->getParent()))
14756 continue;
14757 const SCEV *OldSCEV = SCM.visit(KV.second);
14758 const SCEV *NewSCEV = SE2.getSCEV(I);
14759 const SCEV *Delta = GetDelta(OldSCEV, NewSCEV);
14760 if (Delta && !Delta->isZero()) {
14761 dbgs() << "SCEV for value " << *I << " changed!\n"
14762 << "Old: " << *OldSCEV << "\n"
14763 << "New: " << *NewSCEV << "\n"
14764 << "Delta: " << *Delta << "\n";
14765 std::abort();
14766 }
14767 }
14768 }
14769
14770 for (const auto &KV : ExprValueMap) {
14771 for (Value *V : KV.second) {
14772 const SCEV *S = ValueExprMap.lookup(V);
14773 if (!S) {
14774 dbgs() << "Value " << *V
14775 << " is in ExprValueMap but not in ValueExprMap\n";
14776 std::abort();
14777 }
14778 if (S != KV.first) {
14779 dbgs() << "Value " << *V << " mapped to " << *S << " rather than "
14780 << *KV.first << "\n";
14781 std::abort();
14782 }
14783 }
14784 }
14785
14786 // Verify integrity of SCEV users.
14787 for (const auto &S : UniqueSCEVs) {
14788 for (SCEVUse Op : S.operands()) {
14789 // We do not store dependencies of constants.
14790 if (isa<SCEVConstant>(Op))
14791 continue;
14792 auto It = SCEVUsers.find(Op);
14793 if (It != SCEVUsers.end() && It->second.count(&S))
14794 continue;
14795 dbgs() << "Use of operand " << *Op << " by user " << S
14796 << " is not being tracked!\n";
14797 std::abort();
14798 }
14799 }
14800
14801 // Verify integrity of ValuesAtScopes users.
14802 for (const auto &ValueAndVec : ValuesAtScopes) {
14803 const SCEV *Value = ValueAndVec.first;
14804 for (const auto &LoopAndValueAtScope : ValueAndVec.second) {
14805 const Loop *L = LoopAndValueAtScope.first;
14806 SCEVUse ValueAtScope = LoopAndValueAtScope.second;
14807 if (!isa<SCEVConstant>(ValueAtScope)) {
14808 auto It = ValuesAtScopesUsers.find(ValueAtScope.getPointer());
14809 if (It != ValuesAtScopesUsers.end() &&
14810 is_contained(It->second, std::make_pair(L, Value)))
14811 continue;
14812 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14813 << *ValueAtScope << " missing in ValuesAtScopesUsers\n";
14814 std::abort();
14815 }
14816 }
14817 }
14818
14819 for (const auto &ValueAtScopeAndVec : ValuesAtScopesUsers) {
14820 const SCEV *ValueAtScope = ValueAtScopeAndVec.first;
14821 for (const auto &LoopAndValue : ValueAtScopeAndVec.second) {
14822 const Loop *L = LoopAndValue.first;
14823 const SCEV *Value = LoopAndValue.second;
14825 auto It = ValuesAtScopes.find(Value);
14826 // The recorded value at scope may carry no-wrap flags that are not part
14827 // of the key it is recorded under.
14828 if (It != ValuesAtScopes.end() && any_of(It->second, [&](const auto &LS) {
14829 return LS.first == L && LS.second.getPointer() == ValueAtScope;
14830 }))
14831 continue;
14832 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14833 << *ValueAtScope << " missing in ValuesAtScopes\n";
14834 std::abort();
14835 }
14836 }
14837
14838 // Verify integrity of BECountUsers.
14839 auto VerifyBECountUsers = [&](bool Predicated) {
14840 auto &BECounts =
14841 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14842 for (const auto &LoopAndBEInfo : BECounts) {
14843 for (const ExitNotTakenInfo &ENT : LoopAndBEInfo.second.ExitNotTaken) {
14844 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14845 if (!isa<SCEVConstant>(S)) {
14846 auto UserIt = BECountUsers.find(S);
14847 if (UserIt != BECountUsers.end() &&
14848 UserIt->second.contains({ LoopAndBEInfo.first, Predicated }))
14849 continue;
14850 dbgs() << "Value " << *S << " for loop " << *LoopAndBEInfo.first
14851 << " missing from BECountUsers\n";
14852 std::abort();
14853 }
14854 }
14855 }
14856 }
14857 };
14858 VerifyBECountUsers(/* Predicated */ false);
14859 VerifyBECountUsers(/* Predicated */ true);
14860
14861 // Verify intergity of loop disposition cache.
14862 for (auto &[S, Values] : LoopDispositions) {
14863 for (auto [Loop, CachedDisposition] : Values) {
14864 const auto RecomputedDisposition = SE2.getLoopDisposition(S, Loop);
14865 if (CachedDisposition != RecomputedDisposition) {
14866 dbgs() << "Cached disposition of " << *S << " for loop " << *Loop
14867 << " is incorrect: cached " << CachedDisposition << ", actual "
14868 << RecomputedDisposition << "\n";
14869 std::abort();
14870 }
14871 }
14872 }
14873
14874 // Verify integrity of the block disposition cache.
14875 for (auto &[S, Values] : BlockDispositions) {
14876 for (auto [BB, CachedDisposition] : Values) {
14877 const auto RecomputedDisposition = SE2.getBlockDisposition(S, BB);
14878 if (CachedDisposition != RecomputedDisposition) {
14879 dbgs() << "Cached disposition of " << *S << " for block %"
14880 << BB->getName() << " is incorrect: cached " << CachedDisposition
14881 << ", actual " << RecomputedDisposition << "\n";
14882 std::abort();
14883 }
14884 }
14885 }
14886
14887 // Verify FoldCache/FoldCacheUser caches.
14888 for (auto [FoldID, Expr] : FoldCache) {
14889 auto I = FoldCacheUser.find(Expr);
14890 if (I == FoldCacheUser.end()) {
14891 dbgs() << "Missing entry in FoldCacheUser for cached expression " << *Expr
14892 << "!\n";
14893 std::abort();
14894 }
14895 if (!is_contained(I->second, FoldID)) {
14896 dbgs() << "Missing FoldID in cached users of " << *Expr << "!\n";
14897 std::abort();
14898 }
14899 }
14900 for (auto [Expr, IDs] : FoldCacheUser) {
14901 for (auto &FoldID : IDs) {
14902 const SCEV *S = FoldCache.lookup(FoldID);
14903 if (!S) {
14904 dbgs() << "Missing entry in FoldCache for expression " << *Expr
14905 << "!\n";
14906 std::abort();
14907 }
14908 if (S != Expr) {
14909 dbgs() << "Entry in FoldCache doesn't match FoldCacheUser: " << *S
14910 << " != " << *Expr << "!\n";
14911 std::abort();
14912 }
14913 }
14914 }
14915
14916 // Verify that ConstantMultipleCache computations are correct. We check that
14917 // cached multiples and recomputed multiples are multiples of each other to
14918 // verify correctness. It is possible that a recomputed multiple is different
14919 // from the cached multiple due to strengthened no wrap flags or changes in
14920 // KnownBits computations.
14921 for (auto [S, Multiple] : ConstantMultipleCache) {
14922 APInt RecomputedMultiple = SE2.getConstantMultiple(S);
14923 if ((Multiple != 0 && RecomputedMultiple != 0 &&
14924 Multiple.urem(RecomputedMultiple) != 0 &&
14925 RecomputedMultiple.urem(Multiple) != 0)) {
14926 dbgs() << "Incorrect cached computation in ConstantMultipleCache for "
14927 << *S << " : Computed " << RecomputedMultiple
14928 << " but cache contains " << Multiple << "!\n";
14929 std::abort();
14930 }
14931 }
14932}
14933
14935 Function &F, const PreservedAnalyses &PA,
14936 FunctionAnalysisManager::Invalidator &Inv) {
14937 // Invalidate the ScalarEvolution object whenever it isn't preserved or one
14938 // of its dependencies is invalidated.
14939 auto PAC = PA.getChecker<ScalarEvolutionAnalysis>();
14940 return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>()) ||
14941 Inv.invalidate<AssumptionAnalysis>(F, PA) ||
14942 Inv.invalidate<DominatorTreeAnalysis>(F, PA) ||
14943 Inv.invalidate<LoopAnalysis>(F, PA);
14944}
14945
14946AnalysisKey ScalarEvolutionAnalysis::Key;
14947
14950 auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
14951 auto &AC = AM.getResult<AssumptionAnalysis>(F);
14952 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
14953 auto &LI = AM.getResult<LoopAnalysis>(F);
14954 return ScalarEvolution(F, TLI, AC, DT, LI);
14955}
14956
14962
14965 // For compatibility with opt's -analyze feature under legacy pass manager
14966 // which was not ported to NPM. This keeps tests using
14967 // update_analyze_test_checks.py working.
14968 OS << "Printing analysis 'Scalar Evolution Analysis' for function '"
14969 << F.getName() << "':\n";
14971 return PreservedAnalyses::all();
14972}
14973
14975 "Scalar Evolution Analysis", false, true)
14981 "Scalar Evolution Analysis", false, true)
14982
14983char ScalarEvolutionWrapperPass::ID = 0;
14984
14986
14988 SE.reset(new ScalarEvolution(
14990 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
14992 getAnalysis<LoopInfoWrapperPass>().getLoopInfo()));
14993 return false;
14994}
14995
14997
14999 SE->print(OS);
15000}
15001
15003 if (!VerifySCEV)
15004 return;
15005
15006 SE->verify();
15007}
15008
15016
15018 const SCEV *RHS) {
15019 return getComparePredicate(ICmpInst::ICMP_EQ, LHS, RHS);
15020}
15021
15022const SCEVPredicate *
15024 const SCEV *LHS, const SCEV *RHS) {
15026 assert(LHS->getType() == RHS->getType() &&
15027 "Type mismatch between LHS and RHS");
15028 // Unique this node based on the arguments
15029 ID.AddInteger(SCEVPredicate::P_Compare);
15030 ID.AddInteger(Pred);
15031 ID.AddPointer(LHS);
15032 ID.AddPointer(RHS);
15034 if (const auto *S = UniquePreds.lookup(ID, Token))
15035 return S;
15036 SCEVComparePredicate *Eq = new (SCEVAllocator)
15037 SCEVComparePredicate(ID.Intern(SCEVAllocator), Pred, LHS, RHS);
15038 UniquePreds.insert(Eq, Token);
15039 return Eq;
15040}
15041
15043 const SCEVAddRecExpr *AR,
15046 // Unique this node based on the arguments
15048 ID.AddPointer(AR);
15049 ID.AddInteger(AddedFlags);
15051 if (const auto *S = UniquePreds.lookup(ID, Token))
15052 return S;
15053 auto *OF = new (SCEVAllocator)
15054 SCEVWrapPredicate(ID.Intern(SCEVAllocator), AR, AddedFlags);
15055 UniquePreds.insert(OF, Token);
15056 return OF;
15057}
15058
15059namespace {
15060
15061class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> {
15062public:
15063
15064 /// Rewrites \p S in the context of a loop L and the SCEV predication
15065 /// infrastructure.
15066 ///
15067 /// If \p Pred is non-null, the SCEV expression is rewritten to respect the
15068 /// equivalences present in \p Pred.
15069 ///
15070 /// If \p NewPreds is non-null, rewrite is free to add further predicates to
15071 /// \p NewPreds such that the result will be an AddRecExpr.
15072 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
15074 const SCEVPredicate *Pred) {
15075 SCEVPredicateRewriter Rewriter(L, SE, NewPreds, Pred);
15076 return Rewriter.visit(S);
15077 }
15078
15079 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
15080 if (Pred) {
15081 if (auto *U = dyn_cast<SCEVUnionPredicate>(Pred)) {
15082 for (const auto *Pred : U->getPredicates())
15083 if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Pred))
15084 if (IPred->getLHS() == Expr &&
15085 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15086 return IPred->getRHS();
15087 } else if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Pred)) {
15088 if (IPred->getLHS() == Expr &&
15089 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15090 return IPred->getRHS();
15091 }
15092 }
15093 return convertToAddRecWithPreds(Expr);
15094 }
15095
15096 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
15097 const SCEV *Operand = visit(Expr->getOperand());
15098 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Operand);
15099 if (AR && AR->getLoop() == L && AR->isAffine()) {
15100 // This couldn't be folded because the operand didn't have the nuw
15101 // flag. Add the nusw flag as an assumption that we could make.
15102 const SCEV *Step = AR->getStepRecurrence(SE);
15103 Type *Ty = Expr->getType();
15104 if (addOverflowAssumption(AR, SCEVWrapPredicate::IncrementNUSW))
15105 return SE.getAddRecExpr(SE.getZeroExtendExpr(AR->getStart(), Ty),
15106 SE.getSignExtendExpr(Step, Ty), L,
15107 AR->getNoWrapFlags());
15108 }
15109 return SE.getZeroExtendExpr(Operand, Expr->getType());
15110 }
15111
15112 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
15113 const SCEV *Operand = visit(Expr->getOperand());
15114 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Operand);
15115 if (AR && AR->getLoop() == L && AR->isAffine()) {
15116 // This couldn't be folded because the operand didn't have the nsw
15117 // flag. Add the nssw flag as an assumption that we could make.
15118 const SCEV *Step = AR->getStepRecurrence(SE);
15119 Type *Ty = Expr->getType();
15120 if (addOverflowAssumption(AR, SCEVWrapPredicate::IncrementNSSW))
15121 return SE.getAddRecExpr(SE.getSignExtendExpr(AR->getStart(), Ty),
15122 SE.getSignExtendExpr(Step, Ty), L,
15123 AR->getNoWrapFlags());
15124 }
15125 return SE.getSignExtendExpr(Operand, Expr->getType());
15126 }
15127
15128private:
15129 explicit SCEVPredicateRewriter(
15130 const Loop *L, ScalarEvolution &SE,
15131 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
15132 const SCEVPredicate *Pred)
15133 : SCEVRewriteVisitor(SE), NewPreds(NewPreds), Pred(Pred), L(L) {}
15134
15135 bool addOverflowAssumption(const SCEVPredicate *P) {
15136 if (!NewPreds) {
15137 // Check if we've already made this assumption.
15138 return Pred && Pred->implies(P, SE);
15139 }
15140 NewPreds->push_back(P);
15141 return true;
15142 }
15143
15144 bool addOverflowAssumption(const SCEVAddRecExpr *AR,
15146 auto *A = SE.getWrapPredicate(AR, AddedFlags);
15147 return addOverflowAssumption(A);
15148 }
15149
15150 // If \p Expr represents a PHINode, we try to see if it can be represented
15151 // as an AddRec, possibly under a predicate (PHISCEVPred). If it is possible
15152 // to add this predicate as a runtime overflow check, we return the AddRec.
15153 // If \p Expr does not meet these conditions (is not a PHI node, or we
15154 // couldn't create an AddRec for it, or couldn't add the predicate), we just
15155 // return \p Expr.
15156 const SCEV *convertToAddRecWithPreds(const SCEVUnknown *Expr) {
15157 if (!isa<PHINode>(Expr->getValue()))
15158 return Expr;
15159 std::optional<
15160 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
15161 PredicatedRewrite = SE.createAddRecFromPHIWithCasts(Expr);
15162 if (!PredicatedRewrite)
15163 return Expr;
15164 for (const auto *P : PredicatedRewrite->second){
15165 // Wrap predicates from outer loops are not supported.
15166 if (auto *WP = dyn_cast<const SCEVWrapPredicate>(P)) {
15167 if (L != WP->getExpr()->getLoop())
15168 return Expr;
15169 }
15170 if (!addOverflowAssumption(P))
15171 return Expr;
15172 }
15173 return PredicatedRewrite->first;
15174 }
15175
15176 SmallVectorImpl<const SCEVPredicate *> *NewPreds;
15177 const SCEVPredicate *Pred;
15178 const Loop *L;
15179};
15180
15181} // end anonymous namespace
15182
15183const SCEV *
15185 const SCEVPredicate &Preds) {
15186 return SCEVPredicateRewriter::rewrite(S, L, *this, nullptr, &Preds);
15187}
15188
15190 const SCEV *S, const Loop *L,
15193 S = SCEVPredicateRewriter::rewrite(S, L, *this, &TransformPreds, nullptr);
15194 auto *AddRec = dyn_cast<SCEVAddRecExpr>(S);
15195
15196 if (!AddRec)
15197 return nullptr;
15198
15199 // Check if any of the transformed predicates is known to be false. In that
15200 // case, it doesn't make sense to convert to a predicated AddRec, as the
15201 // versioned loop will never execute.
15202 for (const SCEVPredicate *Pred : TransformPreds) {
15203 auto *WrapPred = dyn_cast<SCEVWrapPredicate>(Pred);
15204 if (!WrapPred || WrapPred->getFlags() != SCEVWrapPredicate::IncrementNSSW)
15205 continue;
15206
15207 const SCEVAddRecExpr *AddRecToCheck = WrapPred->getExpr();
15208 const SCEV *ExitCount = getBackedgeTakenCount(AddRecToCheck->getLoop());
15209 if (isa<SCEVCouldNotCompute>(ExitCount))
15210 continue;
15211
15212 const SCEV *Step = AddRecToCheck->getStepRecurrence(*this);
15213 if (!Step->isOne())
15214 continue;
15215
15216 ExitCount = getTruncateOrSignExtend(ExitCount, Step->getType());
15217 const SCEV *Add = getAddExpr(AddRecToCheck->getStart(), ExitCount);
15218 if (isKnownPredicate(CmpInst::ICMP_SLT, Add, AddRecToCheck->getStart()))
15219 return nullptr;
15220 }
15221
15222 // Since the transformation was successful, we can now transfer the SCEV
15223 // predicates.
15224 Preds.append(TransformPreds.begin(), TransformPreds.end());
15225
15226 return AddRec;
15227}
15228
15229/// SCEV predicates
15233
15235 const ICmpInst::Predicate Pred,
15236 const SCEV *LHS, const SCEV *RHS)
15237 : SCEVPredicate(ID, P_Compare), Pred(Pred), LHS(LHS), RHS(RHS) {
15238 assert(LHS->getType() == RHS->getType() && "LHS and RHS types don't match");
15239 assert(LHS != RHS && "LHS and RHS are the same SCEV");
15240}
15241
15243 ScalarEvolution &SE) const {
15244 const auto *Op = dyn_cast<SCEVComparePredicate>(N);
15245
15246 if (!Op)
15247 return false;
15248
15249 if (Op->LHS != LHS || Op->RHS != RHS)
15250 return false;
15251
15252 return ICmpInst::isImpliedByMatchingCmp(Pred, Op->Pred) == true;
15253}
15254
15255bool SCEVComparePredicate::isAlwaysTrue() const { return false; }
15256
15258 if (Pred == ICmpInst::ICMP_EQ)
15259 OS.indent(Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n";
15260 else
15261 OS.indent(Depth) << "Compare predicate: " << *LHS << " " << Pred << ") "
15262 << *RHS << "\n";
15263
15264}
15265
15267 const SCEVAddRecExpr *AR,
15268 IncrementWrapFlags Flags)
15269 : SCEVPredicate(ID, P_Wrap), AR(AR), Flags(Flags) {}
15270
15271const SCEVAddRecExpr *SCEVWrapPredicate::getExpr() const { return AR; }
15272
15274 ScalarEvolution &SE) const {
15275 const auto *Op = dyn_cast<SCEVWrapPredicate>(N);
15276 if (!Op || setFlags(Flags, Op->Flags) != Flags)
15277 return false;
15278
15279 if (Op->AR == AR)
15280 return true;
15281
15282 if (Flags != SCEVWrapPredicate::IncrementNSSW &&
15284 return false;
15285
15286 const SCEV *Start = AR->getStart();
15287 const SCEV *OpStart = Op->AR->getStart();
15288 if (Start->getType()->isPointerTy() != OpStart->getType()->isPointerTy())
15289 return false;
15290
15291 // Reject pointers to different address spaces.
15292 if (Start->getType()->isPointerTy() && Start->getType() != OpStart->getType())
15293 return false;
15294
15295 // NUSW/NSSW on a wider-type AddRec does not imply the same on a
15296 // narrower-type AddRec.
15297 if (SE.getTypeSizeInBits(AR->getType()) >
15298 SE.getTypeSizeInBits(Op->AR->getType()))
15299 return false;
15300
15301 const SCEV *Step = AR->getStepRecurrence(SE);
15302 const SCEV *OpStep = Op->AR->getStepRecurrence(SE);
15303 if (!SE.isKnownPositive(Step) || !SE.isKnownPositive(OpStep))
15304 return false;
15305
15306 // If both steps are positive, this implies N, if N's start and step are
15307 // ULE/SLE (for NSUW/NSSW) than this'.
15308 Type *WiderTy = SE.getWiderType(Step->getType(), OpStep->getType());
15309 Step = SE.getNoopOrZeroExtend(Step, WiderTy);
15310 OpStep = SE.getNoopOrZeroExtend(OpStep, WiderTy);
15311
15312 bool IsNUW = Flags == SCEVWrapPredicate::IncrementNUSW;
15313 OpStart = IsNUW ? SE.getNoopOrZeroExtend(OpStart, WiderTy)
15314 : SE.getNoopOrSignExtend(OpStart, WiderTy);
15315 Start = IsNUW ? SE.getNoopOrZeroExtend(Start, WiderTy)
15316 : SE.getNoopOrSignExtend(Start, WiderTy);
15318 return SE.isKnownPredicate(Pred, OpStep, Step) &&
15319 SE.isKnownPredicate(Pred, OpStart, Start);
15320}
15321
15323 SCEVFlags ScevFlags = AR->getNoWrapFlags();
15324 IncrementWrapFlags IFlags = Flags;
15325
15326 if (ScalarEvolution::setFlags(ScevFlags, SCEV::FlagNSW) == ScevFlags)
15327 IFlags = clearFlags(IFlags, IncrementNSSW);
15328
15329 return IFlags == IncrementAnyWrap;
15330}
15331
15332void SCEVWrapPredicate::print(raw_ostream &OS, unsigned Depth) const {
15333 OS.indent(Depth) << *getExpr() << " Added Flags: ";
15335 OS << "<nusw>";
15337 OS << "<nssw>";
15338 OS << "\n";
15339}
15340
15341/// Union predicates don't get cached so create a dummy set ID for it.
15343 ScalarEvolution &SE)
15345 for (const auto *P : Preds)
15346 add(P, SE);
15347}
15348
15350 return all_of(Preds,
15351 [](const SCEVPredicate *I) { return I->isAlwaysTrue(); });
15352}
15353
15355 ScalarEvolution &SE) const {
15356 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(N))
15357 return all_of(Set->Preds, [this, &SE](const SCEVPredicate *I) {
15358 return this->implies(I, SE);
15359 });
15360
15361 if (any_of(Preds,
15362 [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); }))
15363 return true;
15364
15365 // A wrap predicate may be implied by a wrap predicate in Preds after applying
15366 // equal predicates.
15367 const auto *NWrap = dyn_cast<SCEVWrapPredicate>(N);
15368 if (!NWrap)
15369 return false;
15370 const Loop *L = NWrap->getExpr()->getLoop();
15371 return any_of(Preds, [&](const SCEVPredicate *I) {
15372 const auto *IWrap = dyn_cast<SCEVWrapPredicate>(I);
15373 if (!IWrap)
15374 return false;
15375 const auto *RewrittenAR = dyn_cast<SCEVAddRecExpr>(
15376 SE.rewriteUsingPredicate(IWrap->getExpr(), L, *this));
15377 return RewrittenAR &&
15378 SE.getWrapPredicate(RewrittenAR, IWrap->getFlags())->implies(N, SE);
15379 });
15380}
15381
15383 for (const auto *Pred : Preds)
15384 Pred->print(OS, Depth);
15385}
15386
15387void SCEVUnionPredicate::add(const SCEVPredicate *N, ScalarEvolution &SE) {
15388 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(N)) {
15389 for (const auto *Pred : Set->Preds)
15390 add(Pred, SE);
15391 return;
15392 }
15393
15394 // Implication checks are quadratic in the number of predicates. Stop doing
15395 // them if there are many predicates, as they should be too expensive to use
15396 // anyway at that point.
15397 bool CheckImplies = Preds.size() < 16;
15398
15399 // Only add predicate if it is not already implied by this union predicate.
15400 if (CheckImplies && implies(N, SE))
15401 return;
15402
15403 // Build a new vector containing the current predicates, except the ones that
15404 // are implied by the new predicate N.
15406 for (auto *P : Preds) {
15407 if (CheckImplies && N->implies(P, SE))
15408 continue;
15409 PrunedPreds.push_back(P);
15410 }
15411 Preds = std::move(PrunedPreds);
15412 Preds.push_back(N);
15413}
15414
15416 Loop &L)
15417 : SE(SE), L(L) {
15419 Preds = std::make_unique<SCEVUnionPredicate>(Empty, SE);
15420}
15421
15423 for (const SCEV *Op : Ops)
15424 // We do not expect that forgetting cached data for SCEVConstants will ever
15425 // open any prospects for sharpening or introduce any correctness issues,
15426 // so we don't bother storing their dependencies.
15427 if (!isa<SCEVConstant>(Op))
15428 SCEVUsers[Op].insert(User);
15429}
15430
15432 const SCEV *Expr = SE.getSCEV(V);
15433 return getPredicatedSCEV(Expr);
15434}
15435
15437 RewriteEntry &Entry = RewriteMap[Expr];
15438
15439 // If we already have an entry and the version matches, return it.
15440 if (Entry.second && Generation == Entry.first)
15441 return Entry.second;
15442
15443 // We found an entry but it's stale. Rewrite the stale entry
15444 // according to the current predicate.
15445 if (Entry.second)
15446 Expr = Entry.second;
15447
15448 const SCEV *NewSCEV = SE.rewriteUsingPredicate(Expr, &L, *Preds);
15449 Entry = {Generation, NewSCEV};
15450
15451 return NewSCEV;
15452}
15453
15455 if (!BackedgeCount) {
15457 BackedgeCount = SE.getPredicatedBackedgeTakenCount(&L, Preds);
15458 for (const auto *P : Preds)
15459 addPredicate(*P);
15460 }
15461 return BackedgeCount;
15462}
15463
15465 if (!SymbolicMaxBackedgeCount) {
15467 SymbolicMaxBackedgeCount =
15468 SE.getPredicatedSymbolicMaxBackedgeTakenCount(&L, Preds);
15469 for (const auto *P : Preds)
15470 addPredicate(*P);
15471 }
15472 return SymbolicMaxBackedgeCount;
15473}
15474
15476 if (!SmallConstantMaxTripCount) {
15478 SmallConstantMaxTripCount = SE.getSmallConstantMaxTripCount(&L, &Preds);
15479 for (const auto *P : Preds)
15480 addPredicate(*P);
15481 }
15482 return *SmallConstantMaxTripCount;
15483}
15484
15486 if (Preds->implies(&Pred, SE))
15487 return;
15488
15489 Preds = std::make_unique<SCEVUnionPredicate>(Preds->getUnionWith(&Pred, SE));
15490 updateGeneration();
15491}
15492
15495 for (const SCEVPredicate *P : Preds)
15496 addPredicate(*P);
15497}
15498
15500 return *Preds;
15501}
15502
15503void PredicatedScalarEvolution::updateGeneration() {
15504 // If the generation number wrapped recompute everything.
15505 if (++Generation == 0) {
15506 for (auto &II : RewriteMap) {
15507 const SCEV *Rewritten = II.second.second;
15508 II.second = {Generation, SE.rewriteUsingPredicate(Rewritten, &L, *Preds)};
15509 }
15510 }
15511}
15512
15515 const SCEV *Expr = this->getSCEV(V);
15517 auto *New = SE.convertSCEVToAddRecWithPredicates(Expr, &L, NewPreds);
15518
15519 if (!New)
15520 return nullptr;
15521
15522 if (ExtraPreds) {
15523 ExtraPreds->append(NewPreds);
15524 return New;
15525 }
15526
15527 addPredicates(NewPreds);
15528
15529 RewriteMap[SE.getSCEV(V)] = {Generation, New};
15530 return New;
15531}
15532
15535 : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L),
15536 Preds(std::make_unique<SCEVUnionPredicate>(*Init.Preds)),
15537 Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) {}
15538
15540 // For each block.
15541 for (auto *BB : L.getBlocks())
15542 for (auto &I : *BB) {
15543 if (!SE.isSCEVable(I.getType()))
15544 continue;
15545
15546 auto *Expr = SE.getSCEV(&I);
15547 auto II = RewriteMap.find(Expr);
15548
15549 if (II == RewriteMap.end())
15550 continue;
15551
15552 // Don't print things that are not interesting.
15553 if (II->second.second == Expr)
15554 continue;
15555
15556 OS.indent(Depth) << "[PSE]" << I << ":\n";
15557 OS.indent(Depth + 2) << *Expr << "\n";
15558 OS.indent(Depth + 2) << "--> " << *II->second.second << "\n";
15559 }
15560}
15561
15562void ScalarEvolution::LoopGuards::collectPending() {
15563 const Loop *L = PendingLoop;
15564 if (!L)
15565 return;
15566 PendingLoop = nullptr;
15567 BasicBlock *Pred = L->getLoopPredecessor();
15568 if (!Pred)
15569 return;
15571 collectFromBlock(SE, *this, L->getHeader(), Pred, VisitedBlocks);
15572}
15573
15574void ScalarEvolution::LoopGuards::collectFromPHI(
15578 unsigned Depth) {
15579 if (!SE.isSCEVable(Phi.getType()))
15580 return;
15581
15582 using MinMaxPattern = std::pair<const SCEVConstant *, SCEVTypes>;
15583 auto GetMinMaxConst = [&](unsigned IncomingIdx) -> MinMaxPattern {
15584 const BasicBlock *InBlock = Phi.getIncomingBlock(IncomingIdx);
15585 if (!VisitedBlocks.insert(InBlock).second)
15586 return {nullptr, scCouldNotCompute};
15587
15588 // Avoid analyzing unreachable blocks so that we don't get trapped
15589 // traversing cycles with ill-formed dominance or infinite cycles
15590 if (!SE.DT.isReachableFromEntry(InBlock))
15591 return {nullptr, scCouldNotCompute};
15592
15593 auto [G, Inserted] = IncomingGuards.try_emplace(InBlock, LoopGuards(SE));
15594 if (Inserted)
15595 collectFromBlock(SE, G->second, Phi.getParent(), InBlock, VisitedBlocks,
15596 Depth + 1);
15597 auto &RewriteMap = G->second.RewriteMap;
15598 if (RewriteMap.empty())
15599 return {nullptr, scCouldNotCompute};
15600 auto S = RewriteMap.find(SE.getSCEV(Phi.getIncomingValue(IncomingIdx)));
15601 if (S == RewriteMap.end())
15602 return {nullptr, scCouldNotCompute};
15603 auto *SM = dyn_cast_if_present<SCEVMinMaxExpr>(S->second);
15604 if (!SM)
15605 return {nullptr, scCouldNotCompute};
15606 if (const SCEVConstant *C0 = dyn_cast<SCEVConstant>(SM->getOperand(0)))
15607 return {C0, SM->getSCEVType()};
15608 return {nullptr, scCouldNotCompute};
15609 };
15610 auto MergeMinMaxConst = [](MinMaxPattern P1,
15611 MinMaxPattern P2) -> MinMaxPattern {
15612 auto [C1, T1] = P1;
15613 auto [C2, T2] = P2;
15614 if (!C1 || !C2 || T1 != T2)
15615 return {nullptr, scCouldNotCompute};
15616 switch (T1) {
15617 case scUMaxExpr:
15618 return {C1->getAPInt().ult(C2->getAPInt()) ? C1 : C2, T1};
15619 case scSMaxExpr:
15620 return {C1->getAPInt().slt(C2->getAPInt()) ? C1 : C2, T1};
15621 case scUMinExpr:
15622 return {C1->getAPInt().ugt(C2->getAPInt()) ? C1 : C2, T1};
15623 case scSMinExpr:
15624 return {C1->getAPInt().sgt(C2->getAPInt()) ? C1 : C2, T1};
15625 default:
15626 llvm_unreachable("Trying to merge non-MinMaxExpr SCEVs.");
15627 }
15628 };
15629 auto P = GetMinMaxConst(0);
15630 for (unsigned int In = 1; In < Phi.getNumIncomingValues(); In++) {
15631 if (!P.first)
15632 break;
15633 P = MergeMinMaxConst(P, GetMinMaxConst(In));
15634 }
15635 if (P.first) {
15636 const SCEV *LHS = SE.getSCEV(const_cast<PHINode *>(&Phi));
15637 SmallVector<SCEVUse, 2> Ops({P.first, LHS});
15638 const SCEV *RHS = SE.getMinMaxExpr(P.second, Ops);
15639 Guards.RewriteMap.insert({LHS, RHS});
15640 }
15641}
15642
15643// Return a new SCEV that modifies \p Expr to the closest number divides by
15644// \p Divisor and less or equal than Expr. For now, only handle constant
15645// Expr.
15647 const APInt &DivisorVal,
15648 ScalarEvolution &SE) {
15649 const APInt *ExprVal;
15650 if (!match(Expr, m_scev_APInt(ExprVal)) || ExprVal->isNegative() ||
15651 DivisorVal.isNonPositive())
15652 return Expr;
15653 APInt Rem = ExprVal->urem(DivisorVal);
15654 // return the SCEV: Expr - Expr % Divisor
15655 return SE.getConstant(*ExprVal - Rem);
15656}
15657
15658// Return a new SCEV that modifies \p Expr to the closest number divides by
15659// \p Divisor and greater or equal than Expr. For now, only handle constant
15660// Expr.
15661static const SCEV *getNextSCEVDivisibleByDivisor(const SCEV *Expr,
15662 const APInt &DivisorVal,
15663 ScalarEvolution &SE) {
15664 const APInt *ExprVal;
15665 if (!match(Expr, m_scev_APInt(ExprVal)) || ExprVal->isNegative() ||
15666 DivisorVal.isNonPositive())
15667 return Expr;
15668 APInt Rem = ExprVal->urem(DivisorVal);
15669 if (Rem.isZero())
15670 return Expr;
15671 // return the SCEV: Expr + Divisor - Expr % Divisor
15672 return SE.getConstant(*ExprVal + DivisorVal - Rem);
15673}
15674
15676 ICmpInst::Predicate Predicate, const SCEV *LHS, const SCEV *RHS,
15679 // If we have LHS == 0, check if LHS is computing a property of some unknown
15680 // SCEV %v which we can rewrite %v to express explicitly.
15682 return false;
15683 // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to
15684 // explicitly express that.
15685 const SCEVUnknown *URemLHS = nullptr;
15686 const SCEV *URemRHS = nullptr;
15687 if (!match(LHS, m_scev_URem(m_SCEVUnknown(URemLHS), m_SCEV(URemRHS), SE)))
15688 return false;
15689
15690 const SCEV *Multiple =
15691 SE.getMulExpr(SE.getUDivExpr(URemLHS, URemRHS), URemRHS);
15692 DivInfo[URemLHS] = Multiple;
15693 if (auto *C = dyn_cast<SCEVConstant>(URemRHS))
15694 Multiples[URemLHS] = C->getAPInt();
15695 return true;
15696}
15697
15698// Check if the condition is a divisibility guard (A % B == 0).
15699static bool isDivisibilityGuard(const SCEV *LHS, const SCEV *RHS,
15700 ScalarEvolution &SE) {
15701 const SCEV *X, *Y;
15702 return match(LHS, m_scev_URem(m_SCEV(X), m_SCEV(Y), SE)) && RHS->isZero();
15703}
15704
15705// Apply divisibility by \p Divisor on MinMaxExpr with constant values,
15706// recursively. This is done by aligning up/down the constant value to the
15707// Divisor.
15708static const SCEV *applyDivisibilityOnMinMaxExpr(const SCEV *MinMaxExpr,
15709 const APInt &Divisor,
15710 ScalarEvolution &SE) {
15711 // Only handle min/max expressions of a non-negative constant and another
15712 // operand.
15713 const SCEVConstant *C;
15714 const SCEV *Op;
15715 if (!match(MinMaxExpr, m_scev_MinMax(m_SCEVConstant(C), m_SCEV(Op))) ||
15716 C->getAPInt().isNegative())
15717 return MinMaxExpr;
15718 auto *DivisibleExpr = isa<SCEVSMinExpr, SCEVUMinExpr>(MinMaxExpr)
15719 ? getPreviousSCEVDivisibleByDivisor(C, Divisor, SE)
15720 : getNextSCEVDivisibleByDivisor(C, Divisor, SE);
15722 DivisibleExpr};
15723 return SE.getMinMaxExpr(MinMaxExpr->getSCEVType(), Ops);
15724}
15725
15726void ScalarEvolution::LoopGuards::collectFromBlock(
15727 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15728 const BasicBlock *Block, const BasicBlock *Pred,
15729 SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks, unsigned Depth) {
15730
15732
15733 SmallVector<SCEVUse> ExprsToRewrite;
15734 auto CollectCondition = [&](ICmpInst::Predicate Predicate, const SCEV *LHS,
15735 const SCEV *RHS,
15736 DenseMap<const SCEV *, const SCEV *> &RewriteMap,
15737 const LoopGuards &DivGuards) {
15738 // WARNING: It is generally unsound to apply any wrap flags to the proposed
15739 // replacement SCEV which isn't directly implied by the structure of that
15740 // SCEV. In particular, using contextual facts to imply flags is *NOT*
15741 // legal. See the scoping rules for flags in the header to understand why.
15742
15743 // Puts rewrite rule \p From -> \p To into the rewrite map. Also if \p From
15744 // and \p FromRewritten are the same (i.e. there has been no rewrite
15745 // registered for \p From), then puts this value in the list of rewritten
15746 // expressions.
15747 auto AddRewrite = [&](const SCEV *From, const SCEV *FromRewritten,
15748 const SCEV *To) {
15749 if (From == FromRewritten)
15750 ExprsToRewrite.push_back(From);
15751 RewriteMap[From] = To;
15752 };
15753
15754 // Checks whether \p S has already been rewritten. In that case returns the
15755 // existing rewrite because we want to chain further rewrites onto the
15756 // already rewritten value. Otherwise returns \p S.
15757 auto GetMaybeRewritten = [&](const SCEV *S) {
15758 return RewriteMap.lookup_or(S, S);
15759 };
15760
15761 // Check for a condition of the form (-C1 + X < C2). InstCombine will
15762 // create this form when combining two checks of the form (X u< C2 + C1) and
15763 // (X >=u C1).
15764 auto MatchRangeCheckIdiom = [&](ICmpInst::Predicate Pred,
15765 const SCEV *MatchLHS,
15766 const SCEV *MatchRHS) {
15767 const SCEVConstant *C1;
15768 const SCEVUnknown *LHSUnknown;
15769 auto *C2 = dyn_cast<SCEVConstant>(MatchRHS);
15770 if (!match(MatchLHS,
15771 m_scev_Add(m_SCEVConstant(C1), m_SCEVUnknown(LHSUnknown))) ||
15772 !C2)
15773 return false;
15774
15775 auto ExactRegion =
15777 .sub(C1->getAPInt());
15778
15779 // Tighten the raw range with what we already know about LHSUnknown
15780 // from prior guards recorded in RewriteMap, or from SCEV's own range
15781 // analysis.
15782 const SCEV *RewrittenLHS = GetMaybeRewritten(LHSUnknown);
15783 ExactRegion = ExactRegion.intersectWith(SE.getUnsignedRange(RewrittenLHS),
15785
15786 // Bail if the guard is inconsistent with prior facts, or if the range
15787 // is still not a monotonic non-wrapping interval after tightening.
15788 if (ExactRegion.isEmptySet() || ExactRegion.isWrappedSet() ||
15789 ExactRegion.isFullSet())
15790 return false;
15791
15792 const SCEV *RegionMin = SE.getConstant(ExactRegion.getUnsignedMin());
15793 const SCEV *RegionMax = SE.getConstant(ExactRegion.getUnsignedMax());
15794 const SCEV *ClampedLHS =
15795 SE.getUMaxExpr(RegionMin, SE.getUMinExpr(RewrittenLHS, RegionMax));
15796 AddRewrite(LHSUnknown, RewrittenLHS, ClampedLHS);
15797 return true;
15798 };
15799 if (MatchRangeCheckIdiom(Predicate, LHS, RHS))
15800 return;
15801
15802 // Do not apply information for constants or if RHS contains an AddRec.
15804 return;
15805
15806 // If RHS is SCEVUnknown, make sure the information is applied to it.
15808 std::swap(LHS, RHS);
15810 }
15811
15812 const SCEV *RewrittenLHS = GetMaybeRewritten(LHS);
15813 // Apply divisibility information when computing the constant multiple.
15814 const APInt &DividesBy =
15815 SE.getConstantMultiple(DivGuards.rewrite(RewrittenLHS));
15816
15817 // Collect rewrites for LHS and its transitive operands based on the
15818 // condition.
15819 // For min/max expressions, also apply the guard to its operands:
15820 // 'min(a, b) >= c' -> '(a >= c) and (b >= c)',
15821 // 'min(a, b) > c' -> '(a > c) and (b > c)',
15822 // 'max(a, b) <= c' -> '(a <= c) and (b <= c)',
15823 // 'max(a, b) < c' -> '(a < c) and (b < c)'.
15824
15825 // We cannot express strict predicates in SCEV, so instead we replace them
15826 // with non-strict ones against plus or minus one of RHS depending on the
15827 // predicate.
15828 const SCEV *One = SE.getOne(RHS->getType());
15829 switch (Predicate) {
15830 case CmpInst::ICMP_ULT:
15831 if (RHS->getType()->isPointerTy())
15832 return;
15833 RHS = SE.getUMaxExpr(RHS, One);
15834 [[fallthrough]];
15835 case CmpInst::ICMP_SLT: {
15836 RHS = SE.getMinusSCEV(RHS, One);
15837 RHS = getPreviousSCEVDivisibleByDivisor(RHS, DividesBy, SE);
15838 break;
15839 }
15840 case CmpInst::ICMP_UGT:
15841 case CmpInst::ICMP_SGT:
15842 RHS = SE.getAddExpr(RHS, One);
15843 RHS = getNextSCEVDivisibleByDivisor(RHS, DividesBy, SE);
15844 break;
15845 case CmpInst::ICMP_ULE:
15846 case CmpInst::ICMP_SLE:
15847 RHS = getPreviousSCEVDivisibleByDivisor(RHS, DividesBy, SE);
15848 break;
15849 case CmpInst::ICMP_UGE:
15850 case CmpInst::ICMP_SGE:
15851 RHS = getNextSCEVDivisibleByDivisor(RHS, DividesBy, SE);
15852 break;
15853 default:
15854 break;
15855 }
15856
15857 SmallVector<SCEVUse, 16> Worklist(1, LHS);
15858 SmallPtrSet<const SCEV *, 16> Visited;
15859
15860 auto EnqueueOperands = [&Worklist](const SCEVNAryExpr *S) {
15861 append_range(Worklist, S->operands());
15862 };
15863
15864 while (!Worklist.empty()) {
15865 const SCEV *From = Worklist.pop_back_val();
15866 if (isa<SCEVConstant>(From))
15867 continue;
15868 if (!Visited.insert(From).second)
15869 continue;
15870 const SCEV *FromRewritten = GetMaybeRewritten(From);
15871 const SCEV *To = nullptr;
15872
15873 switch (Predicate) {
15874 case CmpInst::ICMP_ULT:
15875 case CmpInst::ICMP_ULE:
15876 To = SE.getUMinExpr(FromRewritten, RHS);
15877 if (auto *UMax = dyn_cast<SCEVUMaxExpr>(FromRewritten))
15878 EnqueueOperands(UMax);
15879 break;
15880 case CmpInst::ICMP_SLT:
15881 case CmpInst::ICMP_SLE:
15882 To = SE.getSMinExpr(FromRewritten, RHS);
15883 if (auto *SMax = dyn_cast<SCEVSMaxExpr>(FromRewritten))
15884 EnqueueOperands(SMax);
15885 break;
15886 case CmpInst::ICMP_UGT:
15887 case CmpInst::ICMP_UGE:
15888 To = SE.getUMaxExpr(FromRewritten, RHS);
15889 if (auto *UMin = dyn_cast<SCEVUMinExpr>(FromRewritten))
15890 EnqueueOperands(UMin);
15891 break;
15892 case CmpInst::ICMP_SGT:
15893 case CmpInst::ICMP_SGE: {
15894 // From s>= RHS s>= 0 implies From u>= RHS, so an earlier
15895 // umax(C, From) with C u<= RHS is redundant. Drop it, as the range of
15896 // the umax may be much less precise than the one of From.
15897 const APInt *C, *RHSC;
15898 if (match(RHS, m_scev_APInt(RHSC)) && RHSC->isNonNegative() &&
15899 match(FromRewritten,
15901 C->ule(*RHSC))
15902 FromRewritten = From;
15903 To = SE.getSMaxExpr(FromRewritten, RHS);
15904 if (auto *SMin = dyn_cast<SCEVSMinExpr>(FromRewritten))
15905 EnqueueOperands(SMin);
15906 break;
15907 }
15908 case CmpInst::ICMP_EQ:
15910 To = RHS;
15911 break;
15912 case CmpInst::ICMP_NE:
15913 if (match(RHS, m_scev_Zero())) {
15914 const SCEV *OneAlignedUp =
15915 getNextSCEVDivisibleByDivisor(One, DividesBy, SE);
15916 To = SE.getUMaxExpr(FromRewritten, OneAlignedUp);
15917 } else {
15918 // LHS != RHS can be rewritten as (LHS - RHS) = UMax(1, LHS - RHS),
15919 // but creating the subtraction eagerly is expensive. Track the
15920 // inequalities in a separate map, and materialize the rewrite lazily
15921 // when encountering a suitable subtraction while re-writing.
15922 if (LHS->getType()->isPointerTy()) {
15923 LHS = SE.getPtrToAddrExpr(LHS);
15924 RHS = SE.getPtrToAddrExpr(RHS);
15926 break;
15927 }
15928 const SCEVConstant *C;
15929 const SCEV *A, *B;
15932 RHS = A;
15933 LHS = B;
15934 }
15935 if (LHS > RHS)
15936 std::swap(LHS, RHS);
15937 Guards.NotEqual.insert({LHS, RHS});
15938 continue;
15939 }
15940 break;
15941 default:
15942 break;
15943 }
15944
15945 if (To)
15946 AddRewrite(From, FromRewritten, To);
15947 }
15948 };
15949
15951 // First, collect information from assumptions dominating the loop.
15952 for (auto &AssumeVH : SE.AC.assumptions()) {
15953 if (!AssumeVH)
15954 continue;
15955 auto *AssumeI = cast<CallInst>(AssumeVH);
15956 if (!SE.DT.dominates(AssumeI, Block))
15957 continue;
15958 Terms.emplace_back(AssumeI->getOperand(0), true);
15959 }
15960
15961 // Second, collect information from llvm.experimental.guards dominating the loop.
15962 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
15963 SE.F.getParent(), Intrinsic::experimental_guard);
15964 if (GuardDecl)
15965 for (const auto *GU : GuardDecl->users())
15966 if (const auto *Guard = dyn_cast<IntrinsicInst>(GU))
15967 if (Guard->getFunction() == Block->getParent() &&
15968 SE.DT.dominates(Guard, Block))
15969 Terms.emplace_back(Guard->getArgOperand(0), true);
15970
15971 // Third, collect conditions from dominating branches. Starting at the loop
15972 // predecessor, climb up the predecessor chain, as long as there are
15973 // predecessors that can be found that have unique successors leading to the
15974 // original header. If there is no such predecessor, continue at the
15975 // immediate dominator.
15976 // TODO: share this logic with isLoopEntryGuardedByCond.
15977 unsigned NumCollectedConditions = 0;
15978 unsigned NumDomTreeSteps = 0;
15980 std::pair<const BasicBlock *, const BasicBlock *> Pair(Pred, Block);
15981 for (;; Pair = SE.getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
15982 if (!Pair.first) {
15983 // At the first block without a unique predecessor, try to build
15984 // conditions for its phis by merging those of their incoming blocks.
15985 if (!NumDomTreeSteps &&
15986 Depth < SE.Opts.scalar_evolution_max_loop_guard_collection_depth) {
15987 SmallDenseMap<const BasicBlock *, LoopGuards> IncomingGuards;
15988 for (auto &Phi : Pair.second->phis())
15989 collectFromPHI(SE, Guards, Phi, VisitedBlocks, IncomingGuards, Depth);
15990 }
15991 const DomTreeNode *IDom = SE.DT.getNode(Pair.second)->getIDom();
15992 if (!IDom ||
15993 ++NumDomTreeSteps > SE.Opts.scalar_evolution_max_guard_dom_tree_steps)
15994 break;
15995 Pair.first = IDom->getBlock();
15996 continue;
15997 }
15998 VisitedBlocks.insert(Pair.second);
15999 const CondBrInst *LoopEntryPredicate =
16000 dyn_cast<CondBrInst>(Pair.first->getTerminator());
16001 if (!LoopEntryPredicate)
16002 continue;
16003
16004 Terms.emplace_back(LoopEntryPredicate->getCondition(),
16005 LoopEntryPredicate->getSuccessor(0) == Pair.second);
16006 NumCollectedConditions++;
16007
16008 // If we are recursively collecting guards stop after 2
16009 // conditions to limit compile-time impact for now.
16010 if (Depth > 0 && NumCollectedConditions == 2)
16011 break;
16012 }
16013
16014 // Now apply the information from the collected conditions to
16015 // Guards.RewriteMap. Conditions are processed in reverse order, so the
16016 // earliest conditions is processed first, except guards with divisibility
16017 // information, which are moved to the back. This ensures the SCEVs with the
16018 // shortest dependency chains are constructed first.
16020 GuardsToProcess;
16021 for (auto [Term, EnterIfTrue] : reverse(Terms)) {
16022 SmallVector<Value *, 8> Worklist;
16023 SmallPtrSet<Value *, 8> Visited;
16024 Worklist.push_back(Term);
16025 while (!Worklist.empty()) {
16026 Value *Cond = Worklist.pop_back_val();
16027 if (!Visited.insert(Cond).second)
16028 continue;
16029
16030 if (auto *Cmp = dyn_cast<ICmpInst>(Cond)) {
16031 auto Predicate =
16032 EnterIfTrue ? Cmp->getPredicate() : Cmp->getInversePredicate();
16033 const auto *LHS = SE.getSCEV(Cmp->getOperand(0));
16034 const auto *RHS = SE.getSCEV(Cmp->getOperand(1));
16035 // If LHS is a constant, apply information to the other expression.
16036 // TODO: If LHS is not a constant, check if using compareSCEVComplexity
16037 // can improve results.
16038 if (isa<SCEVConstant>(LHS)) {
16039 std::swap(LHS, RHS);
16041 }
16042 GuardsToProcess.emplace_back(Predicate, LHS, RHS);
16043 continue;
16044 }
16045
16046 Value *L, *R;
16047 if (EnterIfTrue ? match(Cond, m_LogicalAnd(m_Value(L), m_Value(R)))
16048 : match(Cond, m_LogicalOr(m_Value(L), m_Value(R)))) {
16049 Worklist.push_back(L);
16050 Worklist.push_back(R);
16051 }
16052 }
16053 }
16054
16055 // Process divisibility guards in reverse order to populate DivGuards early.
16056 DenseMap<const SCEV *, APInt> Multiples;
16057 LoopGuards DivGuards(SE);
16058 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess) {
16059 if (!isDivisibilityGuard(LHS, RHS, SE))
16060 continue;
16061 collectDivisibilityInformation(Predicate, LHS, RHS, DivGuards.RewriteMap,
16062 Multiples, SE);
16063 }
16064
16065 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess)
16066 CollectCondition(Predicate, LHS, RHS, Guards.RewriteMap, DivGuards);
16067
16068 // Apply divisibility information last. This ensures it is applied to the
16069 // outermost expression after other rewrites for the given value.
16070 for (const auto &[K, Divisor] : Multiples) {
16071 const SCEV *DivisorSCEV = SE.getConstant(Divisor);
16072 Guards.RewriteMap[K] =
16074 Guards.rewrite(K), Divisor, SE),
16075 DivisorSCEV),
16076 DivisorSCEV);
16077 ExprsToRewrite.push_back(K);
16078 }
16079
16080 // Let the rewriter preserve NUW/NSW flags if the unsigned/signed ranges of
16081 // the replacement expressions are contained in the ranges of the replaced
16082 // expressions.
16083 Guards.PreserveNUW = true;
16084 Guards.PreserveNSW = true;
16085 for (const SCEV *Expr : ExprsToRewrite) {
16086 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
16087 Guards.PreserveNUW &=
16088 SE.getUnsignedRange(Expr).contains(SE.getUnsignedRange(RewriteTo));
16089 Guards.PreserveNSW &=
16090 SE.getSignedRange(Expr).contains(SE.getSignedRange(RewriteTo));
16091 }
16092
16093 // Now that all rewrite information is collect, rewrite the collected
16094 // expressions with the information in the map. This applies information to
16095 // sub-expressions. A collected expression is rewritten when it is first
16096 // looked up, so the rewrites of the expressions it contains are used once
16097 // they are final.
16098 if (ExprsToRewrite.size() > 1) {
16099 constexpr unsigned MaxDepth = 16;
16100 unsigned Depth = 0;
16101 SmallPtrSet<const SCEV *, 16> ToRewrite(llvm::from_range, ExprsToRewrite);
16102 std::function<const SCEV *(const SCEV *)> Lookup = [&](const SCEV *Expr) {
16103 if (Depth < MaxDepth && ToRewrite.erase(Expr)) {
16104 auto It = Guards.RewriteMap.find(Expr);
16105 assert(It != Guards.RewriteMap.end() && "Expected rewrite entry");
16106 const SCEV *RewriteTo = It->second;
16107 Guards.RewriteMap.erase(It);
16108 SaveAndRestore Nest(Depth, Depth + 1);
16109 const SCEV *Rewritten = Guards.rewrite(RewriteTo, Lookup);
16110 Guards.RewriteMap[Expr] = Rewritten;
16111 return Rewritten;
16112 }
16113 return Guards.RewriteMap.lookup(Expr);
16114 };
16115 for (const SCEV *Expr : ExprsToRewrite)
16116 Lookup(Expr);
16117 }
16118}
16119
16120const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const {
16121 return rewrite(Expr, nullptr);
16122}
16123
16124const SCEV *ScalarEvolution::LoopGuards::rewrite(
16125 const SCEV *Expr, function_ref<const SCEV *(const SCEV *)> Lookup) const {
16126 /// A rewriter to replace SCEV expressions in Map with the corresponding entry
16127 /// in the map. It skips AddRecExpr because we cannot guarantee that the
16128 /// replacement is loop invariant in the loop of the AddRec.
16129 class SCEVLoopGuardRewriter
16130 : public SCEVRewriteVisitor<SCEVLoopGuardRewriter> {
16133 function_ref<const SCEV *(const SCEV *)> Lookup;
16134
16135 SCEVFlags FlagMask = SCEV::FlagNone;
16136
16137 const SCEV *lookup(const SCEV *Key) {
16138 return Lookup ? Lookup(Key) : Map.lookup(Key);
16139 }
16140
16141 public:
16142 SCEVLoopGuardRewriter(ScalarEvolution &SE,
16143 const ScalarEvolution::LoopGuards &Guards,
16144 function_ref<const SCEV *(const SCEV *)> Lookup)
16145 : SCEVRewriteVisitor(SE), Map(Guards.RewriteMap),
16146 NotEqual(Guards.NotEqual), Lookup(Lookup) {
16147 if (Guards.PreserveNUW)
16148 FlagMask = ScalarEvolution::setFlags(FlagMask, SCEV::FlagNUW);
16149 if (Guards.PreserveNSW)
16150 FlagMask = ScalarEvolution::setFlags(FlagMask, SCEV::FlagNSW);
16151 }
16152
16153 const SCEV *visit(const SCEV *Expr) {
16156 Expr))
16157 if (const SCEV *S = lookup(Expr))
16158 return S;
16159 return SCEVRewriteVisitor::visit(Expr);
16160 }
16161
16162 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
16163
16164 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
16165 // If there is no replacement for the exact ZExt expr, check if there's
16166 // one for a smaller ZExt we can use instead.
16167 Type *Ty = Expr->getType();
16168 const SCEV *Op = Expr->getOperand(0);
16169 unsigned Bitwidth = Ty->getScalarSizeInBits() / 2;
16170 while (Bitwidth % 8 == 0 && Bitwidth >= 8 &&
16171 Bitwidth > Op->getType()->getScalarSizeInBits()) {
16172 Type *NarrowTy = IntegerType::get(SE.getContext(), Bitwidth);
16173 auto *NarrowExt = SE.getZeroExtendExpr(Op, NarrowTy);
16174 if (const SCEV *S = lookup(NarrowExt))
16175 return SE.getZeroExtendExpr(S, Ty);
16176 Bitwidth = Bitwidth / 2;
16177 }
16178
16180 }
16181
16182 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
16183 // Helper to check if S is a subtraction (A - B) where A != B, and if so,
16184 // return UMax(S, 1).
16185 auto RewriteSubtraction = [&](const SCEV *S) -> const SCEV * {
16186 SCEVUse LHS, RHS;
16187 if (MatchBinarySub(S, LHS, RHS)) {
16188 if (LHS > RHS)
16189 std::swap(LHS, RHS);
16190 if (NotEqual.contains({LHS, RHS})) {
16191 const SCEV *OneAlignedUp = getNextSCEVDivisibleByDivisor(
16192 SE.getOne(S->getType()), SE.getConstantMultiple(S), SE);
16193 return SE.getUMaxExpr(OneAlignedUp, S);
16194 }
16195 }
16196 return nullptr;
16197 };
16198
16199 // Check if Expr itself is a subtraction pattern with guard info.
16200 if (const SCEV *Rewritten = RewriteSubtraction(Expr))
16201 return Rewritten;
16202
16203 // Trip count expressions sometimes consist of adding 3 operands, i.e.
16204 // (Const + A + B). There may be guard info for A + B, and if so, apply
16205 // it.
16206 // TODO: Could more generally apply guards to Add sub-expressions.
16207 if (isa<SCEVConstant>(Expr->getOperand(0))) {
16208 if (Expr->getNumOperands() == 3) {
16209 const SCEV *Add =
16210 SE.getAddExpr(Expr->getOperand(1), Expr->getOperand(2));
16211 if (const SCEV *Rewritten = RewriteSubtraction(Add))
16212 return SE.getAddExpr(
16213 Expr->getOperand(0), Rewritten,
16214 ScalarEvolution::maskFlags(Expr->getNoWrapFlags(), FlagMask));
16215 if (const SCEV *S = lookup(Add))
16216 return SE.getAddExpr(Expr->getOperand(0), S);
16217 }
16218
16219 // For expressions of the form (Const + A), check if we have guard info
16220 // for (Const + 1 + A), and rewrite to ((Const + 1 + A) - 1). This makes
16221 // sure we don't lose information when rewriting expressions based on
16222 // back-edge taken counts in some cases.
16223 if (Expr->getNumOperands() == 2) {
16224 const SCEV *S = nullptr;
16225 // Handle (-1 + 1 + A) without constructing SCEVs.
16226 if (match(Expr->getOperand(0), m_scev_AllOnes())) {
16227 S = lookup(Expr->getOperand(1));
16228 } else {
16229 const SCEV *NewC =
16230 SE.getAddExpr(Expr->getOperand(0), SE.getOne(Expr->getType()));
16231 S = lookup(SE.getAddExpr(NewC, Expr->getOperand(1)));
16232 }
16233 if (S)
16234 return SE.getAddExpr(S, SE.getMinusOne(Expr->getType()));
16235 }
16236 }
16238 bool Changed = false;
16239 for (SCEVUse Op : Expr->operands()) {
16240 Operands.push_back(visit(Op));
16241 Changed |= Op != Operands.back();
16242 }
16243 // We are only replacing operands with equivalent values, so transfer the
16244 // flags from the original expression.
16245 return !Changed ? Expr
16246 : SE.getAddExpr(Operands,
16248 Expr->getNoWrapFlags(), FlagMask));
16249 }
16250
16251 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
16253 bool Changed = false;
16254 for (SCEVUse Op : Expr->operands()) {
16255 Operands.push_back(visit(Op));
16256 Changed |= Op != Operands.back();
16257 }
16258 // We are only replacing operands with equivalent values, so transfer the
16259 // flags from the original expression.
16260 return !Changed ? Expr
16261 : SE.getMulExpr(Operands,
16263 Expr->getNoWrapFlags(), FlagMask));
16264 }
16265 };
16266
16267 if (RewriteMap.empty() && NotEqual.empty())
16268 return Expr;
16269
16270 SCEVLoopGuardRewriter Rewriter(SE, *this, Lookup);
16271 return Rewriter.visit(Expr);
16272}
16273
16274const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) {
16275 LoopGuards Guards = LoopGuards::collectOnDemand(L, *this);
16276 return applyLoopGuards(Expr, Guards);
16277}
16278
16280 LoopGuards &Guards) {
16281 // Loop guards never rewrite constants and AddRecs, so there is no need to
16282 // collect pending guards for them.
16284 return Expr;
16285 if (Guards.PendingLoop)
16286 Guards.collectPending();
16287 return Guards.rewrite(Expr);
16288}
const TargetLowering & TLI
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
constexpr LLT S1
Rewrite undef for PHI
This file implements a class to represent arbitrary precision integral constant values and operations...
@ PostInc
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
bpf unwind BPF Unwind Lower
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition Compiler.h:690
This file contains the declarations for the subclasses of Constant, which represent the different fla...
SmallPtrSet< const BasicBlock *, 8 > VisitedBlocks
This file defines the DenseMap class.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
static bool isSigned(unsigned Opcode)
This file defines a hash set that can be used to remove duplication of nodes in a graph.
#define op(i)
Hexagon Common GEP
Value * getPointer(Value *Ptr)
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This defines the Use class.
iv Induction Variable Users
Definition IVUsers.cpp:48
static bool lookup(const GsymReader &GR, GsymDataExtractor &Data, uint64_t &Offset, uint64_t BaseAddr, uint64_t Addr, SourceLocations &SrcLocs, llvm::Error &Err)
A Lookup helper functions.
static bool hasNoUnsignedWrap(BinaryOperator &I)
static constexpr Value * getValue(Ty &ValueOrUse)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:550
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define G(x, y, z)
Definition MD5.cpp:55
#define T
#define T1
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
#define P(N)
ppc ctr loops verify
PowerPC Reduce CR logical Operation
if(PassOpts->AAPipeline)
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
R600 Clause Merge
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
int64_t C1
int64_t C0
unsigned ExpectedFlags
static DominatorTree getDomTree(Function &F)
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
SI Fold Operands
SI optimize exec mask operations pre RA
static void visit(BasicBlock &Start, std::function< bool(BasicBlock *)> op)
This file contains some templates that are useful if you are working with the STL at all.
This file provides utility classes that use RAII to save and restore values.
bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind, SCEVTypes RootKind)
static const Loop * isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI)
static unsigned getConstantTripCount(const SCEVConstant *ExitCount)
static const SCEV * getNextSCEVDivisibleByDivisor(const SCEV *Expr, const APInt &DivisorVal, ScalarEvolution &SE)
static PHINode * getConstantEvolvingPHIOperands(const AnalysisOptions &Opts, Instruction *UseInst, const Loop *L, DenseMap< Instruction *, PHINode * > &PHIMap, const TargetLibraryInfo *TLI, unsigned Depth)
getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by recursing through each instructi...
static void PushLoopPHIs(const Loop *L, SmallVectorImpl< Instruction * > &Worklist, SmallPtrSetImpl< Instruction * > &Visited)
Push PHI nodes in the header of the given loop onto the given Worklist.
static void insertFoldCacheEntry(const ScalarEvolution::FoldID &ID, const SCEV *S, DenseMap< ScalarEvolution::FoldID, const SCEV * > &FoldCache, DenseMap< const SCEV *, SmallVector< ScalarEvolution::FoldID, 2 > > &FoldCacheUser)
static bool hasHugeExpression(const AnalysisOptions &Opts, ArrayRef< SCEVUse > Ops)
Returns true if Ops contains a huge SCEV (the subtree of S contains at least -scalar-evolution-huge-e...
static bool RangeRefPHIAllowedOperands(DominatorTree &DT, PHINode *PHI)
static bool IsKnownPredicateViaAddRecMonotonicity(ScalarEvolution &SE, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Is LHS Pred RHS true because one of them is an AddRec that is known not to go below its own start val...
static std::optional< APInt > MinOptional(std::optional< APInt > X, std::optional< APInt > Y)
Helper function to compare optional APInts: (a) if X and Y both exist, return min(X,...
static std::optional< BinaryOp > MatchBinaryOp(Value *V, const DominatorTree &DT)
Try to map V into a BinaryOp, and return std::nullopt on failure.
static bool canConstantFold(const Instruction *I, const TargetLibraryInfo *TLI)
Return true if we can constant fold an instruction of the specified type, assuming that all operands ...
static BinaryOperator * getCommonInstForPHI(PHINode *PN)
static PHINode * getConstantEvolvingPHI(const AnalysisOptions &Opts, Value *V, const Loop *L, const TargetLibraryInfo *TLI)
getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node in the loop that V is deri...
static bool isDivisibilityGuard(const SCEV *LHS, const SCEV *RHS, ScalarEvolution &SE)
static std::optional< const SCEV * > createNodeForSelectViaUMinSeq(ScalarEvolution *SE, const SCEV *CondExpr, const SCEV *TrueExpr, const SCEV *FalseExpr)
static Constant * BuildConstantFromSCEV(const SCEV *V)
This builds up a Constant using the ConstantExpr interface.
static ConstantInt * EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C, ScalarEvolution &SE)
static const SCEV * BinomialCoefficient(const SCEV *It, unsigned K, ScalarEvolution &SE, Type *ResultTy)
Compute BC(It, K). The result has width W. Assume, K > 0.
static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr, const SCEV *Candidate)
Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
static SCEVFlags getNoWrapFlagsForGEP(GEPOperator *GEP, const SCEV *Accum, ScalarEvolution &SE)
static const SCEV * SolveLinEquationWithOverflow(const APInt &A, const SCEV *B, SmallVectorImpl< const SCEVPredicate * > *Predicates, ScalarEvolution &SE, const Loop *L)
Finds the minimum unsigned root of the following equation:
static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow)
static void PrintSCEVWithTypeHint(raw_ostream &OS, const SCEV *S)
When printing a top-level SCEV for trip counts, it's helpful to include a type for constants which ar...
static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, const Loop *L)
static bool containsConstantInAddMulChain(const SCEV *StartExpr)
Determine if any of the operands in this SCEV are a constant or if any of the add or multiply express...
static const SCEV * getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty, ScalarEvolution *SE, unsigned Depth)
static const SCEV * applyDivisibilityOnMinMaxExpr(const SCEV *MinMaxExpr, const APInt &Divisor, ScalarEvolution &SE)
static const SCEV * constantFoldAndGroupOps(const AnalysisOptions &Opts, ScalarEvolution &SE, LoopInfo &LI, DominatorTree &DT, SmallVectorImpl< SCEVUse > &Ops, FoldT Fold, IsIdentityT IsIdentity, IsAbsorberT IsAbsorber)
Performs a number of common optimizations on the passed Ops.
static int compareValueComplexity(const AnalysisOptions &Opts, const LoopInfo *const LI, Value *LV, Value *RV, unsigned Depth)
Compare the two values LV and RV in terms of their "complexity" where "complexity" is a partial (and ...
static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
static const SCEV * getPreStartForExtend(const SCEVAddRecExpr *AR, ScalarEvolution *SE, unsigned Depth)
static bool collectDivisibilityInformation(ICmpInst::Predicate Predicate, const SCEV *LHS, const SCEV *RHS, DenseMap< const SCEV *, const SCEV * > &DivInfo, DenseMap< const SCEV *, APInt > &Multiples, ScalarEvolution &SE)
static void PushDefUseChildren(Instruction *I, SmallVectorImpl< Instruction * > &Worklist, SmallPtrSetImpl< Instruction * > &Visited)
Push users of the given Instruction onto the given Worklist.
static std::optional< APInt > SolveQuadraticAddRecRange(const SCEVAddRecExpr *AddRec, const ConstantRange &Range, ScalarEvolution &SE)
Let c(n) be the value of the quadratic chrec {0,+,M,+,N} after n iterations.
static int compareSCEVComplexity(const AnalysisOptions &Opts, const LoopInfo *const LI, SCEVUse LHS, SCEVUse RHS, DominatorTree &DT)
static bool getOperandsForSelectLikePHI(DominatorTree &DT, PHINode *PN, Value *&Cond, Value *&LHS, Value *&RHS)
static std::optional< std::tuple< APInt, APInt, APInt, APInt, unsigned > > GetQuadraticEquation(const SCEVAddRecExpr *AddRec)
For a given quadratic addrec, generate coefficients of the corresponding quadratic equation,...
static bool CollectAddOperandsWithScales(SmallDenseMap< const SCEV *, APInt, 16 > &M, SmallVectorImpl< SCEVUse > &NewOps, APInt &AccumulatedConstant, ArrayRef< SCEVUse > Ops, const APInt &Scale, ScalarEvolution &SE)
Process the given Ops list, which is a list of operands to be added under the given scale,...
static bool isKnownPredicateExtendIdiom(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
static std::optional< APInt > SolveQuadraticAddRecExact(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE)
Let c(n) be the value of the quadratic chrec {L,+,M,+,N} after n iterations.
static std::optional< APInt > TruncIfPossible(std::optional< APInt > X, unsigned BitWidth)
Helper function to truncate an optional APInt to a given BitWidth.
static SCEVFlags StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type, ArrayRef< SCEVUse > Ops, SCEVFlags Flags)
static void groupByComplexity(const AnalysisOptions &Opts, SmallVectorImpl< SCEVUse > &Ops, LoopInfo *LI, DominatorTree &DT)
Given a list of SCEV objects, order them by their complexity, and group objects of the same complexit...
static APInt extractConstantWithoutWrapping(ScalarEvolution &SE, const SCEVConstant *ConstantTerm, const SCEVAddExpr *WholeAddExpr)
static bool canConstantEvolve(Instruction *I, const Loop *L, const TargetLibraryInfo *TLI)
Determine whether this instruction can constant evolve within this loop assuming its operands can all...
static bool MatchBinarySub(const SCEV *S, SCEVUse &LHS, SCEVUse &RHS)
static std::optional< ConstantRange > GetRangeFromMetadata(Value *V)
Helper method to assign a range to V from metadata present in the IR.
static Type * isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI, bool &Signed, ScalarEvolution &SE)
Helper function to createAddRecFromPHIWithCasts.
static Constant * EvaluateExpression(Value *V, const Loop *L, DenseMap< Instruction *, Constant * > &Vals, const DataLayout &DL, const TargetLibraryInfo *TLI)
EvaluateExpression - Given an expression that passes the getConstantEvolvingPHI predicate,...
static const SCEV * getPreviousSCEVDivisibleByDivisor(const SCEV *Expr, const APInt &DivisorVal, ScalarEvolution &SE)
static const SCEV * MatchNotExpr(const SCEV *Expr)
If Expr computes ~A, return A else return nullptr.
static std::pair< ConstantRange, bool > getRangeForAffineARHelper(APInt Step, const ConstantRange &StartRange, const APInt &MaxBECount, bool Signed)
static cl::opt< bool, true > VerifySCEVOpt("verify-scev", cl::Hidden, cl::location(VerifySCEV), cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"))
static const SCEV * getSignedOverflowLimitForStep(const SCEV *Step, ICmpInst::Predicate *Pred, ScalarEvolution *SE)
static bool HasSameValue(const SCEV *A, const SCEV *B)
SCEV structural equivalence is usually sufficient for testing whether two expressions are equal,...
static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow)
Compute the result of "n choose k", the binomial coefficient.
static bool scevUnconditionallyPropagatesPoisonFromOperands(SCEVTypes Kind)
static Constant * getOtherIncomingValue(PHINode *PN, BasicBlock *BB)
static const SCEV * getUnsignedOverflowLimitForStep(const SCEV *Step, ICmpInst::Predicate *Pred, ScalarEvolution *SE)
static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Is LHS Pred RHS true on the virtue of LHS or RHS being a Min or Max expression?
static bool BrPHIToSelect(DominatorTree &DT, CondBrInst *BI, PHINode *Merge, Value *&C, Value *&LHS, Value *&RHS)
This file defines the scope_exit class, which executes user-defined cleanup logic at scope exit.
static bool InBlock(const Value *V, const BasicBlock *BB)
Provides some synthesis utilities to produce sequences of values.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
This file contains some functions that are useful when dealing with strings.
#define LLVM_DEBUG(...)
Definition Debug.h:119
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
static std::optional< bool > isImpliedCondOperands(CmpInst::Predicate Pred, const Value *ALHS, const Value *ARHS, const Value *BLHS, const Value *BRHS)
Return true if "icmp Pred BLHS BRHS" is true whenever "icmp PredALHS ARHS" is true.
virt reg Virtual Register Rewriter
static int Lookup(ArrayRef< TableEntry > Table, unsigned Opcode)
Value * RHS
Value * LHS
BinaryOperator * Mul
static const uint32_t IV[8]
Definition blake3_impl.h:83
SCEVCastSinkingRewriter(ScalarEvolution &SE, Type *TargetTy, ConversionFn CreatePtrCast)
static const SCEV * rewrite(const SCEV *Scev, ScalarEvolution &SE, Type *TargetTy, ConversionFn CreatePtrCast)
const SCEV * visitUnknown(const SCEVUnknown *Expr)
const SCEV * visitAddExpr(const SCEVAddExpr *Expr)
const SCEV * visit(const SCEV *S)
Class for arbitrary precision integers.
Definition APInt.h:78
LLVM_ABI APInt umul_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:2018
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1057
bool isMinSignedValue() const
Determine if this is the smallest signed value.
Definition APInt.h:419
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1532
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:970
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition APInt.h:202
APInt abs() const
Get the absolute value.
Definition APInt.h:1815
bool sgt(const APInt &RHS) const
Signed greater than comparison.
Definition APInt.h:1205
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition APInt.h:367
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition APInt.h:1186
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
Definition APInt.h:462
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition APInt.cpp:1695
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition APInt.h:1115
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition APInt.h:205
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
Definition APInt.h:212
bool isNegative() const
Determine sign of this APInt.
Definition APInt.h:325
LLVM_ABI APInt uadd_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1973
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition APInt.h:215
bool isNonPositive() const
Determine if this APInt Value is non-positive (<= 0).
Definition APInt.h:357
unsigned countTrailingZeros() const
Definition APInt.h:1667
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
Definition APInt.h:352
unsigned logBase2() const
Definition APInt.h:1781
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
Definition APInt.h:471
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition APInt.h:829
LLVM_ABI APInt multiplicativeInverse() const
Definition APInt.cpp:1303
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition APInt.h:330
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition APInt.h:1154
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
Definition APInt.cpp:1030
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition APInt.h:875
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:436
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
bool isSignBitSet() const
Determine if sign bit of this APInt is set.
Definition APInt.h:337
bool slt(const APInt &RHS) const
Signed less than comparison.
Definition APInt.h:1134
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition APInt.h:196
bool isIntN(unsigned N) const
Check if this APInt has an N-bits unsigned integer value.
Definition APInt.h:428
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
Definition APInt.h:1241
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition APInt.h:235
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition APInt.h:1225
This templated class represents "all analyses that operate over <aparticular IR unit>" (e....
Definition Analysis.h:50
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
void setPreservesAll()
Set by analyses that do not transform their input at all.
AnalysisUsage & addRequiredTransitive()
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
iterator end() const
Definition ArrayRef.h:130
size_t size() const
Get the array size.
Definition ArrayRef.h:141
iterator begin() const
Definition ArrayRef.h:129
A function analysis which provides an AssumptionCache.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
MutableArrayRef< WeakVH > assumptions()
Access the list of assumption handles currently tracked for this function.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator begin()
Instruction iterator methods.
Definition BasicBlock.h:440
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction & front() const
Definition BasicBlock.h:463
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
LLVM_ABI unsigned getNoWrapKind() const
Returns one of OBO::NoSignedWrap or OBO::NoUnsignedWrap.
LLVM_ABI Instruction::BinaryOps getBinaryOp() const
Returns the binary operation underlying the intrinsic.
BinaryOps getOpcode() const
Definition InstrTypes.h:409
This class represents a function call, abstracting a target machine's calling convention.
virtual void deleted()
Callback for Value destruction.
void setValPtr(Value *P)
This is the base class for all instructions that perform data casts.
Definition InstrTypes.h:512
This class is the base class for the comparison instructions.
Definition InstrTypes.h:728
bool isFalseWhenEqual() const
This is just a convenience.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
bool isSigned() const
Definition InstrTypes.h:993
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition InstrTypes.h:890
bool isTrueWhenEqual() const
This is just a convenience.
Predicate getNonStrictPredicate() const
For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
Definition InstrTypes.h:934
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition InstrTypes.h:852
bool isUnsigned() const
Definition InstrTypes.h:999
bool isRelational() const
Return true if the predicate is relational (not EQ or NE).
Definition InstrTypes.h:989
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
static LLVM_ABI std::optional< CmpPredicate > getMatching(CmpPredicate A, CmpPredicate B)
Compares two CmpPredicates taking samesign into account and returns the canonicalized CmpPredicate if...
CmpInst::Predicate dropSameSign() const
Drops samesign information.
Conditional Branch instruction.
Value * getCondition() const
BasicBlock * getSuccessor(unsigned i) const
static LLVM_ABI Constant * getNot(Constant *C)
static Constant * getPtrAdd(Constant *Ptr, Constant *Offset, GEPNoWrapFlags NW=GEPNoWrapFlags::none(), std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReduced=nullptr)
Create a getelementptr i8, ptr, offset constant expression.
Definition Constants.h:1518
static LLVM_ABI Constant * getPtrToAddr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static LLVM_ABI Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static LLVM_ABI Constant * getNeg(Constant *C, bool HasNSW=false)
static LLVM_ABI Constant * getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
static LLVM_ABI ConstantInt * getBool(LLVMContext &Context, bool V)
This class represents a range of values.
LLVM_ABI ConstantRange add(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an addition of a value in this ran...
LLVM_ABI ConstantRange zextOrTrunc(uint32_t BitWidth) const
Make this range have the bit width given by BitWidth.
PreferredRangeType
If represented precisely, the result of some range operations may consist of multiple disjoint ranges...
LLVM_ABI bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const
Set up Pred and RHS such that ConstantRange::makeExactICmpRegion(Pred, RHS) == *this.
const APInt & getLower() const
Return the lower value for this range.
LLVM_ABI ConstantRange urem(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an unsigned remainder operation of...
LLVM_ABI bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
LLVM_ABI bool icmp(CmpInst::Predicate Pred, const ConstantRange &Other) const
Does the predicate Pred hold between ranges this and Other?
LLVM_ABI bool isEmptySet() const
Return true if this set contains no members.
LLVM_ABI ConstantRange zeroExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
LLVM_ABI bool isSignWrappedSet() const
Return true if this set wraps around the signed domain.
LLVM_ABI APInt getSignedMin() const
Return the smallest signed value contained in the ConstantRange.
LLVM_ABI bool isWrappedSet() const
Return true if this set wraps around the unsigned domain.
LLVM_ABI void print(raw_ostream &OS) const
Print out the bounds to a stream.
bool isSingleElement() const
Return true if this set contains exactly one member.
LLVM_ABI ConstantRange truncate(uint32_t BitWidth, unsigned NoWrapKind=0) const
Return a new range in the specified integer type, which must be strictly smaller than the current typ...
LLVM_ABI ConstantRange signExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
const APInt & getUpper() const
Return the upper value for this range.
LLVM_ABI ConstantRange unionWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the union of this range with another range.
static LLVM_ABI ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
LLVM_ABI bool contains(const APInt &Val) const
Return true if the specified value is in the set.
LLVM_ABI APInt getSignedMax() const
Return the largest signed value contained in the ConstantRange.
static ConstantRange getNonEmpty(APInt Lower, APInt Upper)
Create non-empty constant range with the given bounds.
static LLVM_ABI ConstantRange makeGuaranteedNoWrapRegion(Instruction::BinaryOps BinOp, const ConstantRange &Other, unsigned NoWrapKind)
Produce the largest range containing all X such that "X BinOp Y" is guaranteed not to wrap (overflow)...
LLVM_ABI unsigned getMinSignedBits() const
Compute the maximal number of bits needed to represent every value in this signed range.
uint32_t getBitWidth() const
Get the bit width of this ConstantRange.
LLVM_ABI ConstantRange sub(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a subtraction of a value in this r...
LLVM_ABI ConstantRange sextOrTrunc(uint32_t BitWidth) const
Make this range have the bit width given by BitWidth.
static LLVM_ABI ConstantRange makeExactNoWrapRegion(Instruction::BinaryOps BinOp, const APInt &Other, unsigned NoWrapKind)
Produce the range that contains X if and only if "X BinOp Other" does not wrap.
This is an important base class in LLVM.
Definition Constant.h:44
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
LLVM_ABI const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
LLVM_ABI unsigned getIndexTypeSizeInBits(Type *Ty) const
The size in bits of the index used in GEP calculation for this type.
LLVM_ABI IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition DataLayout.h:794
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition DenseMap.h:757
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition DenseMap.h:762
bool empty() const
Definition DenseMap.h:716
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
iterator end()
Definition DenseMap.h:686
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT > iterator
Definition DenseMap.h:678
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
Definition DenseMap.h:793
iterator find_as(const LookupKeyT &Val)
Alternate version of find() which allows a different, and possibly less expensive,...
Definition DenseMap.h:779
void swap(DenseMapBase &RHS)
Definition DenseMap.h:975
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:827
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:856
DomTreeNodeBase * getIDom() const
NodeT * getBlock() const
Analysis pass which computes a DominatorTree.
Definition Dominators.h:241
DomTreeNodeBase< NodeT > * getNode(const NodeT *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
Legacy analysis pass which computes a DominatorTree.
Definition Dominators.h:277
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
LLVM_ABI bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
This instruction extracts a single (scalar) element from a VectorType value.
This instruction extracts a struct member or array element value from an aggregate value.
Insertion token: a failed lookup fills it in, the matching insert consumes it.
Definition FoldingSet.h:284
This class describes a reference to an interned FoldingSetNodeID, which can be a useful to store node...
Definition FoldingSet.h:123
This class is used to gather all the unique data bits of a node.
Definition FoldingSet.h:162
void AddInteger(signed I)
Definition FoldingSet.h:190
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionPass(char &pid)
Definition Pass.h:314
Represents flags for the getelementptr instruction/expression.
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
static GEPNoWrapFlags none()
static LLVM_ABI Type * getTypeAtIndex(Type *Ty, Value *Idx)
Return the type of the element at the given index of an indexable type.
Module * getParent()
Get the module that this global value is contained inside of...
static bool isPrivateLinkage(LinkageTypes Linkage)
static bool isInternalLinkage(LinkageTypes Linkage)
This instruction compares its operands according to the predicate given to the constructor.
CmpPredicate getCmpPredicate() const
static bool isGE(Predicate P)
Return true if the predicate is SGE or UGE.
CmpPredicate getSwappedCmpPredicate() const
static LLVM_ABI bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
static bool isLT(Predicate P)
Return true if the predicate is SLT or ULT.
CmpPredicate getInverseCmpPredicate() const
Predicate getNonStrictCmpPredicate() const
For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
static bool isGT(Predicate P)
Return true if the predicate is SGT or UGT.
Predicate getFlippedSignednessPredicate() const
For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->EQ.
static CmpPredicate getInverseCmpPredicate(CmpPredicate Pred)
bool isEquality() const
Return true if this predicate is either EQ or NE.
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
static LLVM_ABI std::optional< bool > isImpliedByMatchingCmp(CmpPredicate Pred1, CmpPredicate Pred2)
Determine if Pred1 implies Pred2 is true, false, or if nothing can be inferred about the implication,...
bool isRelational() const
Return true if the predicate is relational (not EQ or NE).
static bool isLE(Predicate P)
Return true if the predicate is SLE or ULE.
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
LLVM_ABI bool hasNoUnsignedWrap() const LLVM_READONLY
Determine whether the no unsigned wrap flag is set.
LLVM_ABI bool hasNoSignedWrap() const LLVM_READONLY
Determine whether the no signed wrap flag is set.
LLVM_ABI bool isIdenticalToWhenDefined(const Instruction *I, bool IntersectAttrs=false) const LLVM_READONLY
This is like isIdenticalTo, except that it ignores the SubclassOptionalData flags,...
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
A helper class to return the specified delimiter string after the first invocation of operator String...
An instruction for reading from memory.
Analysis pass that exposes the LoopInfo for a function.
Definition LoopInfo.h:598
bool contains(const LoopT *L) const
Return true if the specified loop is contained within this loop.
BlockT * getHeader() const
BlockT * getLoopPredecessor() const
If the given loop's header has exactly one unique predecessor outside the loop, return it.
LoopT * getParentLoop() const
Return the parent loop if it exists or nullptr for top level loops.
unsigned getLoopDepth(const BlockT *BB) const
Return the loop nesting level of the specified block.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
The legacy pass manager's analysis pass to compute loop information.
Definition LoopInfo.h:623
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
LLVM_ABI bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
Definition LoopInfo.cpp:68
Metadata node.
Definition Metadata.h:1081
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
Definition Operator.h:43
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
Definition Operator.h:78
bool hasNoSignedWrap() const
Test whether this operation is known to never undergo signed overflow, aka the nsw property.
Definition Operator.h:113
bool hasNoUnsignedWrap() const
Test whether this operation is known to never undergo unsigned overflow, aka the nuw property.
Definition Operator.h:107
iterator_range< const_block_iterator > blocks() const
op_range incoming_values()
Value * getIncomingValueForBlock(const BasicBlock *BB) const
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
PointerIntPair - This class implements a pair of a pointer and small integer.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
LLVM_ABI void addPredicate(const SCEVPredicate &Pred)
Adds a new predicate.
LLVM_ABI const SCEVPredicate & getPredicate() const
LLVM_ABI const SCEV * getPredicatedSCEV(const SCEV *Expr)
Returns the rewritten SCEV for Expr in the context of the current SCEV predicate.
LLVM_ABI bool areAddRecsEqualWithPreds(const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2, ArrayRef< const SCEVPredicate * > ExtraPreds={}) const
Check if AR1 and AR2 are equal, while taking into account Equal predicates in Preds and ExtraPreds.
LLVM_ABI const SCEVAddRecExpr * getAsAddRec(Value *V, SmallVectorImpl< const SCEVPredicate * > *WrapPredsAdded=nullptr)
Attempts to produce an AddRecExpr for V by adding additional SCEV predicates.
LLVM_ABI void print(raw_ostream &OS, unsigned Depth) const
Print the SCEV mappings done by the Predicated Scalar Evolution.
LLVM_ABI PredicatedScalarEvolution(ScalarEvolution &SE, Loop &L)
LLVM_ABI unsigned getSmallConstantMaxTripCount()
Returns the upper bound of the loop trip count as a normal unsigned value, or 0 if the trip count is ...
LLVM_ABI void addPredicates(ArrayRef< const SCEVPredicate * > Preds)
Adds all predicates in Preds.
LLVM_ABI const SCEV * getBackedgeTakenCount()
Get the (predicated) backedge count for the analyzed loop.
LLVM_ABI const SCEV * getSymbolicMaxBackedgeTakenCount()
Get the (predicated) symbolic max backedge count for the analyzed loop.
LLVM_ABI const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalysisChecker getChecker() const
Build a checker for this PreservedAnalyses and the specified analysis type.
Definition Analysis.h:276
constexpr bool isValid() const
Definition Register.h:112
This node represents an addition of some number of SCEVs.
This node represents a polynomial recurrence on the trip count of the specified loop.
LLVM_ABI SCEVUse getExitValue(ScalarEvolution &SE) const
Return the value of this recurrences when its loop exits, i.e.
LLVM_ABI const SCEV * evaluateAtIteration(const SCEV *It, ScalarEvolution &SE) const
Return the value of this chain of recurrences at the specified iteration number.
void setNoWrapFlags(SCEVFlags Flags)
Set flags for a recurrence without clearing any previously set flags.
bool isAffine() const
Return true if this represents an expression A + B*x where A and B are loop invariant values.
bool isQuadratic() const
Return true if this represents an expression A + B*x + C*x^2 where A, B and C are loop invariant valu...
LLVM_ABI const SCEV * getNumIterationsInRange(const ConstantRange &Range, ScalarEvolution &SE) const
Return the number of iterations of this loop that produce values in the specified constant range.
LLVM_ABI const SCEVAddRecExpr * getPostIncExpr(ScalarEvolution &SE) const
Return an expression representing the value of this expression one iteration of the loop ahead.
SCEVUse getStepRecurrence(ScalarEvolution &SE) const
Constructs and returns the recurrence indicating how much this expression steps by.
This is the base class for unary cast operator classes.
LLVM_ABI SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, SCEVUse op, Type *ty)
This class represents an assumption that the expression LHS Pred RHS evaluates to true,...
SCEVComparePredicate(const FoldingSetNodeIDRef ID, const ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
bool isAlwaysTrue() const override
Returns true if the predicate is always true.
void print(raw_ostream &OS, unsigned Depth=0) const override
Prints a textual representation of this predicate with an indentation of Depth.
bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const override
Implementation of the SCEVPredicate interface.
This class represents a constant integer value.
ConstantInt * getValue() const
const APInt & getAPInt() const
This is the base class for unary integral cast operator classes.
LLVM_ABI SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, SCEVUse op, Type *ty)
This node is the base class min/max selections.
static enum SCEVTypes negate(enum SCEVTypes T)
This node represents multiplication of some number of SCEVs.
This node is a base class providing common functionality for n'ary operators.
ArrayRef< SCEVUse > operands() const
SCEVFlags getNoWrapFlags(SCEVFlags Mask=FlagsNoWrapMask) const
SCEVUse getOperand(unsigned i) const
This class represents an assumption made using SCEV expressions which can be checked at run-time.
SCEVPredicate(const SCEVPredicate &)=default
virtual bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const =0
Returns true if this predicate implies N.
SCEVPredicateKind Kind
This class represents a cast from a pointer to a pointer-sized integer value, without capturing the p...
This visitor recursively visits a SCEV expression and re-writes it.
const SCEV * visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr)
This class represents a signed minimum selection.
This node is the base class for sequential/in-order min/max selections.
static SCEVTypes getEquivalentNonSequentialSCEVType(SCEVTypes Ty)
This class represents a sign extension of a small integer value to a larger integer value.
Visit all nodes in the expression tree using worklist traversal.
This class represents a truncation of an integer value to a smaller integer value.
This class represents a binary unsigned division operation.
This class represents an unsigned minimum selection.
This class represents a composition of other SCEV predicates, and is the class that most clients will...
void print(raw_ostream &OS, unsigned Depth) const override
Prints a textual representation of this predicate with an indentation of Depth.
bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const override
Returns true if this predicate implies N.
SCEVUnionPredicate(ArrayRef< const SCEVPredicate * > Preds, ScalarEvolution &SE)
Union predicates don't get cached so create a dummy set ID for it.
bool isAlwaysTrue() const override
Implementation of the SCEVPredicate interface.
SCEVUnionPredicate getUnionWith(const SCEVPredicate *N, ScalarEvolution &SE) const
Returns a new SCEVUnionPredicate that is the union of this predicate and the given predicate N.
This means that we are dealing with an entirely unknown SCEV value, and only represent it as its LLVM...
This class represents the value of vscale, as used when defining the length of a scalable vector or r...
This class represents an assumption made on an AddRec expression.
IncrementWrapFlags
Similar to SCEVFlags, but with slightly different semantics for FlagNUSW.
SCEVWrapPredicate(const FoldingSetNodeIDRef ID, const SCEVAddRecExpr *AR, IncrementWrapFlags Flags)
bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const override
Returns true if this predicate implies N.
static SCEVWrapPredicate::IncrementWrapFlags setFlags(SCEVWrapPredicate::IncrementWrapFlags Flags, SCEVWrapPredicate::IncrementWrapFlags OnFlags)
void print(raw_ostream &OS, unsigned Depth=0) const override
Prints a textual representation of this predicate with an indentation of Depth.
bool isAlwaysTrue() const override
Returns true if the predicate is always true.
const SCEVAddRecExpr * getExpr() const
Implementation of the SCEVPredicate interface.
static SCEVWrapPredicate::IncrementWrapFlags clearFlags(SCEVWrapPredicate::IncrementWrapFlags Flags, SCEVWrapPredicate::IncrementWrapFlags OffFlags)
Convenient IncrementWrapFlags manipulation methods.
IncrementWrapFlags getFlags() const
Returns the set assumed no overflow flags.
This class represents a zero extension of a small integer value to a larger integer value.
This class represents an analyzed expression in the program.
unsigned short getExpressionSize() const
static constexpr auto FlagsNoWrapMask
LLVM_ABI bool isOne() const
Return true if the expression is a constant one.
SCEV(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, unsigned short ExpressionSize, Type *Ty)
static constexpr auto FlagNUW
LLVM_ABI void computeAndSetCanonical(ScalarEvolution &SE)
Compute and set the canonical SCEV, by constructing a SCEV with the same operands,...
LLVM_ABI bool isZero() const
Return true if the expression is a constant zero.
const SCEV * getCanonical() const
Return the canonical SCEV.
const SCEV * CanonicalSCEV
Pointer to the canonical version of the SCEV, i.e.
LLVM_ABI void dump() const
This method is used for debugging.
LLVM_ABI bool isAllOnesValue() const
Return true if the expression is a constant all-ones value.
LLVM_ABI bool isNonConstantNegative() const
Return true if the specified scev is negated, but not a constant.
static constexpr auto FlagNSW
LLVM_ABI ArrayRef< SCEVUse > operands() const
Return operands of this SCEV expression.
Type * getType() const
Return the LLVM type of this SCEV expression.
static constexpr auto FlagNone
LLVM_ABI void print(raw_ostream &OS) const
Print out the internal representation of this scalar to the specified stream.
SCEVTypes getSCEVType() const
static constexpr auto FlagNW
Analysis pass that exposes the ScalarEvolution for a function.
LLVM_ABI ScalarEvolution run(Function &F, FunctionAnalysisManager &AM)
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
void print(raw_ostream &OS, const Module *=nullptr) const override
print - Print out the internal state of the pass.
bool runOnFunction(Function &F) override
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
void releaseMemory() override
releaseMemory() - This member can be implemented by a pass if it wants to be able to release its memo...
void verifyAnalysis() const override
verifyAnalysis() - This member can be implemented by a analysis pass to check state of analysis infor...
static LoopGuards collectOnDemand(const Loop *L, ScalarEvolution &SE)
Return loop guards for loop L, which are collected on first use.
The main scalar evolution driver.
LLVM_ABI const SCEV * getUDivExpr(SCEVUse LHS, SCEVUse RHS)
Get a canonical unsigned division expression, or something simpler if possible.
const SCEV * getConstantMaxBackedgeTakenCount(const Loop *L)
When successful, this returns a SCEVConstant that is greater than or equal to (i.e.
const DataLayout & getDataLayout() const
Return the DataLayout associated with the module this SCEV instance is operating on.
LLVM_ABI bool isKnownNonNegative(const SCEV *S)
Test if the given expression is known to be non-negative.
LLVM_ABI bool isKnownOnEveryIteration(CmpPredicate Pred, const SCEVAddRecExpr *LHS, const SCEV *RHS)
Test if the condition described by Pred, LHS, RHS is known to be true on every iteration of the loop ...
static bool hasFlags(SCEVFlags Flags, SCEVFlags TestFlags)
LLVM_ABI std::optional< LoopInvariantPredicate > getLoopInvariantExitCondDuringFirstIterationsImpl(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L, const Instruction *CtxI, const SCEV *MaxIter)
LLVM_ABI const SCEV * getZeroExtendExpr(SCEVUse Op, Type *Ty, unsigned Depth=0)
LLVM_ABI const SCEV * getUDivCeilSCEV(const SCEV *N, const SCEV *D)
Compute ceil(N / D).
LLVM_ABI std::optional< LoopInvariantPredicate > getLoopInvariantExitCondDuringFirstIterations(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L, const Instruction *CtxI, const SCEV *MaxIter)
If the result of the predicate LHS Pred RHS is loop invariant with respect to L at given Context duri...
LLVM_ABI Type * getWiderType(Type *Ty1, Type *Ty2) const
LLVM_ABI const SCEV * getAbsExpr(const SCEV *Op, bool IsNSW)
LLVM_ABI bool isKnownNonPositive(const SCEV *S)
Test if the given expression is known to be non-positive.
LLVM_ABI const SCEV * getElementCount(Type *Ty, ElementCount EC, SCEVFlags Flags=SCEV::FlagNone)
LLVM_ABI bool isKnownNegative(const SCEV *S)
Test if the given expression is known to be negative.
LLVM_ABI const SCEV * getPredicatedConstantMaxBackedgeTakenCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > &Predicates)
Similar to getConstantMaxBackedgeTakenCount, except it will add a set of SCEV predicates to Predicate...
LLVM_ABI const SCEV * removePointerBase(const SCEV *S)
Compute an expression equivalent to S - getPointerBase(S).
LLVM_ABI bool isLoopEntryGuardedByCond(const Loop *L, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test whether entry to the loop is protected by a conditional between LHS and RHS.
LLVM_ABI bool isKnownNonZero(const SCEV *S)
Test if the given expression is known to be non-zero.
LLVM_ABI const SCEV * getURemExpr(SCEVUse LHS, SCEVUse RHS)
Represents an unsigned remainder expression based on unsigned division.
LLVM_ABI const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
LLVM_ABI const SCEV * getSMinExpr(SCEVUse LHS, SCEVUse RHS)
LLVM_ABI const SCEV * getUMaxFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS)
Promote the operands to the wider of the types using zero-extension, and then perform a umax operatio...
const SCEV * getZero(Type *Ty)
Return a SCEV for the constant 0 of a specific type.
LLVM_ABI bool willNotOverflow(Instruction::BinaryOps BinOp, bool Signed, const SCEV *LHS, const SCEV *RHS, const Instruction *CtxI=nullptr)
Is operation BinOp between LHS and RHS provably does not have a signed/unsigned overflow (Signed)?
LLVM_ABI const SCEV * getMinusSCEV(SCEVUse LHS, SCEVUse RHS, SCEVFlags Flags=SCEV::FlagNone, unsigned Depth=0)
Return LHS-RHS.
LLVM_ABI ExitLimit computeExitLimitFromCond(const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit, bool AllowPredicates=false)
Compute the number of times the backedge of the specified loop will execute if its exit condition wer...
LLVM_ABI const SCEV * getMinMaxExpr(SCEVTypes Kind, SmallVectorImpl< SCEVUse > &Operands)
LLVM_ABI const SCEVPredicate * getEqualPredicate(const SCEV *LHS, const SCEV *RHS)
LLVM_ABI unsigned getSmallConstantTripMultiple(const Loop *L, const SCEV *ExitCount)
Returns the largest constant divisor of the trip count as a normal unsigned value,...
LLVM_ABI SCEVUse getSCEVAtScope(const SCEV *S, const Loop *L)
Return a SCEV expression for the specified value at the specified scope in the program.
LLVM_ABI uint64_t getTypeSizeInBits(Type *Ty) const
Return the size in bits of the specified type, for which isSCEVable must return true.
LLVM_ABI void registerUser(const SCEV *User, ArrayRef< SCEVUse > Ops)
Notify this ScalarEvolution that User directly uses SCEVs in Ops.
LLVM_ABI const SCEV * getConstant(ConstantInt *V)
LLVM_ABI const SCEV * getPredicatedBackedgeTakenCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > &Predicates)
Similar to getBackedgeTakenCount, except it will add a set of SCEV predicates to Predicates that are ...
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
ConstantRange getSignedRange(const SCEV *S)
Determine the signed range for a particular SCEV.
LLVM_ABI const SCEV * getNoopOrSignExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
static SCEVFlags setFlags(SCEVFlags Flags, SCEVFlags OnFlags)
static LLVM_ABI bool isGuaranteedNotToBePoison(const SCEV *Op)
Returns true if Op is guaranteed to not be poison.
bool loopHasNoAbnormalExits(const Loop *L)
Return true if the loop has no abnormal exits.
LLVM_ABI const SCEV * getTripCountFromExitCount(const SCEV *ExitCount)
A version of getTripCountFromExitCount below which always picks an evaluation type which can not resu...
LLVM_ABI ScalarEvolution(Function &F, TargetLibraryInfo &TLI, AssumptionCache &AC, DominatorTree &DT, LoopInfo &LI)
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
LLVM_ABI const SCEV * getTruncateOrNoop(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI void forgetValues(ArrayRef< Value * > Values)
Batched forgetValue: invalidates all Values in one shared def-use walk, avoiding the redundant re-tra...
LLVM_ABI const SCEV * getSequentialMinMaxExpr(SCEVTypes Kind, SmallVectorImpl< SCEVUse > &Operands)
LLVM_ABI const SCEV * getCastExpr(SCEVTypes Kind, SCEVUse Op, Type *Ty)
LLVM_ABI std::optional< bool > evaluatePredicateAt(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Instruction *CtxI)
Check whether the condition described by Pred, LHS, and RHS is true or false in the given Context.
LLVM_ABI SCEVUse getAddRecExpr(SCEVUse Start, SCEVUse Step, const Loop *L, SCEVFlagsPair Flags)
Get an add recurrence expression for the specified loop.
LLVM_ABI unsigned getSmallConstantMaxTripCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > *Predicates=nullptr)
Returns the upper bound of the loop trip count as a normal unsigned value.
LLVM_ABI bool isKnownMultipleOf(const SCEV *S, uint64_t M, SmallVectorImpl< const SCEVPredicate * > *Predicates=nullptr)
Check that S is a multiple of M.
LLVM_ABI bool isBackedgeTakenCountMaxOrZero(const Loop *L)
Return true if the backedge taken count is either the value returned by getConstantMaxBackedgeTakenCo...
LLVM_ABI void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
LLVM_ABI bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
LLVM_ABI bool isKnownPositive(const SCEV *S)
Test if the given expression is known to be positive.
LLVM_ABI bool SimplifyICmpOperands(CmpPredicate &Pred, SCEVUse &LHS, SCEVUse &RHS, unsigned Depth=0)
Simplify LHS and RHS in a comparison with predicate Pred.
APInt getUnsignedRangeMin(const SCEV *S)
Determine the min of the unsigned range for a particular SCEV.
static SCEVFlags clearFlags(SCEVFlags Flags, SCEVFlags OffFlags)
LLVM_ABI const SCEV * getOffsetOfExpr(Type *IntTy, StructType *STy, unsigned FieldNo)
Return an expression for offsetof on the given field with type IntTy.
LLVM_ABI LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L)
Return the "disposition" of the given SCEV with respect to the given loop.
static SCEVFlags maskFlags(SCEVFlags Flags, SCEVFlags Mask)
Convenient SCEVFlags manipulation.
LLVM_ABI bool containsAddRecurrence(const SCEV *S)
Return true if the SCEV is a scAddRecExpr or it contains scAddRecExpr.
LLVM_ABI const SCEV * getTruncateExpr(SCEVUse Op, Type *Ty, unsigned Depth=0)
LLVM_ABI SCEVUse getAddExpr(SmallVectorImpl< SCEVUse > &Ops, SCEVFlagsPair Flags={}, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
LLVM_ABI const SCEV * getZeroExtendExprImpl(SCEVUse Op, Type *Ty, unsigned Depth=0)
LLVM_ABI bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
LLVM_ABI Type * getEffectiveSCEVType(Type *Ty) const
Return a type with the same bitwidth as the given type and which represents how SCEV will treat the g...
LLVM_ABI const SCEVPredicate * getComparePredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
LLVM_ABI bool haveSameSign(const SCEV *S1, const SCEV *S2)
Return true if we know that S1 and S2 must have the same sign.
LLVM_ABI const SCEV * getNotSCEV(const SCEV *V)
Return the SCEV object corresponding to ~V.
LLVM_ABI bool instructionCouldExistWithOperands(const SCEV *A, const SCEV *B)
Return true if there exists a point in the program at which both A and B could be operands to the sam...
LLVM_ABI std::optional< SCEVFlags > getStrengthenedNoWrapFlagsFromBinOp(const OverflowingBinaryOperator *OBO)
Parse NSW/NUW flags from add/sub/mul IR binary operation Op into SCEV no-wrap flags,...
ConstantRange getUnsignedRange(const SCEV *S)
Determine the unsigned range for a particular SCEV.
LLVM_ABI void print(raw_ostream &OS) const
LLVM_ABI const SCEV * getAnyExtendExpr(SCEVUse Op, Type *Ty)
getAnyExtendExpr - Return a SCEV for the given operand extended with unspecified bits out to the give...
LLVM_ABI const SCEV * getPredicatedExitCount(const Loop *L, const BasicBlock *ExitingBlock, SmallVectorImpl< const SCEVPredicate * > *Predicates, ExitCountKind Kind=Exact)
Same as above except this uses the predicated backedge taken info and may require predicates.
LLVM_ABI void forgetTopmostLoop(const Loop *L)
LLVM_ABI void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
APInt getSignedRangeMin(const SCEV *S)
Determine the min of the signed range for a particular SCEV.
LLVM_ABI bool isLoopUniform(const SCEV *S, const Loop *L)
Returns true if the given SCEV is loop-uniform with respect to the specified loop L.
LLVM_ABI const SCEV * getNoopOrAnyExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI void forgetBlockAndLoopDispositions(Value *V=nullptr)
Called when the client has changed the disposition of values in a loop or block.
LLVM_ABI const SCEV * getSignExtendExpr(SCEVUse Op, Type *Ty, unsigned Depth=0)
LLVM_ABI const SCEV * getUMaxExpr(SCEVUse LHS, SCEVUse RHS)
LLVM_ABI std::optional< LoopInvariantPredicate > getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L, const Instruction *CtxI=nullptr)
If the result of the predicate LHS Pred RHS is loop invariant with respect to L, return a LoopInvaria...
LLVM_ABI const SCEV * getStoreSizeOfExpr(Type *IntTy, Type *StoreTy)
Return an expression for the store size of StoreTy that is type IntTy.
LLVM_ABI const SCEVPredicate * getWrapPredicate(const SCEVAddRecExpr *AR, SCEVWrapPredicate::IncrementWrapFlags AddedFlags)
LLVM_ABI bool isLoopBackedgeGuardedByCond(const Loop *L, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test whether the backedge of the loop is protected by a conditional between LHS and RHS.
LLVM_ABI APInt getNonZeroConstantMultiple(const SCEV *S)
const SCEV * getMinusOne(Type *Ty)
Return a SCEV for the constant -1 of a specific type.
LLVM_ABI bool hasLoopInvariantBackedgeTakenCount(const Loop *L)
Return true if the specified loop has an analyzable loop-invariant backedge-taken count.
LLVM_ABI BlockDisposition getBlockDisposition(const SCEV *S, const BasicBlock *BB)
Return the "disposition" of the given SCEV with respect to the given block.
LLVM_ABI const SCEV * getNoopOrZeroExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA, FunctionAnalysisManager::Invalidator &Inv)
LLVM_ABI const SCEV * getUMinFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS, bool Sequential=false)
Promote the operands to the wider of the types using zero-extension, and then perform a umin operatio...
LLVM_ABI bool loopIsFiniteByAssumption(const Loop *L)
Return true if this loop is finite by assumption.
LLVM_ABI SCEVUse getSCEVAtExit(const SCEV *S, const Loop *L, const BasicBlock *ExitingBlock)
Return the SCEV expression at the specified loop exit.
LLVM_ABI const SCEV * getExistingSCEV(Value *V)
Return an existing SCEV for V if there is one, otherwise return nullptr.
LLVM_ABI APInt getConstantMultiple(const SCEV *S, const Instruction *CtxI=nullptr)
Returns the max constant multiple of S.
LoopDisposition
An enum describing the relationship between a SCEV and a loop.
@ LoopComputable
The SCEV varies predictably with the loop.
@ LoopVariant
The SCEV is loop-variant (unknown).
@ LoopInvariant
The SCEV is loop-invariant.
@ LoopUniform
The SCEV is loop-uniform.
LLVM_ABI bool isKnownToBeAPowerOfTwo(const SCEV *S, bool OrZero=false, bool OrNegative=false)
Test if the given expression is known to be a power of 2.
LLVM_ABI void forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V)
Forget LCSSA phi node V of loop L to which a new predecessor was added, such that it may no longer be...
LLVM_ABI bool containsUndefs(const SCEV *S) const
Return true if the SCEV expression contains an undef value.
LLVM_ABI std::optional< MonotonicPredicateType > getMonotonicPredicateType(const SCEVAddRecExpr *LHS, ICmpInst::Predicate Pred)
If, for all loop invariant X, the predicate "LHS `Pred` X" is monotonically increasing or decreasing,...
LLVM_ABI const SCEV * getCouldNotCompute()
LLVM_ABI bool isAvailableAtLoopEntry(const SCEV *S, const Loop *L)
Determine if the SCEV can be evaluated at loop's entry.
LLVM_ABI uint32_t getMinTrailingZeros(const SCEV *S, const Instruction *CtxI=nullptr)
Determine the minimum number of zero bits that S is guaranteed to end in (at every loop iteration).
BlockDisposition
An enum describing the relationship between a SCEV and a basic block.
@ DominatesBlock
The SCEV dominates the block.
@ ProperlyDominatesBlock
The SCEV properly dominates the block.
@ DoesNotDominateBlock
The SCEV does not dominate the block.
LLVM_ABI const SCEV * getExitCount(const Loop *L, const BasicBlock *ExitingBlock, ExitCountKind Kind=Exact)
Return the number of times the backedge executes before the given exit would be taken; if not exactly...
LLVM_ABI void getPoisonGeneratingValues(SmallPtrSetImpl< const Value * > &Result, const SCEV *S)
Return the set of Values that, if poison, will definitively result in S being poison as well.
LLVM_ABI void setNoWrapFlags(SCEVAddRecExpr *AddRec, SCEVFlags Flags)
Update no-wrap flags of an AddRec.
LLVM_ABI void forgetLoopDispositions()
Called when the client has changed the disposition of values in this loop.
LLVM_ABI const SCEV * getVScale(Type *Ty)
LLVM_ABI SCEVUse getMulExpr(SmallVectorImpl< SCEVUse > &Ops, SCEVFlagsPair Flags={}, unsigned Depth=0)
Get a canonical multiply expression, or something simpler if possible.
LLVM_ABI unsigned getSmallConstantTripCount(const Loop *L)
Returns the exact trip count of the loop if we can compute it, and the result is a small constant.
LLVM_ABI bool hasComputableLoopEvolution(const SCEV *S, const Loop *L)
Return true if the given SCEV changes value in a known way in the specified loop.
LLVM_ABI const SCEV * getPointerBase(const SCEV *V)
Transitively follow the chain of pointer-type operands until reaching a SCEV that does not have a sin...
LLVM_ABI void forgetAllLoops()
LLVM_ABI const SCEV * getSignExtendExprImpl(SCEVUse Op, Type *Ty, unsigned Depth=0)
LLVM_ABI bool dominates(const SCEV *S, const BasicBlock *BB)
Return true if elements that makes up the given SCEV dominate the specified basic block.
APInt getUnsignedRangeMax(const SCEV *S)
Determine the max of the unsigned range for a particular SCEV.
ExitCountKind
The terms "backedge taken count" and "exit count" are used interchangeably to refer to the number of ...
@ SymbolicMaximum
An expression which provides an upper bound on the exact trip count.
@ ConstantMaximum
A constant which provides an upper bound on the exact trip count.
@ Exact
An expression exactly describing the number of times the backedge has executed when a loop is exited.
LLVM_ABI bool isKnownPredicate(CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
LLVM_ABI const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
LLVM_ABI const SCEV * getPtrToAddrExpr(const SCEV *Op)
LLVM_ABI const SCEVAddRecExpr * convertSCEVToAddRecWithPredicates(const SCEV *S, const Loop *L, SmallVectorImpl< const SCEVPredicate * > &Preds)
Tries to convert the S expression to an AddRec expression, adding additional predicates to Preds as r...
LLVM_ABI const SCEV * getSMaxExpr(SCEVUse LHS, SCEVUse RHS)
LLVM_ABI const SCEV * getElementSize(Instruction *Inst)
Return the size of an element read or written by Inst.
LLVM_ABI bool isGuaranteedNotToCauseUB(const SCEV *Op)
Returns true if Op is guaranteed not to cause immediate UB.
LLVM_ABI const SCEV * getSizeOfExpr(Type *IntTy, TypeSize Size)
Return an expression for a TypeSize.
LLVM_ABI std::optional< bool > evaluatePredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Check whether the condition described by Pred, LHS, and RHS is true or false.
LLVM_ABI const SCEV * getUnknown(Value *V)
LLVM_ABI std::optional< std::pair< const SCEV *, SmallVector< const SCEVPredicate *, 3 > > > createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI)
Checks if SymbolicPHI can be rewritten as an AddRecExpr under some Predicates.
LLVM_ABI const SCEV * getTruncateOrZeroExtend(const SCEV *V, Type *Ty, unsigned Depth=0)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI bool isKnownViaInduction(CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS)
We'd like to check the predicate on every iteration of the most dominated loop between loops used in ...
LLVM_ABI std::optional< APInt > computeConstantDifference(const SCEV *LHS, const SCEV *RHS)
Compute LHS - RHS and returns the result as an APInt if it is a constant, and std::nullopt if it isn'...
LLVM_ABI bool properlyDominates(const SCEV *S, const BasicBlock *BB)
Return true if elements that makes up the given SCEV properly dominate the specified basic block.
LLVM_ABI const SCEV * getNegativeSCEV(const SCEV *V, SCEVFlags Flags=SCEV::FlagNone)
Return the SCEV object corresponding to -V.
LLVM_ABI const SCEV * getUDivExactExpr(SCEVUse LHS, SCEVUse RHS)
Get a canonical unsigned division expression, or something simpler if possible.
LLVM_ABI const SCEV * rewriteUsingPredicate(const SCEV *S, const Loop *L, const SCEVPredicate &A)
Re-writes the SCEV according to the Predicates in A.
LLVM_ABI std::pair< const SCEV *, const SCEV * > SplitIntoInitAndPostInc(const Loop *L, const SCEV *S)
Splits SCEV expression S into two SCEVs.
LLVM_ABI bool canReuseInstruction(const SCEV *S, Instruction *I, SmallVectorImpl< Instruction * > &DropPoisonGeneratingInsts)
Check whether it is poison-safe to represent the expression S using the instruction I.
LLVM_ABI bool isKnownPredicateAt(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Instruction *CtxI)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
LLVM_ABI const SCEV * getPredicatedSymbolicMaxBackedgeTakenCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > &Predicates)
Similar to getSymbolicMaxBackedgeTakenCount, except it will add a set of SCEV predicates to Predicate...
LLVM_ABI const SCEV * getGEPExpr(GEPOperator *GEP, ArrayRef< SCEVUse > IndexExprs)
Returns an expression for a GEP.
LLVM_ABI const SCEV * getUMinExpr(SCEVUse LHS, SCEVUse RHS, bool Sequential=false)
LLVM_ABI bool isBasicBlockEntryGuardedByCond(const BasicBlock *BB, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test whether entry to the basic block is protected by a conditional between LHS and RHS.
LLVM_ABI const SCEV * getTruncateOrSignExtend(const SCEV *V, Type *Ty, unsigned Depth=0)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI bool containsErasedValue(const SCEV *S) const
Return true if the SCEV expression contains a Value that has been optimised out and is now a nullptr.
const SCEV * getSymbolicMaxBackedgeTakenCount(const Loop *L)
When successful, this returns a SCEV that is greater than or equal to (i.e.
APInt getSignedRangeMax(const SCEV *S)
Determine the max of the signed range for a particular SCEV.
LLVM_ABI void verify() const
LLVMContext & getContext() const
This class represents the LLVM 'select' instruction.
Implements a dense probed hash-table based set with some number of buckets stored inline.
Definition DenseSet.h:295
size_type size() const
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition DataLayout.h:746
TypeSize getElementOffset(unsigned Idx) const
Definition DataLayout.h:777
TypeSize getSizeInBits() const
Definition DataLayout.h:757
Class to represent struct types.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:291
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:298
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
op_range operands()
Definition User.h:267
Use & Op()
Definition User.h:171
Value * getOperand(unsigned i) const
Definition User.h:207
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
iterator_range< user_iterator > users()
Definition Value.h:428
unsigned getValueID() const
Return an ID for the concrete type of this object.
Definition Value.h:545
LLVM_ABI void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition DenseSet.h:183
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition ilist_node.h:34
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
raw_ostream & indent(unsigned NumSpaces)
indent - Insert 'NumSpaces' spaces.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
const APInt & smin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be signed.
Definition APInt.h:2274
const APInt & smax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be signed.
Definition APInt.h:2279
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be unsigned.
Definition APInt.h:2284
LLVM_ABI std::optional< APInt > SolveQuadraticEquationWrap(APInt A, APInt B, APInt C, unsigned RangeWidth)
Let q(n) = An^2 + Bn + C, and BW = bit width of the value range (e.g.
Definition APInt.cpp:2859
LLVM_ABI APInt GreatestCommonDivisor(APInt A, APInt B, bool IsSigned=false)
Compute GCD of two APInt values.
Definition APInt.cpp:826
const APInt & umax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be unsigned.
Definition APInt.h:2289
constexpr bool any(E Val)
@ Entry
Definition COFF.h:862
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
LLVM_ABI Function * getDeclarationIfExists(const Module *M, ID id)
Look up the Function declaration of the intrinsic id in the Module M and return it if it exists.
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
match_combine_or< Ty... > m_CombineOr(const Ty &...Ps)
Combine pattern matchers matching any of Ps patterns.
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
bool match(Val *V, const Pattern &P)
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
auto m_BasicBlock()
Match an arbitrary basic block value and ignore it.
ExtractValue_match< Ind, Val_t > m_ExtractValue(const Val_t &V)
Match a single index ExtractValue instruction.
auto m_Value()
Match an arbitrary value and ignore it.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
match_bind< WithOverflowInst > m_WithOverflowInst(WithOverflowInst *&I)
Match a with overflow intrinsic, capturing it if we match.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
BinaryOp_match< LHS, RHS, Instruction::SDiv > m_SDiv(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
brc_match< Cond_t, match_bind< BasicBlock >, match_bind< BasicBlock > > m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F)
CastOperator_match< OpTy, Instruction::PtrToInt > m_PtrToInt(const OpTy &Op)
Matches PtrToInt.
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
bind_cst_ty m_scev_APInt(const APInt *&C)
Match an SCEV constant and bind it to an APInt.
cst_pred_ty< is_all_ones > m_scev_AllOnes()
Match an integer with all bits set.
SCEVUnaryExpr_match< SCEVZeroExtendExpr, Op0_t > m_scev_ZExt(const Op0_t &Op0)
is_undef_or_poison m_scev_UndefOrPoison()
Match an SCEVUnknown wrapping undef or poison.
SCEVBinaryExpr_match< SCEVMinMaxExpr, Op0_t, Op1_t > m_scev_MinMax(const Op0_t &Op0, const Op1_t &Op1)
cst_pred_ty< is_one > m_scev_One()
Match an integer 1.
specificloop_ty m_SpecificLoop(const Loop *L)
SCEVUnaryExpr_match< SCEVSignExtendExpr, Op0_t > m_scev_SExt(const Op0_t &Op0)
match_bind< const SCEVMulExpr > m_scev_Mul(const SCEVMulExpr *&V)
cst_pred_ty< is_zero > m_scev_Zero()
Match an integer 0.
SCEVUnaryExpr_match< SCEVTruncateExpr, Op0_t > m_scev_Trunc(const Op0_t &Op0)
bool match(const SCEV *S, const Pattern &P)
SCEVBinaryExpr_match< SCEVUDivExpr, Op0_t, Op1_t > m_scev_UDiv(const Op0_t &Op0, const Op1_t &Op1)
SCEVBinaryExpr_match< SCEVAddExpr, Op0_t, Op1_t, SCEV::FlagNSW > m_scev_NSWAdd(const Op0_t &Op0, const Op1_t &Op1)
specificscev_ty m_scev_Specific(const SCEV *S)
Match if we have a specific specified SCEV.
SCEVAffineAddRec_match< Op0_t, Op1_t, match_isa< const Loop > > m_scev_AffineAddRec(const Op0_t &Op0, const Op1_t &Op1)
match_bind< const SCEVUnknown > m_SCEVUnknown(const SCEVUnknown *&V)
SCEVBinaryExpr_match< SCEVMulExpr, Op0_t, Op1_t, SCEV::FlagNUW, true > m_scev_c_NUWMul(const Op0_t &Op0, const Op1_t &Op1)
SCEVBinaryExpr_match< SCEVUMaxExpr, Op0_t, Op1_t, SCEV::FlagNone, true > m_scev_UMax(const Op0_t &Op0, const Op1_t &Op1)
SCEVBinaryExpr_match< SCEVSMaxExpr, Op0_t, Op1_t, SCEV::FlagNone, true > m_scev_SMax(const Op0_t &Op0, const Op1_t &Op1)
SCEVBinaryExpr_match< SCEVMulExpr, Op0_t, Op1_t, SCEV::FlagNone, true > m_scev_c_Mul(const Op0_t &Op0, const Op1_t &Op1)
match_bind< const SCEVAddExpr > m_scev_Add(const SCEVAddExpr *&V)
SCEVURem_match< Op0_t, Op1_t > m_scev_URem(Op0_t LHS, Op1_t RHS, ScalarEvolution &SE)
Match the mathematical pattern A - (A / B) * B, where A and B can be arbitrary expressions.
@ Valid
The data is already valid.
LocationClass< Ty > location(Ty &L)
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
Definition CoroShape.h:32
constexpr double e
NodeAddr< PhiNode * > Phi
Definition RDFGraph.h:390
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
void visitAll(const SCEV *Root, SV &Visitor)
Use SCEVTraversal to visit all nodes in the given expression tree.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
SaveAndRestore(T &) -> SaveAndRestore< T >
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
LLVM_ABI bool canCreatePoison(const Operator *Op, bool ConsiderFlagsAndMetadata=true)
LLVM_ABI bool mustTriggerUB(const Instruction *I, const SmallPtrSetImpl< const Value * > &KnownPoison)
Return true if the given instruction must trigger undefined behavior when I is executed with any oper...
RelativeUniformCounterPtr Values
Definition InstrProf.h:91
@ Known
Known to have no common set bits.
@ Dead
Unused definition.
InterleavedRange< Range > interleaved(const Range &R, StringRef Separator=", ", StringRef Prefix="", StringRef Suffix="")
Output range R as a sequence of interleaved elements.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
auto successors(const MachineBasicBlock *BB)
scope_exit(Callable) -> scope_exit< Callable >
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
@ BinaryOp
One of the operands is a binary op.
constexpr from_range_t from_range
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
bool set_is_subset(const S1Ty &S1, const S2Ty &S2)
set_is_subset(A, B) - Return true iff A in B
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2224
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition MathExtras.h:244
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
void * PointerTy
LLVM_ABI bool VerifySCEV
auto uninitialized_copy(R &&Src, IterTy Dst)
Definition STLExtras.h:2127
bool isa_and_nonnull(const Y &Val)
Definition Casting.h:676
LLVM_ABI unsigned ComputeNumSignBits(const Value *Op, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Return the number of times the sign bit of the register is replicated into the other bits.
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
auto map_range(ContainerTy &&C, FuncTy F)
Return a range that applies F to the elements of C.
Definition STLExtras.h:366
LLVM_ABI bool canConstantFoldCallTo(const CallBase *Call, const Function *F, const TargetLibraryInfo *TLI=nullptr)
canConstantFoldCallTo - Return true if its even possible to fold a call to the specified function.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
LLVM_ABI Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q)
See if we can compute a simplified version of this instruction.
LLVM_ABI bool isOverflowIntrinsicNoWrap(const WithOverflowInst *WO, const DominatorTree &DT)
Returns true if the arithmetic part of the WO 's result is used only along the paths control dependen...
DomTreeNodeBase< BasicBlock > DomTreeNode
Definition Dominators.h:65
LLVM_ABI bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start, Value *&Step)
Attempt to match a simple first order recurrence cycle of the form: iv = phi Ty [Start,...
LLVM_ABI Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Function *CtxF=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
const AnalysisOptions & getAnalysisOptions(const LLVMContext &C)
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
Definition STLExtras.h:2216
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
LLVM_ABI bool isMustProgress(const Loop *L)
Return true if this loop can be assumed to make progress.
LLVM_ABI bool impliesPoison(const Value *ValAssumedPoison, const Value *V)
Return true if V is poison given that ValAssumedPoison is already poison.
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
LLVM_ABI bool isFinite(const Loop *L)
Return true if this loop can be assumed to run for a finite number of iterations.
unsigned short computeExpressionSize(ArrayRef< SCEVUse > Args)
LLVM_ABI bool programUndefinedIfPoison(const Instruction *Inst)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
bool isPointerTy(const Type *T)
Definition SPIRVUtils.h:373
LLVM_ABI ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:389
LLVM_ABI bool propagatesPoison(const Use &PoisonOp)
Return true if PoisonOp's user yields poison or raises UB if its operand PoisonOp is poison.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Mul
Product of integers.
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ SMin
Signed integer min implemented in terms of select(cmp()).
@ Add
Sum of integers.
@ UMax
Unsigned integer max implemented in terms of select(cmp()).
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition STLExtras.h:2028
DWARFExpression::Operation Op
auto max_element(R &&Range)
Provide wrappers to std::max_element which take ranges instead of having to pass begin/end explicitly...
Definition STLExtras.h:2104
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
SCEVFlags
SCEVFlags are bitfield indices into SCEV's SubclassData.
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr unsigned BitWidth
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1933
LLVM_ABI bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition STLExtras.h:2035
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:341
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
Definition MathExtras.h:249
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
iterator_range< df_iterator< T > > depth_first(const T &G)
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
bool equal(L &&LRange, R &&RRange)
Wrapper function around std::equal to detect if pair-wise elements between two ranges are the same.
Definition STLExtras.h:2162
LLVM_ABI bool isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Returns true if V cannot be poison, but may be undef.
LLVM_ABI Constant * ConstantFoldInstOperands(const Instruction *I, ArrayRef< Constant * > Ops, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, bool AllowNonDeterministic=true)
ConstantFoldInstOperands - Attempt to constant fold an instruction with the specified operands.
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
constexpr detail::IsaCheckPredicate< Types... > IsaPred
Function object wrapper for the llvm::isa type check.
Definition Casting.h:866
SCEVUseT< const SCEV * > SCEVUse
bool SCEVExprContains(const SCEV *Root, PredTy Pred)
Return true if any node in Root satisfies the predicate Pred.
Implement std::hash so that hash_code can be used in STL containers.
Definition BitVector.h:879
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
#define NC
Definition regutils.h:42
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition Analysis.h:29
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
Definition KnownBits.h:315
static LLVM_ABI KnownBits ashr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for ashr(LHS, RHS).
static LLVM_ABI KnownBits lshr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for lshr(LHS, RHS).
static LLVM_ABI KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
An object of this class is returned by queries that could not be answered.
static LLVM_ABI bool classof(const SCEV *S)
Methods for support type inquiry through isa, cast, and dyn_cast:
The no-wrap flags to apply when creating a SCEV expression, to the expression and use respectively.
SCEVFlags ExprFlags
Flags applied directly to a SCEV expression, must be valid wherever the expression is valid.
SCEVFlags UseFlags
Flags only applied to a SCEVUse.
static SCEVUseT getCommon(SCEVUseT A, SCEVUseT B)
SCEVPtrT getPointer() const
This class defines a simple visitor class that may be used for various SCEV analysis purposes.
A utility class that uses RAII to save and restore the value of a variable.
Information about the number of loop iterations for which a loop exit's branch condition evaluates to...
LLVM_ABI ExitLimit(const SCEV *E)
Construct either an exact exit limit from a constant, or an unknown one from a SCEVCouldNotCompute.
SmallVector< const SCEVPredicate *, 4 > Predicates
A vector of predicate guards for this ExitLimit.