LLVM 22.0.0git
ScalarEvolution.cpp
Go to the documentation of this file.
1//===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the scalar evolution analysis
10// engine, which is used primarily to analyze expressions involving induction
11// variables in loops.
12//
13// There are several aspects to this library. First is the representation of
14// scalar expressions, which are represented as subclasses of the SCEV class.
15// These classes are used to represent certain types of subexpressions that we
16// can handle. We only create one SCEV of a particular shape, so
17// pointer-comparisons for equality are legal.
18//
19// One important aspect of the SCEV objects is that they are never cyclic, even
20// if there is a cycle in the dataflow for an expression (ie, a PHI node). If
21// the PHI node is one of the idioms that we can represent (e.g., a polynomial
22// recurrence) then we represent it directly as a recurrence node, otherwise we
23// represent it as a SCEVUnknown node.
24//
25// In addition to being able to represent expressions of various types, we also
26// have folders that are used to build the *canonical* representation for a
27// particular expression. These folders are capable of using a variety of
28// rewrite rules to simplify the expressions.
29//
30// Once the folders are defined, we can implement the more interesting
31// higher-level code, such as the code that recognizes PHI nodes of various
32// types, computes the execution count of a loop, etc.
33//
34// TODO: We should use these routines and value representations to implement
35// dependence analysis!
36//
37//===----------------------------------------------------------------------===//
38//
39// There are several good references for the techniques used in this analysis.
40//
41// Chains of recurrences -- a method to expedite the evaluation
42// of closed-form functions
43// Olaf Bachmann, Paul S. Wang, Eugene V. Zima
44//
45// On computational properties of chains of recurrences
46// Eugene V. Zima
47//
48// Symbolic Evaluation of Chains of Recurrences for Loop Optimization
49// Robert A. van Engelen
50//
51// Efficient Symbolic Analysis for Optimizing Compilers
52// Robert A. van Engelen
53//
54// Using the chains of recurrences algebra for data dependence testing and
55// induction variable substitution
56// MS Thesis, Johnie Birch
57//
58//===----------------------------------------------------------------------===//
59
61#include "llvm/ADT/APInt.h"
62#include "llvm/ADT/ArrayRef.h"
63#include "llvm/ADT/DenseMap.h"
65#include "llvm/ADT/FoldingSet.h"
66#include "llvm/ADT/STLExtras.h"
67#include "llvm/ADT/ScopeExit.h"
68#include "llvm/ADT/Sequence.h"
71#include "llvm/ADT/Statistic.h"
73#include "llvm/ADT/StringRef.h"
83#include "llvm/Config/llvm-config.h"
84#include "llvm/IR/Argument.h"
85#include "llvm/IR/BasicBlock.h"
86#include "llvm/IR/CFG.h"
87#include "llvm/IR/Constant.h"
89#include "llvm/IR/Constants.h"
90#include "llvm/IR/DataLayout.h"
92#include "llvm/IR/Dominators.h"
93#include "llvm/IR/Function.h"
94#include "llvm/IR/GlobalAlias.h"
95#include "llvm/IR/GlobalValue.h"
97#include "llvm/IR/InstrTypes.h"
98#include "llvm/IR/Instruction.h"
101#include "llvm/IR/Intrinsics.h"
102#include "llvm/IR/LLVMContext.h"
103#include "llvm/IR/Operator.h"
104#include "llvm/IR/PatternMatch.h"
105#include "llvm/IR/Type.h"
106#include "llvm/IR/Use.h"
107#include "llvm/IR/User.h"
108#include "llvm/IR/Value.h"
109#include "llvm/IR/Verifier.h"
111#include "llvm/Pass.h"
112#include "llvm/Support/Casting.h"
115#include "llvm/Support/Debug.h"
121#include <algorithm>
122#include <cassert>
123#include <climits>
124#include <cstdint>
125#include <cstdlib>
126#include <map>
127#include <memory>
128#include <numeric>
129#include <optional>
130#include <tuple>
131#include <utility>
132#include <vector>
133
134using namespace llvm;
135using namespace PatternMatch;
136using namespace SCEVPatternMatch;
137
138#define DEBUG_TYPE "scalar-evolution"
139
140STATISTIC(NumExitCountsComputed,
141 "Number of loop exits with predictable exit counts");
142STATISTIC(NumExitCountsNotComputed,
143 "Number of loop exits without predictable exit counts");
144STATISTIC(NumBruteForceTripCountsComputed,
145 "Number of loops with trip counts computed by force");
146
147#ifdef EXPENSIVE_CHECKS
148bool llvm::VerifySCEV = true;
149#else
150bool llvm::VerifySCEV = false;
151#endif
152
154 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
155 cl::desc("Maximum number of iterations SCEV will "
156 "symbolically execute a constant "
157 "derived loop"),
158 cl::init(100));
159
161 "verify-scev", cl::Hidden, cl::location(VerifySCEV),
162 cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
164 "verify-scev-strict", cl::Hidden,
165 cl::desc("Enable stricter verification with -verify-scev is passed"));
166
168 "scev-verify-ir", cl::Hidden,
169 cl::desc("Verify IR correctness when making sensitive SCEV queries (slow)"),
170 cl::init(false));
171
173 "scev-mulops-inline-threshold", cl::Hidden,
174 cl::desc("Threshold for inlining multiplication operands into a SCEV"),
175 cl::init(32));
176
178 "scev-addops-inline-threshold", cl::Hidden,
179 cl::desc("Threshold for inlining addition operands into a SCEV"),
180 cl::init(500));
181
183 "scalar-evolution-max-scev-compare-depth", cl::Hidden,
184 cl::desc("Maximum depth of recursive SCEV complexity comparisons"),
185 cl::init(32));
186
188 "scalar-evolution-max-scev-operations-implication-depth", cl::Hidden,
189 cl::desc("Maximum depth of recursive SCEV operations implication analysis"),
190 cl::init(2));
191
193 "scalar-evolution-max-value-compare-depth", cl::Hidden,
194 cl::desc("Maximum depth of recursive value complexity comparisons"),
195 cl::init(2));
196
198 MaxArithDepth("scalar-evolution-max-arith-depth", cl::Hidden,
199 cl::desc("Maximum depth of recursive arithmetics"),
200 cl::init(32));
201
203 "scalar-evolution-max-constant-evolving-depth", cl::Hidden,
204 cl::desc("Maximum depth of recursive constant evolving"), cl::init(32));
205
207 MaxCastDepth("scalar-evolution-max-cast-depth", cl::Hidden,
208 cl::desc("Maximum depth of recursive SExt/ZExt/Trunc"),
209 cl::init(8));
210
212 MaxAddRecSize("scalar-evolution-max-add-rec-size", cl::Hidden,
213 cl::desc("Max coefficients in AddRec during evolving"),
214 cl::init(8));
215
217 HugeExprThreshold("scalar-evolution-huge-expr-threshold", cl::Hidden,
218 cl::desc("Size of the expression which is considered huge"),
219 cl::init(4096));
220
222 "scev-range-iter-threshold", cl::Hidden,
223 cl::desc("Threshold for switching to iteratively computing SCEV ranges"),
224 cl::init(32));
225
227 "scalar-evolution-max-loop-guard-collection-depth", cl::Hidden,
228 cl::desc("Maximum depth for recursive loop guard collection"), cl::init(1));
229
230static cl::opt<bool>
231ClassifyExpressions("scalar-evolution-classify-expressions",
232 cl::Hidden, cl::init(true),
233 cl::desc("When printing analysis, include information on every instruction"));
234
236 "scalar-evolution-use-expensive-range-sharpening", cl::Hidden,
237 cl::init(false),
238 cl::desc("Use more powerful methods of sharpening expression ranges. May "
239 "be costly in terms of compile time"));
240
242 "scalar-evolution-max-scc-analysis-depth", cl::Hidden,
243 cl::desc("Maximum amount of nodes to process while searching SCEVUnknown "
244 "Phi strongly connected components"),
245 cl::init(8));
246
247static cl::opt<bool>
248 EnableFiniteLoopControl("scalar-evolution-finite-loop", cl::Hidden,
249 cl::desc("Handle <= and >= in finite loops"),
250 cl::init(true));
251
253 "scalar-evolution-use-context-for-no-wrap-flag-strenghening", cl::Hidden,
254 cl::desc("Infer nuw/nsw flags using context where suitable"),
255 cl::init(true));
256
257//===----------------------------------------------------------------------===//
258// SCEV class definitions
259//===----------------------------------------------------------------------===//
260
261//===----------------------------------------------------------------------===//
262// Implementation of the SCEV class.
263//
264
265#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
267 print(dbgs());
268 dbgs() << '\n';
269}
270#endif
271
272void SCEV::print(raw_ostream &OS) const {
273 switch (getSCEVType()) {
274 case scConstant:
275 cast<SCEVConstant>(this)->getValue()->printAsOperand(OS, false);
276 return;
277 case scVScale:
278 OS << "vscale";
279 return;
280 case scPtrToInt: {
281 const SCEVPtrToIntExpr *PtrToInt = cast<SCEVPtrToIntExpr>(this);
282 const SCEV *Op = PtrToInt->getOperand();
283 OS << "(ptrtoint " << *Op->getType() << " " << *Op << " to "
284 << *PtrToInt->getType() << ")";
285 return;
286 }
287 case scTruncate: {
288 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(this);
289 const SCEV *Op = Trunc->getOperand();
290 OS << "(trunc " << *Op->getType() << " " << *Op << " to "
291 << *Trunc->getType() << ")";
292 return;
293 }
294 case scZeroExtend: {
296 const SCEV *Op = ZExt->getOperand();
297 OS << "(zext " << *Op->getType() << " " << *Op << " to "
298 << *ZExt->getType() << ")";
299 return;
300 }
301 case scSignExtend: {
303 const SCEV *Op = SExt->getOperand();
304 OS << "(sext " << *Op->getType() << " " << *Op << " to "
305 << *SExt->getType() << ")";
306 return;
307 }
308 case scAddRecExpr: {
309 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(this);
310 OS << "{" << *AR->getOperand(0);
311 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
312 OS << ",+," << *AR->getOperand(i);
313 OS << "}<";
314 if (AR->hasNoUnsignedWrap())
315 OS << "nuw><";
316 if (AR->hasNoSignedWrap())
317 OS << "nsw><";
318 if (AR->hasNoSelfWrap() &&
320 OS << "nw><";
321 AR->getLoop()->getHeader()->printAsOperand(OS, /*PrintType=*/false);
322 OS << ">";
323 return;
324 }
325 case scAddExpr:
326 case scMulExpr:
327 case scUMaxExpr:
328 case scSMaxExpr:
329 case scUMinExpr:
330 case scSMinExpr:
332 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(this);
333 const char *OpStr = nullptr;
334 switch (NAry->getSCEVType()) {
335 case scAddExpr: OpStr = " + "; break;
336 case scMulExpr: OpStr = " * "; break;
337 case scUMaxExpr: OpStr = " umax "; break;
338 case scSMaxExpr: OpStr = " smax "; break;
339 case scUMinExpr:
340 OpStr = " umin ";
341 break;
342 case scSMinExpr:
343 OpStr = " smin ";
344 break;
346 OpStr = " umin_seq ";
347 break;
348 default:
349 llvm_unreachable("There are no other nary expression types.");
350 }
351 OS << "("
353 << ")";
354 switch (NAry->getSCEVType()) {
355 case scAddExpr:
356 case scMulExpr:
357 if (NAry->hasNoUnsignedWrap())
358 OS << "<nuw>";
359 if (NAry->hasNoSignedWrap())
360 OS << "<nsw>";
361 break;
362 default:
363 // Nothing to print for other nary expressions.
364 break;
365 }
366 return;
367 }
368 case scUDivExpr: {
369 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(this);
370 OS << "(" << *UDiv->getLHS() << " /u " << *UDiv->getRHS() << ")";
371 return;
372 }
373 case scUnknown:
374 cast<SCEVUnknown>(this)->getValue()->printAsOperand(OS, false);
375 return;
377 OS << "***COULDNOTCOMPUTE***";
378 return;
379 }
380 llvm_unreachable("Unknown SCEV kind!");
381}
382
384 switch (getSCEVType()) {
385 case scConstant:
386 return cast<SCEVConstant>(this)->getType();
387 case scVScale:
388 return cast<SCEVVScale>(this)->getType();
389 case scPtrToInt:
390 case scTruncate:
391 case scZeroExtend:
392 case scSignExtend:
393 return cast<SCEVCastExpr>(this)->getType();
394 case scAddRecExpr:
395 return cast<SCEVAddRecExpr>(this)->getType();
396 case scMulExpr:
397 return cast<SCEVMulExpr>(this)->getType();
398 case scUMaxExpr:
399 case scSMaxExpr:
400 case scUMinExpr:
401 case scSMinExpr:
402 return cast<SCEVMinMaxExpr>(this)->getType();
404 return cast<SCEVSequentialMinMaxExpr>(this)->getType();
405 case scAddExpr:
406 return cast<SCEVAddExpr>(this)->getType();
407 case scUDivExpr:
408 return cast<SCEVUDivExpr>(this)->getType();
409 case scUnknown:
410 return cast<SCEVUnknown>(this)->getType();
412 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
413 }
414 llvm_unreachable("Unknown SCEV kind!");
415}
416
418 switch (getSCEVType()) {
419 case scConstant:
420 case scVScale:
421 case scUnknown:
422 return {};
423 case scPtrToInt:
424 case scTruncate:
425 case scZeroExtend:
426 case scSignExtend:
427 return cast<SCEVCastExpr>(this)->operands();
428 case scAddRecExpr:
429 case scAddExpr:
430 case scMulExpr:
431 case scUMaxExpr:
432 case scSMaxExpr:
433 case scUMinExpr:
434 case scSMinExpr:
436 return cast<SCEVNAryExpr>(this)->operands();
437 case scUDivExpr:
438 return cast<SCEVUDivExpr>(this)->operands();
440 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
441 }
442 llvm_unreachable("Unknown SCEV kind!");
443}
444
445bool SCEV::isZero() const { return match(this, m_scev_Zero()); }
446
447bool SCEV::isOne() const { return match(this, m_scev_One()); }
448
449bool SCEV::isAllOnesValue() const { return match(this, m_scev_AllOnes()); }
450
453 if (!Mul) return false;
454
455 // If there is a constant factor, it will be first.
456 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0));
457 if (!SC) return false;
458
459 // Return true if the value is negative, this matches things like (-42 * V).
460 return SC->getAPInt().isNegative();
461}
462
465
467 return S->getSCEVType() == scCouldNotCompute;
468}
469
472 ID.AddInteger(scConstant);
473 ID.AddPointer(V);
474 void *IP = nullptr;
475 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
476 SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V);
477 UniqueSCEVs.InsertNode(S, IP);
478 return S;
479}
480
482 return getConstant(ConstantInt::get(getContext(), Val));
483}
484
485const SCEV *
488 return getConstant(ConstantInt::get(ITy, V, isSigned));
489}
490
493 ID.AddInteger(scVScale);
494 ID.AddPointer(Ty);
495 void *IP = nullptr;
496 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
497 return S;
498 SCEV *S = new (SCEVAllocator) SCEVVScale(ID.Intern(SCEVAllocator), Ty);
499 UniqueSCEVs.InsertNode(S, IP);
500 return S;
501}
502
504 SCEV::NoWrapFlags Flags) {
505 const SCEV *Res = getConstant(Ty, EC.getKnownMinValue());
506 if (EC.isScalable())
507 Res = getMulExpr(Res, getVScale(Ty), Flags);
508 return Res;
509}
510
512 const SCEV *op, Type *ty)
513 : SCEV(ID, SCEVTy, computeExpressionSize(op)), Op(op), Ty(ty) {}
514
515SCEVPtrToIntExpr::SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, const SCEV *Op,
516 Type *ITy)
517 : SCEVCastExpr(ID, scPtrToInt, Op, ITy) {
518 assert(getOperand()->getType()->isPointerTy() && Ty->isIntegerTy() &&
519 "Must be a non-bit-width-changing pointer-to-integer cast!");
520}
521
523 SCEVTypes SCEVTy, const SCEV *op,
524 Type *ty)
525 : SCEVCastExpr(ID, SCEVTy, op, ty) {}
526
527SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, const SCEV *op,
528 Type *ty)
530 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
531 "Cannot truncate non-integer value!");
532}
533
534SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID,
535 const SCEV *op, Type *ty)
537 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
538 "Cannot zero extend non-integer value!");
539}
540
541SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID,
542 const SCEV *op, Type *ty)
544 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
545 "Cannot sign extend non-integer value!");
546}
547
549 // Clear this SCEVUnknown from various maps.
550 SE->forgetMemoizedResults(this);
551
552 // Remove this SCEVUnknown from the uniquing map.
553 SE->UniqueSCEVs.RemoveNode(this);
554
555 // Release the value.
556 setValPtr(nullptr);
557}
558
559void SCEVUnknown::allUsesReplacedWith(Value *New) {
560 // Clear this SCEVUnknown from various maps.
561 SE->forgetMemoizedResults(this);
562
563 // Remove this SCEVUnknown from the uniquing map.
564 SE->UniqueSCEVs.RemoveNode(this);
565
566 // Replace the value pointer in case someone is still using this SCEVUnknown.
567 setValPtr(New);
568}
569
570//===----------------------------------------------------------------------===//
571// SCEV Utilities
572//===----------------------------------------------------------------------===//
573
574/// Compare the two values \p LV and \p RV in terms of their "complexity" where
575/// "complexity" is a partial (and somewhat ad-hoc) relation used to order
576/// operands in SCEV expressions.
577static int CompareValueComplexity(const LoopInfo *const LI, Value *LV,
578 Value *RV, unsigned Depth) {
580 return 0;
581
582 // Order pointer values after integer values. This helps SCEVExpander form
583 // GEPs.
584 bool LIsPointer = LV->getType()->isPointerTy(),
585 RIsPointer = RV->getType()->isPointerTy();
586 if (LIsPointer != RIsPointer)
587 return (int)LIsPointer - (int)RIsPointer;
588
589 // Compare getValueID values.
590 unsigned LID = LV->getValueID(), RID = RV->getValueID();
591 if (LID != RID)
592 return (int)LID - (int)RID;
593
594 // Sort arguments by their position.
595 if (const auto *LA = dyn_cast<Argument>(LV)) {
596 const auto *RA = cast<Argument>(RV);
597 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
598 return (int)LArgNo - (int)RArgNo;
599 }
600
601 if (const auto *LGV = dyn_cast<GlobalValue>(LV)) {
602 const auto *RGV = cast<GlobalValue>(RV);
603
604 if (auto L = LGV->getLinkage() - RGV->getLinkage())
605 return L;
606
607 const auto IsGVNameSemantic = [&](const GlobalValue *GV) {
608 auto LT = GV->getLinkage();
609 return !(GlobalValue::isPrivateLinkage(LT) ||
611 };
612
613 // Use the names to distinguish the two values, but only if the
614 // names are semantically important.
615 if (IsGVNameSemantic(LGV) && IsGVNameSemantic(RGV))
616 return LGV->getName().compare(RGV->getName());
617 }
618
619 // For instructions, compare their loop depth, and their operand count. This
620 // is pretty loose.
621 if (const auto *LInst = dyn_cast<Instruction>(LV)) {
622 const auto *RInst = cast<Instruction>(RV);
623
624 // Compare loop depths.
625 const BasicBlock *LParent = LInst->getParent(),
626 *RParent = RInst->getParent();
627 if (LParent != RParent) {
628 unsigned LDepth = LI->getLoopDepth(LParent),
629 RDepth = LI->getLoopDepth(RParent);
630 if (LDepth != RDepth)
631 return (int)LDepth - (int)RDepth;
632 }
633
634 // Compare the number of operands.
635 unsigned LNumOps = LInst->getNumOperands(),
636 RNumOps = RInst->getNumOperands();
637 if (LNumOps != RNumOps)
638 return (int)LNumOps - (int)RNumOps;
639
640 for (unsigned Idx : seq(LNumOps)) {
641 int Result = CompareValueComplexity(LI, LInst->getOperand(Idx),
642 RInst->getOperand(Idx), Depth + 1);
643 if (Result != 0)
644 return Result;
645 }
646 }
647
648 return 0;
649}
650
651// Return negative, zero, or positive, if LHS is less than, equal to, or greater
652// than RHS, respectively. A three-way result allows recursive comparisons to be
653// more efficient.
654// If the max analysis depth was reached, return std::nullopt, assuming we do
655// not know if they are equivalent for sure.
656static std::optional<int>
657CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS,
658 const SCEV *RHS, DominatorTree &DT, unsigned Depth = 0) {
659 // Fast-path: SCEVs are uniqued so we can do a quick equality check.
660 if (LHS == RHS)
661 return 0;
662
663 // Primarily, sort the SCEVs by their getSCEVType().
664 SCEVTypes LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
665 if (LType != RType)
666 return (int)LType - (int)RType;
667
669 return std::nullopt;
670
671 // Aside from the getSCEVType() ordering, the particular ordering
672 // isn't very important except that it's beneficial to be consistent,
673 // so that (a + b) and (b + a) don't end up as different expressions.
674 switch (LType) {
675 case scUnknown: {
676 const SCEVUnknown *LU = cast<SCEVUnknown>(LHS);
677 const SCEVUnknown *RU = cast<SCEVUnknown>(RHS);
678
679 int X =
680 CompareValueComplexity(LI, LU->getValue(), RU->getValue(), Depth + 1);
681 return X;
682 }
683
684 case scConstant: {
687
688 // Compare constant values.
689 const APInt &LA = LC->getAPInt();
690 const APInt &RA = RC->getAPInt();
691 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
692 if (LBitWidth != RBitWidth)
693 return (int)LBitWidth - (int)RBitWidth;
694 return LA.ult(RA) ? -1 : 1;
695 }
696
697 case scVScale: {
698 const auto *LTy = cast<IntegerType>(cast<SCEVVScale>(LHS)->getType());
699 const auto *RTy = cast<IntegerType>(cast<SCEVVScale>(RHS)->getType());
700 return LTy->getBitWidth() - RTy->getBitWidth();
701 }
702
703 case scAddRecExpr: {
706
707 // There is always a dominance between two recs that are used by one SCEV,
708 // so we can safely sort recs by loop header dominance. We require such
709 // order in getAddExpr.
710 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
711 if (LLoop != RLoop) {
712 const BasicBlock *LHead = LLoop->getHeader(), *RHead = RLoop->getHeader();
713 assert(LHead != RHead && "Two loops share the same header?");
714 if (DT.dominates(LHead, RHead))
715 return 1;
716 assert(DT.dominates(RHead, LHead) &&
717 "No dominance between recurrences used by one SCEV?");
718 return -1;
719 }
720
721 [[fallthrough]];
722 }
723
724 case scTruncate:
725 case scZeroExtend:
726 case scSignExtend:
727 case scPtrToInt:
728 case scAddExpr:
729 case scMulExpr:
730 case scUDivExpr:
731 case scSMaxExpr:
732 case scUMaxExpr:
733 case scSMinExpr:
734 case scUMinExpr:
736 ArrayRef<const SCEV *> LOps = LHS->operands();
737 ArrayRef<const SCEV *> ROps = RHS->operands();
738
739 // Lexicographically compare n-ary-like expressions.
740 unsigned LNumOps = LOps.size(), RNumOps = ROps.size();
741 if (LNumOps != RNumOps)
742 return (int)LNumOps - (int)RNumOps;
743
744 for (unsigned i = 0; i != LNumOps; ++i) {
745 auto X = CompareSCEVComplexity(LI, LOps[i], ROps[i], DT, Depth + 1);
746 if (X != 0)
747 return X;
748 }
749 return 0;
750 }
751
753 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
754 }
755 llvm_unreachable("Unknown SCEV kind!");
756}
757
758/// Given a list of SCEV objects, order them by their complexity, and group
759/// objects of the same complexity together by value. When this routine is
760/// finished, we know that any duplicates in the vector are consecutive and that
761/// complexity is monotonically increasing.
762///
763/// Note that we go take special precautions to ensure that we get deterministic
764/// results from this routine. In other words, we don't want the results of
765/// this to depend on where the addresses of various SCEV objects happened to
766/// land in memory.
768 LoopInfo *LI, DominatorTree &DT) {
769 if (Ops.size() < 2) return; // Noop
770
771 // Whether LHS has provably less complexity than RHS.
772 auto IsLessComplex = [&](const SCEV *LHS, const SCEV *RHS) {
773 auto Complexity = CompareSCEVComplexity(LI, LHS, RHS, DT);
774 return Complexity && *Complexity < 0;
775 };
776 if (Ops.size() == 2) {
777 // This is the common case, which also happens to be trivially simple.
778 // Special case it.
779 const SCEV *&LHS = Ops[0], *&RHS = Ops[1];
780 if (IsLessComplex(RHS, LHS))
781 std::swap(LHS, RHS);
782 return;
783 }
784
785 // Do the rough sort by complexity.
786 llvm::stable_sort(Ops, [&](const SCEV *LHS, const SCEV *RHS) {
787 return IsLessComplex(LHS, RHS);
788 });
789
790 // Now that we are sorted by complexity, group elements of the same
791 // complexity. Note that this is, at worst, N^2, but the vector is likely to
792 // be extremely short in practice. Note that we take this approach because we
793 // do not want to depend on the addresses of the objects we are grouping.
794 for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) {
795 const SCEV *S = Ops[i];
796 unsigned Complexity = S->getSCEVType();
797
798 // If there are any objects of the same complexity and same value as this
799 // one, group them.
800 for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
801 if (Ops[j] == S) { // Found a duplicate.
802 // Move it to immediately after i'th element.
803 std::swap(Ops[i+1], Ops[j]);
804 ++i; // no need to rescan it.
805 if (i == e-2) return; // Done!
806 }
807 }
808 }
809}
810
811/// Returns true if \p Ops contains a huge SCEV (the subtree of S contains at
812/// least HugeExprThreshold nodes).
814 return any_of(Ops, [](const SCEV *S) {
816 });
817}
818
819/// Performs a number of common optimizations on the passed \p Ops. If the
820/// whole expression reduces down to a single operand, it will be returned.
821///
822/// The following optimizations are performed:
823/// * Fold constants using the \p Fold function.
824/// * Remove identity constants satisfying \p IsIdentity.
825/// * If a constant satisfies \p IsAbsorber, return it.
826/// * Sort operands by complexity.
827template <typename FoldT, typename IsIdentityT, typename IsAbsorberT>
828static const SCEV *
831 IsIdentityT IsIdentity, IsAbsorberT IsAbsorber) {
832 const SCEVConstant *Folded = nullptr;
833 for (unsigned Idx = 0; Idx < Ops.size();) {
834 const SCEV *Op = Ops[Idx];
835 if (const auto *C = dyn_cast<SCEVConstant>(Op)) {
836 if (!Folded)
837 Folded = C;
838 else
839 Folded = cast<SCEVConstant>(
840 SE.getConstant(Fold(Folded->getAPInt(), C->getAPInt())));
841 Ops.erase(Ops.begin() + Idx);
842 continue;
843 }
844 ++Idx;
845 }
846
847 if (Ops.empty()) {
848 assert(Folded && "Must have folded value");
849 return Folded;
850 }
851
852 if (Folded && IsAbsorber(Folded->getAPInt()))
853 return Folded;
854
855 GroupByComplexity(Ops, &LI, DT);
856 if (Folded && !IsIdentity(Folded->getAPInt()))
857 Ops.insert(Ops.begin(), Folded);
858
859 return Ops.size() == 1 ? Ops[0] : nullptr;
860}
861
862//===----------------------------------------------------------------------===//
863// Simple SCEV method implementations
864//===----------------------------------------------------------------------===//
865
866/// Compute BC(It, K). The result has width W. Assume, K > 0.
867static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
868 ScalarEvolution &SE,
869 Type *ResultTy) {
870 // Handle the simplest case efficiently.
871 if (K == 1)
872 return SE.getTruncateOrZeroExtend(It, ResultTy);
873
874 // We are using the following formula for BC(It, K):
875 //
876 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
877 //
878 // Suppose, W is the bitwidth of the return value. We must be prepared for
879 // overflow. Hence, we must assure that the result of our computation is
880 // equal to the accurate one modulo 2^W. Unfortunately, division isn't
881 // safe in modular arithmetic.
882 //
883 // However, this code doesn't use exactly that formula; the formula it uses
884 // is something like the following, where T is the number of factors of 2 in
885 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
886 // exponentiation:
887 //
888 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
889 //
890 // This formula is trivially equivalent to the previous formula. However,
891 // this formula can be implemented much more efficiently. The trick is that
892 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
893 // arithmetic. To do exact division in modular arithmetic, all we have
894 // to do is multiply by the inverse. Therefore, this step can be done at
895 // width W.
896 //
897 // The next issue is how to safely do the division by 2^T. The way this
898 // is done is by doing the multiplication step at a width of at least W + T
899 // bits. This way, the bottom W+T bits of the product are accurate. Then,
900 // when we perform the division by 2^T (which is equivalent to a right shift
901 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get
902 // truncated out after the division by 2^T.
903 //
904 // In comparison to just directly using the first formula, this technique
905 // is much more efficient; using the first formula requires W * K bits,
906 // but this formula less than W + K bits. Also, the first formula requires
907 // a division step, whereas this formula only requires multiplies and shifts.
908 //
909 // It doesn't matter whether the subtraction step is done in the calculation
910 // width or the input iteration count's width; if the subtraction overflows,
911 // the result must be zero anyway. We prefer here to do it in the width of
912 // the induction variable because it helps a lot for certain cases; CodeGen
913 // isn't smart enough to ignore the overflow, which leads to much less
914 // efficient code if the width of the subtraction is wider than the native
915 // register width.
916 //
917 // (It's possible to not widen at all by pulling out factors of 2 before
918 // the multiplication; for example, K=2 can be calculated as
919 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
920 // extra arithmetic, so it's not an obvious win, and it gets
921 // much more complicated for K > 3.)
922
923 // Protection from insane SCEVs; this bound is conservative,
924 // but it probably doesn't matter.
925 if (K > 1000)
926 return SE.getCouldNotCompute();
927
928 unsigned W = SE.getTypeSizeInBits(ResultTy);
929
930 // Calculate K! / 2^T and T; we divide out the factors of two before
931 // multiplying for calculating K! / 2^T to avoid overflow.
932 // Other overflow doesn't matter because we only care about the bottom
933 // W bits of the result.
934 APInt OddFactorial(W, 1);
935 unsigned T = 1;
936 for (unsigned i = 3; i <= K; ++i) {
937 unsigned TwoFactors = countr_zero(i);
938 T += TwoFactors;
939 OddFactorial *= (i >> TwoFactors);
940 }
941
942 // We need at least W + T bits for the multiplication step
943 unsigned CalculationBits = W + T;
944
945 // Calculate 2^T, at width T+W.
946 APInt DivFactor = APInt::getOneBitSet(CalculationBits, T);
947
948 // Calculate the multiplicative inverse of K! / 2^T;
949 // this multiplication factor will perform the exact division by
950 // K! / 2^T.
951 APInt MultiplyFactor = OddFactorial.multiplicativeInverse();
952
953 // Calculate the product, at width T+W
954 IntegerType *CalculationTy = IntegerType::get(SE.getContext(),
955 CalculationBits);
956 const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy);
957 for (unsigned i = 1; i != K; ++i) {
958 const SCEV *S = SE.getMinusSCEV(It, SE.getConstant(It->getType(), i));
959 Dividend = SE.getMulExpr(Dividend,
960 SE.getTruncateOrZeroExtend(S, CalculationTy));
961 }
962
963 // Divide by 2^T
964 const SCEV *DivResult = SE.getUDivExpr(Dividend, SE.getConstant(DivFactor));
965
966 // Truncate the result, and divide by K! / 2^T.
967
968 return SE.getMulExpr(SE.getConstant(MultiplyFactor),
969 SE.getTruncateOrZeroExtend(DivResult, ResultTy));
970}
971
972/// Return the value of this chain of recurrences at the specified iteration
973/// number. We can evaluate this recurrence by multiplying each element in the
974/// chain by the binomial coefficient corresponding to it. In other words, we
975/// can evaluate {A,+,B,+,C,+,D} as:
976///
977/// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
978///
979/// where BC(It, k) stands for binomial coefficient.
981 ScalarEvolution &SE) const {
982 return evaluateAtIteration(operands(), It, SE);
983}
984
985const SCEV *
987 const SCEV *It, ScalarEvolution &SE) {
988 assert(Operands.size() > 0);
989 const SCEV *Result = Operands[0];
990 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
991 // The computation is correct in the face of overflow provided that the
992 // multiplication is performed _after_ the evaluation of the binomial
993 // coefficient.
994 const SCEV *Coeff = BinomialCoefficient(It, i, SE, Result->getType());
995 if (isa<SCEVCouldNotCompute>(Coeff))
996 return Coeff;
997
998 Result = SE.getAddExpr(Result, SE.getMulExpr(Operands[i], Coeff));
999 }
1000 return Result;
1001}
1002
1003//===----------------------------------------------------------------------===//
1004// SCEV Expression folder implementations
1005//===----------------------------------------------------------------------===//
1006
1008 unsigned Depth) {
1009 assert(Depth <= 1 &&
1010 "getLosslessPtrToIntExpr() should self-recurse at most once.");
1011
1012 // We could be called with an integer-typed operands during SCEV rewrites.
1013 // Since the operand is an integer already, just perform zext/trunc/self cast.
1014 if (!Op->getType()->isPointerTy())
1015 return Op;
1016
1017 // What would be an ID for such a SCEV cast expression?
1019 ID.AddInteger(scPtrToInt);
1020 ID.AddPointer(Op);
1021
1022 void *IP = nullptr;
1023
1024 // Is there already an expression for such a cast?
1025 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
1026 return S;
1027
1028 // It isn't legal for optimizations to construct new ptrtoint expressions
1029 // for non-integral pointers.
1030 if (getDataLayout().isNonIntegralPointerType(Op->getType()))
1031 return getCouldNotCompute();
1032
1033 Type *IntPtrTy = getDataLayout().getIntPtrType(Op->getType());
1034
1035 // We can only trivially model ptrtoint if SCEV's effective (integer) type
1036 // is sufficiently wide to represent all possible pointer values.
1037 // We could theoretically teach SCEV to truncate wider pointers, but
1038 // that isn't implemented for now.
1040 getDataLayout().getTypeSizeInBits(IntPtrTy))
1041 return getCouldNotCompute();
1042
1043 // If not, is this expression something we can't reduce any further?
1044 if (auto *U = dyn_cast<SCEVUnknown>(Op)) {
1045 // Perform some basic constant folding. If the operand of the ptr2int cast
1046 // is a null pointer, don't create a ptr2int SCEV expression (that will be
1047 // left as-is), but produce a zero constant.
1048 // NOTE: We could handle a more general case, but lack motivational cases.
1049 if (isa<ConstantPointerNull>(U->getValue()))
1050 return getZero(IntPtrTy);
1051
1052 // Create an explicit cast node.
1053 // We can reuse the existing insert position since if we get here,
1054 // we won't have made any changes which would invalidate it.
1055 SCEV *S = new (SCEVAllocator)
1056 SCEVPtrToIntExpr(ID.Intern(SCEVAllocator), Op, IntPtrTy);
1057 UniqueSCEVs.InsertNode(S, IP);
1058 registerUser(S, Op);
1059 return S;
1060 }
1061
1062 assert(Depth == 0 && "getLosslessPtrToIntExpr() should not self-recurse for "
1063 "non-SCEVUnknown's.");
1064
1065 // Otherwise, we've got some expression that is more complex than just a
1066 // single SCEVUnknown. But we don't want to have a SCEVPtrToIntExpr of an
1067 // arbitrary expression, we want to have SCEVPtrToIntExpr of an SCEVUnknown
1068 // only, and the expressions must otherwise be integer-typed.
1069 // So sink the cast down to the SCEVUnknown's.
1070
1071 /// The SCEVPtrToIntSinkingRewriter takes a scalar evolution expression,
1072 /// which computes a pointer-typed value, and rewrites the whole expression
1073 /// tree so that *all* the computations are done on integers, and the only
1074 /// pointer-typed operands in the expression are SCEVUnknown.
1075 class SCEVPtrToIntSinkingRewriter
1076 : public SCEVRewriteVisitor<SCEVPtrToIntSinkingRewriter> {
1078
1079 public:
1080 SCEVPtrToIntSinkingRewriter(ScalarEvolution &SE) : SCEVRewriteVisitor(SE) {}
1081
1082 static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE) {
1083 SCEVPtrToIntSinkingRewriter Rewriter(SE);
1084 return Rewriter.visit(Scev);
1085 }
1086
1087 const SCEV *visit(const SCEV *S) {
1088 Type *STy = S->getType();
1089 // If the expression is not pointer-typed, just keep it as-is.
1090 if (!STy->isPointerTy())
1091 return S;
1092 // Else, recursively sink the cast down into it.
1093 return Base::visit(S);
1094 }
1095
1096 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
1098 bool Changed = false;
1099 for (const auto *Op : Expr->operands()) {
1100 Operands.push_back(visit(Op));
1101 Changed |= Op != Operands.back();
1102 }
1103 return !Changed ? Expr : SE.getAddExpr(Operands, Expr->getNoWrapFlags());
1104 }
1105
1106 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
1108 bool Changed = false;
1109 for (const auto *Op : Expr->operands()) {
1110 Operands.push_back(visit(Op));
1111 Changed |= Op != Operands.back();
1112 }
1113 return !Changed ? Expr : SE.getMulExpr(Operands, Expr->getNoWrapFlags());
1114 }
1115
1116 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
1117 assert(Expr->getType()->isPointerTy() &&
1118 "Should only reach pointer-typed SCEVUnknown's.");
1119 return SE.getLosslessPtrToIntExpr(Expr, /*Depth=*/1);
1120 }
1121 };
1122
1123 // And actually perform the cast sinking.
1124 const SCEV *IntOp = SCEVPtrToIntSinkingRewriter::rewrite(Op, *this);
1125 assert(IntOp->getType()->isIntegerTy() &&
1126 "We must have succeeded in sinking the cast, "
1127 "and ending up with an integer-typed expression!");
1128 return IntOp;
1129}
1130
1132 assert(Ty->isIntegerTy() && "Target type must be an integer type!");
1133
1134 const SCEV *IntOp = getLosslessPtrToIntExpr(Op);
1135 if (isa<SCEVCouldNotCompute>(IntOp))
1136 return IntOp;
1137
1138 return getTruncateOrZeroExtend(IntOp, Ty);
1139}
1140
1142 unsigned Depth) {
1143 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
1144 "This is not a truncating conversion!");
1145 assert(isSCEVable(Ty) &&
1146 "This is not a conversion to a SCEVable type!");
1147 assert(!Op->getType()->isPointerTy() && "Can't truncate pointer!");
1148 Ty = getEffectiveSCEVType(Ty);
1149
1151 ID.AddInteger(scTruncate);
1152 ID.AddPointer(Op);
1153 ID.AddPointer(Ty);
1154 void *IP = nullptr;
1155 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
1156
1157 // Fold if the operand is constant.
1158 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
1159 return getConstant(
1160 cast<ConstantInt>(ConstantExpr::getTrunc(SC->getValue(), Ty)));
1161
1162 // trunc(trunc(x)) --> trunc(x)
1164 return getTruncateExpr(ST->getOperand(), Ty, Depth + 1);
1165
1166 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
1168 return getTruncateOrSignExtend(SS->getOperand(), Ty, Depth + 1);
1169
1170 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
1172 return getTruncateOrZeroExtend(SZ->getOperand(), Ty, Depth + 1);
1173
1174 if (Depth > MaxCastDepth) {
1175 SCEV *S =
1176 new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), Op, Ty);
1177 UniqueSCEVs.InsertNode(S, IP);
1178 registerUser(S, Op);
1179 return S;
1180 }
1181
1182 // trunc(x1 + ... + xN) --> trunc(x1) + ... + trunc(xN) and
1183 // trunc(x1 * ... * xN) --> trunc(x1) * ... * trunc(xN),
1184 // if after transforming we have at most one truncate, not counting truncates
1185 // that replace other casts.
1187 auto *CommOp = cast<SCEVCommutativeExpr>(Op);
1189 unsigned numTruncs = 0;
1190 for (unsigned i = 0, e = CommOp->getNumOperands(); i != e && numTruncs < 2;
1191 ++i) {
1192 const SCEV *S = getTruncateExpr(CommOp->getOperand(i), Ty, Depth + 1);
1193 if (!isa<SCEVIntegralCastExpr>(CommOp->getOperand(i)) &&
1195 numTruncs++;
1196 Operands.push_back(S);
1197 }
1198 if (numTruncs < 2) {
1199 if (isa<SCEVAddExpr>(Op))
1200 return getAddExpr(Operands);
1201 if (isa<SCEVMulExpr>(Op))
1202 return getMulExpr(Operands);
1203 llvm_unreachable("Unexpected SCEV type for Op.");
1204 }
1205 // Although we checked in the beginning that ID is not in the cache, it is
1206 // possible that during recursion and different modification ID was inserted
1207 // into the cache. So if we find it, just return it.
1208 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
1209 return S;
1210 }
1211
1212 // If the input value is a chrec scev, truncate the chrec's operands.
1213 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) {
1215 for (const SCEV *Op : AddRec->operands())
1216 Operands.push_back(getTruncateExpr(Op, Ty, Depth + 1));
1217 return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagAnyWrap);
1218 }
1219
1220 // Return zero if truncating to known zeros.
1221 uint32_t MinTrailingZeros = getMinTrailingZeros(Op);
1222 if (MinTrailingZeros >= getTypeSizeInBits(Ty))
1223 return getZero(Ty);
1224
1225 // The cast wasn't folded; create an explicit cast node. We can reuse
1226 // the existing insert position since if we get here, we won't have
1227 // made any changes which would invalidate it.
1228 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator),
1229 Op, Ty);
1230 UniqueSCEVs.InsertNode(S, IP);
1231 registerUser(S, Op);
1232 return S;
1233}
1234
1235// Get the limit of a recurrence such that incrementing by Step cannot cause
1236// signed overflow as long as the value of the recurrence within the
1237// loop does not exceed this limit before incrementing.
1238static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step,
1239 ICmpInst::Predicate *Pred,
1240 ScalarEvolution *SE) {
1241 unsigned BitWidth = SE->getTypeSizeInBits(Step->getType());
1242 if (SE->isKnownPositive(Step)) {
1243 *Pred = ICmpInst::ICMP_SLT;
1245 SE->getSignedRangeMax(Step));
1246 }
1247 if (SE->isKnownNegative(Step)) {
1248 *Pred = ICmpInst::ICMP_SGT;
1250 SE->getSignedRangeMin(Step));
1251 }
1252 return nullptr;
1253}
1254
1255// Get the limit of a recurrence such that incrementing by Step cannot cause
1256// unsigned overflow as long as the value of the recurrence within the loop does
1257// not exceed this limit before incrementing.
1259 ICmpInst::Predicate *Pred,
1260 ScalarEvolution *SE) {
1261 unsigned BitWidth = SE->getTypeSizeInBits(Step->getType());
1262 *Pred = ICmpInst::ICMP_ULT;
1263
1265 SE->getUnsignedRangeMax(Step));
1266}
1267
1268namespace {
1269
1270struct ExtendOpTraitsBase {
1271 typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(const SCEV *, Type *,
1272 unsigned);
1273};
1274
1275// Used to make code generic over signed and unsigned overflow.
1276template <typename ExtendOp> struct ExtendOpTraits {
1277 // Members present:
1278 //
1279 // static const SCEV::NoWrapFlags WrapType;
1280 //
1281 // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr;
1282 //
1283 // static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1284 // ICmpInst::Predicate *Pred,
1285 // ScalarEvolution *SE);
1286};
1287
1288template <>
1289struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase {
1290 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNSW;
1291
1292 static const GetExtendExprTy GetExtendExpr;
1293
1294 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1295 ICmpInst::Predicate *Pred,
1296 ScalarEvolution *SE) {
1297 return getSignedOverflowLimitForStep(Step, Pred, SE);
1298 }
1299};
1300
1301const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1303
1304template <>
1305struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase {
1306 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNUW;
1307
1308 static const GetExtendExprTy GetExtendExpr;
1309
1310 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1311 ICmpInst::Predicate *Pred,
1312 ScalarEvolution *SE) {
1313 return getUnsignedOverflowLimitForStep(Step, Pred, SE);
1314 }
1315};
1316
1317const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1319
1320} // end anonymous namespace
1321
1322// The recurrence AR has been shown to have no signed/unsigned wrap or something
1323// close to it. Typically, if we can prove NSW/NUW for AR, then we can just as
1324// easily prove NSW/NUW for its preincrement or postincrement sibling. This
1325// allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step +
1326// Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the
1327// expression "Step + sext/zext(PreIncAR)" is congruent with
1328// "sext/zext(PostIncAR)"
1329template <typename ExtendOpTy>
1330static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR, Type *Ty,
1331 ScalarEvolution *SE, unsigned Depth) {
1332 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1333 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1334
1335 const Loop *L = AR->getLoop();
1336 const SCEV *Start = AR->getStart();
1337 const SCEV *Step = AR->getStepRecurrence(*SE);
1338
1339 // Check for a simple looking step prior to loop entry.
1340 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Start);
1341 if (!SA)
1342 return nullptr;
1343
1344 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
1345 // subtraction is expensive. For this purpose, perform a quick and dirty
1346 // difference, by checking for Step in the operand list. Note, that
1347 // SA might have repeated ops, like %a + %a + ..., so only remove one.
1349 for (auto It = DiffOps.begin(); It != DiffOps.end(); ++It)
1350 if (*It == Step) {
1351 DiffOps.erase(It);
1352 break;
1353 }
1354
1355 if (DiffOps.size() == SA->getNumOperands())
1356 return nullptr;
1357
1358 // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` +
1359 // `Step`:
1360
1361 // 1. NSW/NUW flags on the step increment.
1362 auto PreStartFlags =
1364 const SCEV *PreStart = SE->getAddExpr(DiffOps, PreStartFlags);
1366 SE->getAddRecExpr(PreStart, Step, L, SCEV::FlagAnyWrap));
1367
1368 // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies
1369 // "S+X does not sign/unsign-overflow".
1370 //
1371
1372 const SCEV *BECount = SE->getBackedgeTakenCount(L);
1373 if (PreAR && PreAR->getNoWrapFlags(WrapType) &&
1374 !isa<SCEVCouldNotCompute>(BECount) && SE->isKnownPositive(BECount))
1375 return PreStart;
1376
1377 // 2. Direct overflow check on the step operation's expression.
1378 unsigned BitWidth = SE->getTypeSizeInBits(AR->getType());
1379 Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2);
1380 const SCEV *OperandExtendedStart =
1381 SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy, Depth),
1382 (SE->*GetExtendExpr)(Step, WideTy, Depth));
1383 if ((SE->*GetExtendExpr)(Start, WideTy, Depth) == OperandExtendedStart) {
1384 if (PreAR && AR->getNoWrapFlags(WrapType)) {
1385 // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW
1386 // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then
1387 // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`. Cache this fact.
1388 SE->setNoWrapFlags(const_cast<SCEVAddRecExpr *>(PreAR), WrapType);
1389 }
1390 return PreStart;
1391 }
1392
1393 // 3. Loop precondition.
1395 const SCEV *OverflowLimit =
1396 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
1397
1398 if (OverflowLimit &&
1399 SE->isLoopEntryGuardedByCond(L, Pred, PreStart, OverflowLimit))
1400 return PreStart;
1401
1402 return nullptr;
1403}
1404
1405// Get the normalized zero or sign extended expression for this AddRec's Start.
1406template <typename ExtendOpTy>
1407static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty,
1408 ScalarEvolution *SE,
1409 unsigned Depth) {
1410 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1411
1412 const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, Ty, SE, Depth);
1413 if (!PreStart)
1414 return (SE->*GetExtendExpr)(AR->getStart(), Ty, Depth);
1415
1416 return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(*SE), Ty,
1417 Depth),
1418 (SE->*GetExtendExpr)(PreStart, Ty, Depth));
1419}
1420
1421// Try to prove away overflow by looking at "nearby" add recurrences. A
1422// motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it
1423// does not itself wrap then we can conclude that `{1,+,4}` is `nuw`.
1424//
1425// Formally:
1426//
1427// {S,+,X} == {S-T,+,X} + T
1428// => Ext({S,+,X}) == Ext({S-T,+,X} + T)
1429//
1430// If ({S-T,+,X} + T) does not overflow ... (1)
1431//
1432// RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T)
1433//
1434// If {S-T,+,X} does not overflow ... (2)
1435//
1436// RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T)
1437// == {Ext(S-T)+Ext(T),+,Ext(X)}
1438//
1439// If (S-T)+T does not overflow ... (3)
1440//
1441// RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)}
1442// == {Ext(S),+,Ext(X)} == LHS
1443//
1444// Thus, if (1), (2) and (3) are true for some T, then
1445// Ext({S,+,X}) == {Ext(S),+,Ext(X)}
1446//
1447// (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T)
1448// does not overflow" restricted to the 0th iteration. Therefore we only need
1449// to check for (1) and (2).
1450//
1451// In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T
1452// is `Delta` (defined below).
1453template <typename ExtendOpTy>
1454bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
1455 const SCEV *Step,
1456 const Loop *L) {
1457 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1458
1459 // We restrict `Start` to a constant to prevent SCEV from spending too much
1460 // time here. It is correct (but more expensive) to continue with a
1461 // non-constant `Start` and do a general SCEV subtraction to compute
1462 // `PreStart` below.
1463 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start);
1464 if (!StartC)
1465 return false;
1466
1467 APInt StartAI = StartC->getAPInt();
1468
1469 for (unsigned Delta : {-2, -1, 1, 2}) {
1470 const SCEV *PreStart = getConstant(StartAI - Delta);
1471
1472 FoldingSetNodeID ID;
1473 ID.AddInteger(scAddRecExpr);
1474 ID.AddPointer(PreStart);
1475 ID.AddPointer(Step);
1476 ID.AddPointer(L);
1477 void *IP = nullptr;
1478 const auto *PreAR =
1479 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
1480
1481 // Give up if we don't already have the add recurrence we need because
1482 // actually constructing an add recurrence is relatively expensive.
1483 if (PreAR && PreAR->getNoWrapFlags(WrapType)) { // proves (2)
1484 const SCEV *DeltaS = getConstant(StartC->getType(), Delta);
1486 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
1487 DeltaS, &Pred, this);
1488 if (Limit && isKnownPredicate(Pred, PreAR, Limit)) // proves (1)
1489 return true;
1490 }
1491 }
1492
1493 return false;
1494}
1495
1496// Finds an integer D for an expression (C + x + y + ...) such that the top
1497// level addition in (D + (C - D + x + y + ...)) would not wrap (signed or
1498// unsigned) and the number of trailing zeros of (C - D + x + y + ...) is
1499// maximized, where C is the \p ConstantTerm, x, y, ... are arbitrary SCEVs, and
1500// the (C + x + y + ...) expression is \p WholeAddExpr.
1502 const SCEVConstant *ConstantTerm,
1503 const SCEVAddExpr *WholeAddExpr) {
1504 const APInt &C = ConstantTerm->getAPInt();
1505 const unsigned BitWidth = C.getBitWidth();
1506 // Find number of trailing zeros of (x + y + ...) w/o the C first:
1507 uint32_t TZ = BitWidth;
1508 for (unsigned I = 1, E = WholeAddExpr->getNumOperands(); I < E && TZ; ++I)
1509 TZ = std::min(TZ, SE.getMinTrailingZeros(WholeAddExpr->getOperand(I)));
1510 if (TZ) {
1511 // Set D to be as many least significant bits of C as possible while still
1512 // guaranteeing that adding D to (C - D + x + y + ...) won't cause a wrap:
1513 return TZ < BitWidth ? C.trunc(TZ).zext(BitWidth) : C;
1514 }
1515 return APInt(BitWidth, 0);
1516}
1517
1518// Finds an integer D for an affine AddRec expression {C,+,x} such that the top
1519// level addition in (D + {C-D,+,x}) would not wrap (signed or unsigned) and the
1520// number of trailing zeros of (C - D + x * n) is maximized, where C is the \p
1521// ConstantStart, x is an arbitrary \p Step, and n is the loop trip count.
1523 const APInt &ConstantStart,
1524 const SCEV *Step) {
1525 const unsigned BitWidth = ConstantStart.getBitWidth();
1526 const uint32_t TZ = SE.getMinTrailingZeros(Step);
1527 if (TZ)
1528 return TZ < BitWidth ? ConstantStart.trunc(TZ).zext(BitWidth)
1529 : ConstantStart;
1530 return APInt(BitWidth, 0);
1531}
1532
1534 const ScalarEvolution::FoldID &ID, const SCEV *S,
1537 &FoldCacheUser) {
1538 auto I = FoldCache.insert({ID, S});
1539 if (!I.second) {
1540 // Remove FoldCacheUser entry for ID when replacing an existing FoldCache
1541 // entry.
1542 auto &UserIDs = FoldCacheUser[I.first->second];
1543 assert(count(UserIDs, ID) == 1 && "unexpected duplicates in UserIDs");
1544 for (unsigned I = 0; I != UserIDs.size(); ++I)
1545 if (UserIDs[I] == ID) {
1546 std::swap(UserIDs[I], UserIDs.back());
1547 break;
1548 }
1549 UserIDs.pop_back();
1550 I.first->second = S;
1551 }
1552 FoldCacheUser[S].push_back(ID);
1553}
1554
1555const SCEV *
1557 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1558 "This is not an extending conversion!");
1559 assert(isSCEVable(Ty) &&
1560 "This is not a conversion to a SCEVable type!");
1561 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1562 Ty = getEffectiveSCEVType(Ty);
1563
1564 FoldID ID(scZeroExtend, Op, Ty);
1565 if (const SCEV *S = FoldCache.lookup(ID))
1566 return S;
1567
1568 const SCEV *S = getZeroExtendExprImpl(Op, Ty, Depth);
1570 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1571 return S;
1572}
1573
1575 unsigned Depth) {
1576 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1577 "This is not an extending conversion!");
1578 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1579 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1580
1581 // Fold if the operand is constant.
1582 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
1583 return getConstant(SC->getAPInt().zext(getTypeSizeInBits(Ty)));
1584
1585 // zext(zext(x)) --> zext(x)
1587 return getZeroExtendExpr(SZ->getOperand(), Ty, Depth + 1);
1588
1589 // Before doing any expensive analysis, check to see if we've already
1590 // computed a SCEV for this Op and Ty.
1592 ID.AddInteger(scZeroExtend);
1593 ID.AddPointer(Op);
1594 ID.AddPointer(Ty);
1595 void *IP = nullptr;
1596 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
1597 if (Depth > MaxCastDepth) {
1598 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
1599 Op, Ty);
1600 UniqueSCEVs.InsertNode(S, IP);
1601 registerUser(S, Op);
1602 return S;
1603 }
1604
1605 // zext(trunc(x)) --> zext(x) or x or trunc(x)
1607 // It's possible the bits taken off by the truncate were all zero bits. If
1608 // so, we should be able to simplify this further.
1609 const SCEV *X = ST->getOperand();
1611 unsigned TruncBits = getTypeSizeInBits(ST->getType());
1612 unsigned NewBits = getTypeSizeInBits(Ty);
1613 if (CR.truncate(TruncBits).zeroExtend(NewBits).contains(
1614 CR.zextOrTrunc(NewBits)))
1615 return getTruncateOrZeroExtend(X, Ty, Depth);
1616 }
1617
1618 // If the input value is a chrec scev, and we can prove that the value
1619 // did not overflow the old, smaller, value, we can zero extend all of the
1620 // operands (often constants). This allows analysis of something like
1621 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
1623 if (AR->isAffine()) {
1624 const SCEV *Start = AR->getStart();
1625 const SCEV *Step = AR->getStepRecurrence(*this);
1626 unsigned BitWidth = getTypeSizeInBits(AR->getType());
1627 const Loop *L = AR->getLoop();
1628
1629 // If we have special knowledge that this addrec won't overflow,
1630 // we don't need to do any further analysis.
1631 if (AR->hasNoUnsignedWrap()) {
1632 Start =
1634 Step = getZeroExtendExpr(Step, Ty, Depth + 1);
1635 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1636 }
1637
1638 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1639 // Note that this serves two purposes: It filters out loops that are
1640 // simply not analyzable, and it covers the case where this code is
1641 // being called from within backedge-taken count analysis, such that
1642 // attempting to ask for the backedge-taken count would likely result
1643 // in infinite recursion. In the later case, the analysis code will
1644 // cope with a conservative value, and it will take care to purge
1645 // that value once it has finished.
1646 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
1647 if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
1648 // Manually compute the final value for AR, checking for overflow.
1649
1650 // Check whether the backedge-taken count can be losslessly casted to
1651 // the addrec's type. The count is always unsigned.
1652 const SCEV *CastedMaxBECount =
1653 getTruncateOrZeroExtend(MaxBECount, Start->getType(), Depth);
1654 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
1655 CastedMaxBECount, MaxBECount->getType(), Depth);
1656 if (MaxBECount == RecastedMaxBECount) {
1657 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
1658 // Check whether Start+Step*MaxBECount has no unsigned overflow.
1659 const SCEV *ZMul = getMulExpr(CastedMaxBECount, Step,
1661 const SCEV *ZAdd = getZeroExtendExpr(getAddExpr(Start, ZMul,
1663 Depth + 1),
1664 WideTy, Depth + 1);
1665 const SCEV *WideStart = getZeroExtendExpr(Start, WideTy, Depth + 1);
1666 const SCEV *WideMaxBECount =
1667 getZeroExtendExpr(CastedMaxBECount, WideTy, Depth + 1);
1668 const SCEV *OperandExtendedAdd =
1669 getAddExpr(WideStart,
1670 getMulExpr(WideMaxBECount,
1671 getZeroExtendExpr(Step, WideTy, Depth + 1),
1674 if (ZAdd == OperandExtendedAdd) {
1675 // Cache knowledge of AR NUW, which is propagated to this AddRec.
1676 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNUW);
1677 // Return the expression with the addrec on the outside.
1678 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this,
1679 Depth + 1);
1680 Step = getZeroExtendExpr(Step, Ty, Depth + 1);
1681 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1682 }
1683 // Similar to above, only this time treat the step value as signed.
1684 // This covers loops that count down.
1685 OperandExtendedAdd =
1686 getAddExpr(WideStart,
1687 getMulExpr(WideMaxBECount,
1688 getSignExtendExpr(Step, WideTy, Depth + 1),
1691 if (ZAdd == OperandExtendedAdd) {
1692 // Cache knowledge of AR NW, which is propagated to this AddRec.
1693 // Negative step causes unsigned wrap, but it still can't self-wrap.
1694 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW);
1695 // Return the expression with the addrec on the outside.
1696 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this,
1697 Depth + 1);
1698 Step = getSignExtendExpr(Step, Ty, Depth + 1);
1699 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1700 }
1701 }
1702 }
1703
1704 // Normally, in the cases we can prove no-overflow via a
1705 // backedge guarding condition, we can also compute a backedge
1706 // taken count for the loop. The exceptions are assumptions and
1707 // guards present in the loop -- SCEV is not great at exploiting
1708 // these to compute max backedge taken counts, but can still use
1709 // these to prove lack of overflow. Use this fact to avoid
1710 // doing extra work that may not pay off.
1711 if (!isa<SCEVCouldNotCompute>(MaxBECount) || HasGuards ||
1712 !AC.assumptions().empty()) {
1713
1714 auto NewFlags = proveNoUnsignedWrapViaInduction(AR);
1715 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), NewFlags);
1716 if (AR->hasNoUnsignedWrap()) {
1717 // Same as nuw case above - duplicated here to avoid a compile time
1718 // issue. It's not clear that the order of checks does matter, but
1719 // it's one of two issue possible causes for a change which was
1720 // reverted. Be conservative for the moment.
1721 Start =
1723 Step = getZeroExtendExpr(Step, Ty, Depth + 1);
1724 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1725 }
1726
1727 // For a negative step, we can extend the operands iff doing so only
1728 // traverses values in the range zext([0,UINT_MAX]).
1729 if (isKnownNegative(Step)) {
1731 getSignedRangeMin(Step));
1734 // Cache knowledge of AR NW, which is propagated to this
1735 // AddRec. Negative step causes unsigned wrap, but it
1736 // still can't self-wrap.
1737 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW);
1738 // Return the expression with the addrec on the outside.
1739 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this,
1740 Depth + 1);
1741 Step = getSignExtendExpr(Step, Ty, Depth + 1);
1742 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1743 }
1744 }
1745 }
1746
1747 // zext({C,+,Step}) --> (zext(D) + zext({C-D,+,Step}))<nuw><nsw>
1748 // if D + (C - D + Step * n) could be proven to not unsigned wrap
1749 // where D maximizes the number of trailing zeros of (C - D + Step * n)
1750 if (const auto *SC = dyn_cast<SCEVConstant>(Start)) {
1751 const APInt &C = SC->getAPInt();
1752 const APInt &D = extractConstantWithoutWrapping(*this, C, Step);
1753 if (D != 0) {
1754 const SCEV *SZExtD = getZeroExtendExpr(getConstant(D), Ty, Depth);
1755 const SCEV *SResidual =
1756 getAddRecExpr(getConstant(C - D), Step, L, AR->getNoWrapFlags());
1757 const SCEV *SZExtR = getZeroExtendExpr(SResidual, Ty, Depth + 1);
1758 return getAddExpr(SZExtD, SZExtR,
1760 Depth + 1);
1761 }
1762 }
1763
1764 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
1765 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNUW);
1766 Start =
1768 Step = getZeroExtendExpr(Step, Ty, Depth + 1);
1769 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
1770 }
1771 }
1772
1773 // zext(A % B) --> zext(A) % zext(B)
1774 {
1775 const SCEV *LHS;
1776 const SCEV *RHS;
1777 if (match(Op, m_scev_URem(m_SCEV(LHS), m_SCEV(RHS), *this)))
1778 return getURemExpr(getZeroExtendExpr(LHS, Ty, Depth + 1),
1779 getZeroExtendExpr(RHS, Ty, Depth + 1));
1780 }
1781
1782 // zext(A / B) --> zext(A) / zext(B).
1783 if (auto *Div = dyn_cast<SCEVUDivExpr>(Op))
1784 return getUDivExpr(getZeroExtendExpr(Div->getLHS(), Ty, Depth + 1),
1785 getZeroExtendExpr(Div->getRHS(), Ty, Depth + 1));
1786
1787 if (auto *SA = dyn_cast<SCEVAddExpr>(Op)) {
1788 // zext((A + B + ...)<nuw>) --> (zext(A) + zext(B) + ...)<nuw>
1789 if (SA->hasNoUnsignedWrap()) {
1790 // If the addition does not unsign overflow then we can, by definition,
1791 // commute the zero extension with the addition operation.
1793 for (const auto *Op : SA->operands())
1794 Ops.push_back(getZeroExtendExpr(Op, Ty, Depth + 1));
1795 return getAddExpr(Ops, SCEV::FlagNUW, Depth + 1);
1796 }
1797
1798 // zext(C + x + y + ...) --> (zext(D) + zext((C - D) + x + y + ...))
1799 // if D + (C - D + x + y + ...) could be proven to not unsigned wrap
1800 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1801 //
1802 // Often address arithmetics contain expressions like
1803 // (zext (add (shl X, C1), C2)), for instance, (zext (5 + (4 * X))).
1804 // This transformation is useful while proving that such expressions are
1805 // equal or differ by a small constant amount, see LoadStoreVectorizer pass.
1806 if (const auto *SC = dyn_cast<SCEVConstant>(SA->getOperand(0))) {
1807 const APInt &D = extractConstantWithoutWrapping(*this, SC, SA);
1808 if (D != 0) {
1809 const SCEV *SZExtD = getZeroExtendExpr(getConstant(D), Ty, Depth);
1810 const SCEV *SResidual =
1812 const SCEV *SZExtR = getZeroExtendExpr(SResidual, Ty, Depth + 1);
1813 return getAddExpr(SZExtD, SZExtR,
1815 Depth + 1);
1816 }
1817 }
1818 }
1819
1820 if (auto *SM = dyn_cast<SCEVMulExpr>(Op)) {
1821 // zext((A * B * ...)<nuw>) --> (zext(A) * zext(B) * ...)<nuw>
1822 if (SM->hasNoUnsignedWrap()) {
1823 // If the multiply does not unsign overflow then we can, by definition,
1824 // commute the zero extension with the multiply operation.
1826 for (const auto *Op : SM->operands())
1827 Ops.push_back(getZeroExtendExpr(Op, Ty, Depth + 1));
1828 return getMulExpr(Ops, SCEV::FlagNUW, Depth + 1);
1829 }
1830
1831 // zext(2^K * (trunc X to iN)) to iM ->
1832 // 2^K * (zext(trunc X to i{N-K}) to iM)<nuw>
1833 //
1834 // Proof:
1835 //
1836 // zext(2^K * (trunc X to iN)) to iM
1837 // = zext((trunc X to iN) << K) to iM
1838 // = zext((trunc X to i{N-K}) << K)<nuw> to iM
1839 // (because shl removes the top K bits)
1840 // = zext((2^K * (trunc X to i{N-K}))<nuw>) to iM
1841 // = (2^K * (zext(trunc X to i{N-K}) to iM))<nuw>.
1842 //
1843 const APInt *C;
1844 const SCEV *TruncRHS;
1845 if (match(SM,
1846 m_scev_Mul(m_scev_APInt(C), m_scev_Trunc(m_SCEV(TruncRHS)))) &&
1847 C->isPowerOf2()) {
1848 int NewTruncBits =
1849 getTypeSizeInBits(SM->getOperand(1)->getType()) - C->logBase2();
1850 Type *NewTruncTy = IntegerType::get(getContext(), NewTruncBits);
1851 return getMulExpr(
1852 getZeroExtendExpr(SM->getOperand(0), Ty),
1853 getZeroExtendExpr(getTruncateExpr(TruncRHS, NewTruncTy), Ty),
1854 SCEV::FlagNUW, Depth + 1);
1855 }
1856 }
1857
1858 // zext(umin(x, y)) -> umin(zext(x), zext(y))
1859 // zext(umax(x, y)) -> umax(zext(x), zext(y))
1863 for (auto *Operand : MinMax->operands())
1864 Operands.push_back(getZeroExtendExpr(Operand, Ty));
1866 return getUMinExpr(Operands);
1867 return getUMaxExpr(Operands);
1868 }
1869
1870 // zext(umin_seq(x, y)) -> umin_seq(zext(x), zext(y))
1872 assert(isa<SCEVSequentialUMinExpr>(MinMax) && "Not supported!");
1874 for (auto *Operand : MinMax->operands())
1875 Operands.push_back(getZeroExtendExpr(Operand, Ty));
1876 return getUMinExpr(Operands, /*Sequential*/ true);
1877 }
1878
1879 // The cast wasn't folded; create an explicit cast node.
1880 // Recompute the insert position, as it may have been invalidated.
1881 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
1882 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
1883 Op, Ty);
1884 UniqueSCEVs.InsertNode(S, IP);
1885 registerUser(S, Op);
1886 return S;
1887}
1888
1889const SCEV *
1891 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1892 "This is not an extending conversion!");
1893 assert(isSCEVable(Ty) &&
1894 "This is not a conversion to a SCEVable type!");
1895 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1896 Ty = getEffectiveSCEVType(Ty);
1897
1898 FoldID ID(scSignExtend, Op, Ty);
1899 if (const SCEV *S = FoldCache.lookup(ID))
1900 return S;
1901
1902 const SCEV *S = getSignExtendExprImpl(Op, Ty, Depth);
1904 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1905 return S;
1906}
1907
1909 unsigned Depth) {
1910 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1911 "This is not an extending conversion!");
1912 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1913 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1914 Ty = getEffectiveSCEVType(Ty);
1915
1916 // Fold if the operand is constant.
1917 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
1918 return getConstant(SC->getAPInt().sext(getTypeSizeInBits(Ty)));
1919
1920 // sext(sext(x)) --> sext(x)
1922 return getSignExtendExpr(SS->getOperand(), Ty, Depth + 1);
1923
1924 // sext(zext(x)) --> zext(x)
1926 return getZeroExtendExpr(SZ->getOperand(), Ty, Depth + 1);
1927
1928 // Before doing any expensive analysis, check to see if we've already
1929 // computed a SCEV for this Op and Ty.
1931 ID.AddInteger(scSignExtend);
1932 ID.AddPointer(Op);
1933 ID.AddPointer(Ty);
1934 void *IP = nullptr;
1935 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
1936 // Limit recursion depth.
1937 if (Depth > MaxCastDepth) {
1938 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
1939 Op, Ty);
1940 UniqueSCEVs.InsertNode(S, IP);
1941 registerUser(S, Op);
1942 return S;
1943 }
1944
1945 // sext(trunc(x)) --> sext(x) or x or trunc(x)
1947 // It's possible the bits taken off by the truncate were all sign bits. If
1948 // so, we should be able to simplify this further.
1949 const SCEV *X = ST->getOperand();
1951 unsigned TruncBits = getTypeSizeInBits(ST->getType());
1952 unsigned NewBits = getTypeSizeInBits(Ty);
1953 if (CR.truncate(TruncBits).signExtend(NewBits).contains(
1954 CR.sextOrTrunc(NewBits)))
1955 return getTruncateOrSignExtend(X, Ty, Depth);
1956 }
1957
1958 if (auto *SA = dyn_cast<SCEVAddExpr>(Op)) {
1959 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
1960 if (SA->hasNoSignedWrap()) {
1961 // If the addition does not sign overflow then we can, by definition,
1962 // commute the sign extension with the addition operation.
1964 for (const auto *Op : SA->operands())
1965 Ops.push_back(getSignExtendExpr(Op, Ty, Depth + 1));
1966 return getAddExpr(Ops, SCEV::FlagNSW, Depth + 1);
1967 }
1968
1969 // sext(C + x + y + ...) --> (sext(D) + sext((C - D) + x + y + ...))
1970 // if D + (C - D + x + y + ...) could be proven to not signed wrap
1971 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1972 //
1973 // For instance, this will bring two seemingly different expressions:
1974 // 1 + sext(5 + 20 * %x + 24 * %y) and
1975 // sext(6 + 20 * %x + 24 * %y)
1976 // to the same form:
1977 // 2 + sext(4 + 20 * %x + 24 * %y)
1978 if (const auto *SC = dyn_cast<SCEVConstant>(SA->getOperand(0))) {
1979 const APInt &D = extractConstantWithoutWrapping(*this, SC, SA);
1980 if (D != 0) {
1981 const SCEV *SSExtD = getSignExtendExpr(getConstant(D), Ty, Depth);
1982 const SCEV *SResidual =
1984 const SCEV *SSExtR = getSignExtendExpr(SResidual, Ty, Depth + 1);
1985 return getAddExpr(SSExtD, SSExtR,
1987 Depth + 1);
1988 }
1989 }
1990 }
1991 // If the input value is a chrec scev, and we can prove that the value
1992 // did not overflow the old, smaller, value, we can sign extend all of the
1993 // operands (often constants). This allows analysis of something like
1994 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; }
1996 if (AR->isAffine()) {
1997 const SCEV *Start = AR->getStart();
1998 const SCEV *Step = AR->getStepRecurrence(*this);
1999 unsigned BitWidth = getTypeSizeInBits(AR->getType());
2000 const Loop *L = AR->getLoop();
2001
2002 // If we have special knowledge that this addrec won't overflow,
2003 // we don't need to do any further analysis.
2004 if (AR->hasNoSignedWrap()) {
2005 Start =
2007 Step = getSignExtendExpr(Step, Ty, Depth + 1);
2008 return getAddRecExpr(Start, Step, L, SCEV::FlagNSW);
2009 }
2010
2011 // Check whether the backedge-taken count is SCEVCouldNotCompute.
2012 // Note that this serves two purposes: It filters out loops that are
2013 // simply not analyzable, and it covers the case where this code is
2014 // being called from within backedge-taken count analysis, such that
2015 // attempting to ask for the backedge-taken count would likely result
2016 // in infinite recursion. In the later case, the analysis code will
2017 // cope with a conservative value, and it will take care to purge
2018 // that value once it has finished.
2019 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
2020 if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
2021 // Manually compute the final value for AR, checking for
2022 // overflow.
2023
2024 // Check whether the backedge-taken count can be losslessly casted to
2025 // the addrec's type. The count is always unsigned.
2026 const SCEV *CastedMaxBECount =
2027 getTruncateOrZeroExtend(MaxBECount, Start->getType(), Depth);
2028 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
2029 CastedMaxBECount, MaxBECount->getType(), Depth);
2030 if (MaxBECount == RecastedMaxBECount) {
2031 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
2032 // Check whether Start+Step*MaxBECount has no signed overflow.
2033 const SCEV *SMul = getMulExpr(CastedMaxBECount, Step,
2035 const SCEV *SAdd = getSignExtendExpr(getAddExpr(Start, SMul,
2037 Depth + 1),
2038 WideTy, Depth + 1);
2039 const SCEV *WideStart = getSignExtendExpr(Start, WideTy, Depth + 1);
2040 const SCEV *WideMaxBECount =
2041 getZeroExtendExpr(CastedMaxBECount, WideTy, Depth + 1);
2042 const SCEV *OperandExtendedAdd =
2043 getAddExpr(WideStart,
2044 getMulExpr(WideMaxBECount,
2045 getSignExtendExpr(Step, WideTy, Depth + 1),
2048 if (SAdd == OperandExtendedAdd) {
2049 // Cache knowledge of AR NSW, which is propagated to this AddRec.
2050 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNSW);
2051 // Return the expression with the addrec on the outside.
2052 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this,
2053 Depth + 1);
2054 Step = getSignExtendExpr(Step, Ty, Depth + 1);
2055 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
2056 }
2057 // Similar to above, only this time treat the step value as unsigned.
2058 // This covers loops that count up with an unsigned step.
2059 OperandExtendedAdd =
2060 getAddExpr(WideStart,
2061 getMulExpr(WideMaxBECount,
2062 getZeroExtendExpr(Step, WideTy, Depth + 1),
2065 if (SAdd == OperandExtendedAdd) {
2066 // If AR wraps around then
2067 //
2068 // abs(Step) * MaxBECount > unsigned-max(AR->getType())
2069 // => SAdd != OperandExtendedAdd
2070 //
2071 // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=>
2072 // (SAdd == OperandExtendedAdd => AR is NW)
2073
2074 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW);
2075
2076 // Return the expression with the addrec on the outside.
2077 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this,
2078 Depth + 1);
2079 Step = getZeroExtendExpr(Step, Ty, Depth + 1);
2080 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
2081 }
2082 }
2083 }
2084
2085 auto NewFlags = proveNoSignedWrapViaInduction(AR);
2086 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), NewFlags);
2087 if (AR->hasNoSignedWrap()) {
2088 // Same as nsw case above - duplicated here to avoid a compile time
2089 // issue. It's not clear that the order of checks does matter, but
2090 // it's one of two issue possible causes for a change which was
2091 // reverted. Be conservative for the moment.
2092 Start =
2094 Step = getSignExtendExpr(Step, Ty, Depth + 1);
2095 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
2096 }
2097
2098 // sext({C,+,Step}) --> (sext(D) + sext({C-D,+,Step}))<nuw><nsw>
2099 // if D + (C - D + Step * n) could be proven to not signed wrap
2100 // where D maximizes the number of trailing zeros of (C - D + Step * n)
2101 if (const auto *SC = dyn_cast<SCEVConstant>(Start)) {
2102 const APInt &C = SC->getAPInt();
2103 const APInt &D = extractConstantWithoutWrapping(*this, C, Step);
2104 if (D != 0) {
2105 const SCEV *SSExtD = getSignExtendExpr(getConstant(D), Ty, Depth);
2106 const SCEV *SResidual =
2107 getAddRecExpr(getConstant(C - D), Step, L, AR->getNoWrapFlags());
2108 const SCEV *SSExtR = getSignExtendExpr(SResidual, Ty, Depth + 1);
2109 return getAddExpr(SSExtD, SSExtR,
2111 Depth + 1);
2112 }
2113 }
2114
2115 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
2116 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNSW);
2117 Start =
2119 Step = getSignExtendExpr(Step, Ty, Depth + 1);
2120 return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
2121 }
2122 }
2123
2124 // If the input value is provably positive and we could not simplify
2125 // away the sext build a zext instead.
2127 return getZeroExtendExpr(Op, Ty, Depth + 1);
2128
2129 // sext(smin(x, y)) -> smin(sext(x), sext(y))
2130 // sext(smax(x, y)) -> smax(sext(x), sext(y))
2134 for (auto *Operand : MinMax->operands())
2135 Operands.push_back(getSignExtendExpr(Operand, Ty));
2137 return getSMinExpr(Operands);
2138 return getSMaxExpr(Operands);
2139 }
2140
2141 // The cast wasn't folded; create an explicit cast node.
2142 // Recompute the insert position, as it may have been invalidated.
2143 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
2144 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
2145 Op, Ty);
2146 UniqueSCEVs.InsertNode(S, IP);
2147 registerUser(S, { Op });
2148 return S;
2149}
2150
2152 Type *Ty) {
2153 switch (Kind) {
2154 case scTruncate:
2155 return getTruncateExpr(Op, Ty);
2156 case scZeroExtend:
2157 return getZeroExtendExpr(Op, Ty);
2158 case scSignExtend:
2159 return getSignExtendExpr(Op, Ty);
2160 case scPtrToInt:
2161 return getPtrToIntExpr(Op, Ty);
2162 default:
2163 llvm_unreachable("Not a SCEV cast expression!");
2164 }
2165}
2166
2167/// getAnyExtendExpr - Return a SCEV for the given operand extended with
2168/// unspecified bits out to the given type.
2170 Type *Ty) {
2171 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
2172 "This is not an extending conversion!");
2173 assert(isSCEVable(Ty) &&
2174 "This is not a conversion to a SCEVable type!");
2175 Ty = getEffectiveSCEVType(Ty);
2176
2177 // Sign-extend negative constants.
2178 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
2179 if (SC->getAPInt().isNegative())
2180 return getSignExtendExpr(Op, Ty);
2181
2182 // Peel off a truncate cast.
2184 const SCEV *NewOp = T->getOperand();
2185 if (getTypeSizeInBits(NewOp->getType()) < getTypeSizeInBits(Ty))
2186 return getAnyExtendExpr(NewOp, Ty);
2187 return getTruncateOrNoop(NewOp, Ty);
2188 }
2189
2190 // Next try a zext cast. If the cast is folded, use it.
2191 const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
2192 if (!isa<SCEVZeroExtendExpr>(ZExt))
2193 return ZExt;
2194
2195 // Next try a sext cast. If the cast is folded, use it.
2196 const SCEV *SExt = getSignExtendExpr(Op, Ty);
2197 if (!isa<SCEVSignExtendExpr>(SExt))
2198 return SExt;
2199
2200 // Force the cast to be folded into the operands of an addrec.
2201 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) {
2203 for (const SCEV *Op : AR->operands())
2204 Ops.push_back(getAnyExtendExpr(Op, Ty));
2205 return getAddRecExpr(Ops, AR->getLoop(), SCEV::FlagNW);
2206 }
2207
2208 // If the expression is obviously signed, use the sext cast value.
2209 if (isa<SCEVSMaxExpr>(Op))
2210 return SExt;
2211
2212 // Absent any other information, use the zext cast value.
2213 return ZExt;
2214}
2215
2216/// Process the given Ops list, which is a list of operands to be added under
2217/// the given scale, update the given map. This is a helper function for
2218/// getAddRecExpr. As an example of what it does, given a sequence of operands
2219/// that would form an add expression like this:
2220///
2221/// m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
2222///
2223/// where A and B are constants, update the map with these values:
2224///
2225/// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
2226///
2227/// and add 13 + A*B*29 to AccumulatedConstant.
2228/// This will allow getAddRecExpr to produce this:
2229///
2230/// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
2231///
2232/// This form often exposes folding opportunities that are hidden in
2233/// the original operand list.
2234///
2235/// Return true iff it appears that any interesting folding opportunities
2236/// may be exposed. This helps getAddRecExpr short-circuit extra work in
2237/// the common case where no interesting opportunities are present, and
2238/// is also used as a check to avoid infinite recursion.
2239static bool
2242 APInt &AccumulatedConstant,
2243 ArrayRef<const SCEV *> Ops, const APInt &Scale,
2244 ScalarEvolution &SE) {
2245 bool Interesting = false;
2246
2247 // Iterate over the add operands. They are sorted, with constants first.
2248 unsigned i = 0;
2249 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) {
2250 ++i;
2251 // Pull a buried constant out to the outside.
2252 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
2253 Interesting = true;
2254 AccumulatedConstant += Scale * C->getAPInt();
2255 }
2256
2257 // Next comes everything else. We're especially interested in multiplies
2258 // here, but they're in the middle, so just visit the rest with one loop.
2259 for (; i != Ops.size(); ++i) {
2261 if (Mul && isa<SCEVConstant>(Mul->getOperand(0))) {
2262 APInt NewScale =
2263 Scale * cast<SCEVConstant>(Mul->getOperand(0))->getAPInt();
2264 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) {
2265 // A multiplication of a constant with another add; recurse.
2266 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1));
2267 Interesting |=
2268 CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
2269 Add->operands(), NewScale, SE);
2270 } else {
2271 // A multiplication of a constant with some other value. Update
2272 // the map.
2273 SmallVector<const SCEV *, 4> MulOps(drop_begin(Mul->operands()));
2274 const SCEV *Key = SE.getMulExpr(MulOps);
2275 auto Pair = M.insert({Key, NewScale});
2276 if (Pair.second) {
2277 NewOps.push_back(Pair.first->first);
2278 } else {
2279 Pair.first->second += NewScale;
2280 // The map already had an entry for this value, which may indicate
2281 // a folding opportunity.
2282 Interesting = true;
2283 }
2284 }
2285 } else {
2286 // An ordinary operand. Update the map.
2287 std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair =
2288 M.insert({Ops[i], Scale});
2289 if (Pair.second) {
2290 NewOps.push_back(Pair.first->first);
2291 } else {
2292 Pair.first->second += Scale;
2293 // The map already had an entry for this value, which may indicate
2294 // a folding opportunity.
2295 Interesting = true;
2296 }
2297 }
2298 }
2299
2300 return Interesting;
2301}
2302
2304 const SCEV *LHS, const SCEV *RHS,
2305 const Instruction *CtxI) {
2306 const SCEV *(ScalarEvolution::*Operation)(const SCEV *, const SCEV *,
2307 SCEV::NoWrapFlags, unsigned);
2308 switch (BinOp) {
2309 default:
2310 llvm_unreachable("Unsupported binary op");
2311 case Instruction::Add:
2313 break;
2314 case Instruction::Sub:
2316 break;
2317 case Instruction::Mul:
2319 break;
2320 }
2321
2322 const SCEV *(ScalarEvolution::*Extension)(const SCEV *, Type *, unsigned) =
2325
2326 // Check ext(LHS op RHS) == ext(LHS) op ext(RHS)
2327 auto *NarrowTy = cast<IntegerType>(LHS->getType());
2328 auto *WideTy =
2329 IntegerType::get(NarrowTy->getContext(), NarrowTy->getBitWidth() * 2);
2330
2331 const SCEV *A = (this->*Extension)(
2332 (this->*Operation)(LHS, RHS, SCEV::FlagAnyWrap, 0), WideTy, 0);
2333 const SCEV *LHSB = (this->*Extension)(LHS, WideTy, 0);
2334 const SCEV *RHSB = (this->*Extension)(RHS, WideTy, 0);
2335 const SCEV *B = (this->*Operation)(LHSB, RHSB, SCEV::FlagAnyWrap, 0);
2336 if (A == B)
2337 return true;
2338 // Can we use context to prove the fact we need?
2339 if (!CtxI)
2340 return false;
2341 // TODO: Support mul.
2342 if (BinOp == Instruction::Mul)
2343 return false;
2344 auto *RHSC = dyn_cast<SCEVConstant>(RHS);
2345 // TODO: Lift this limitation.
2346 if (!RHSC)
2347 return false;
2348 APInt C = RHSC->getAPInt();
2349 unsigned NumBits = C.getBitWidth();
2350 bool IsSub = (BinOp == Instruction::Sub);
2351 bool IsNegativeConst = (Signed && C.isNegative());
2352 // Compute the direction and magnitude by which we need to check overflow.
2353 bool OverflowDown = IsSub ^ IsNegativeConst;
2354 APInt Magnitude = C;
2355 if (IsNegativeConst) {
2356 if (C == APInt::getSignedMinValue(NumBits))
2357 // TODO: SINT_MIN on inversion gives the same negative value, we don't
2358 // want to deal with that.
2359 return false;
2360 Magnitude = -C;
2361 }
2362
2364 if (OverflowDown) {
2365 // To avoid overflow down, we need to make sure that MIN + Magnitude <= LHS.
2366 APInt Min = Signed ? APInt::getSignedMinValue(NumBits)
2367 : APInt::getMinValue(NumBits);
2368 APInt Limit = Min + Magnitude;
2369 return isKnownPredicateAt(Pred, getConstant(Limit), LHS, CtxI);
2370 } else {
2371 // To avoid overflow up, we need to make sure that LHS <= MAX - Magnitude.
2372 APInt Max = Signed ? APInt::getSignedMaxValue(NumBits)
2373 : APInt::getMaxValue(NumBits);
2374 APInt Limit = Max - Magnitude;
2375 return isKnownPredicateAt(Pred, LHS, getConstant(Limit), CtxI);
2376 }
2377}
2378
2379std::optional<SCEV::NoWrapFlags>
2381 const OverflowingBinaryOperator *OBO) {
2382 // It cannot be done any better.
2383 if (OBO->hasNoUnsignedWrap() && OBO->hasNoSignedWrap())
2384 return std::nullopt;
2385
2387
2388 if (OBO->hasNoUnsignedWrap())
2390 if (OBO->hasNoSignedWrap())
2392
2393 bool Deduced = false;
2394
2395 if (OBO->getOpcode() != Instruction::Add &&
2396 OBO->getOpcode() != Instruction::Sub &&
2397 OBO->getOpcode() != Instruction::Mul)
2398 return std::nullopt;
2399
2400 const SCEV *LHS = getSCEV(OBO->getOperand(0));
2401 const SCEV *RHS = getSCEV(OBO->getOperand(1));
2402
2403 const Instruction *CtxI =
2405 if (!OBO->hasNoUnsignedWrap() &&
2407 /* Signed */ false, LHS, RHS, CtxI)) {
2409 Deduced = true;
2410 }
2411
2412 if (!OBO->hasNoSignedWrap() &&
2414 /* Signed */ true, LHS, RHS, CtxI)) {
2416 Deduced = true;
2417 }
2418
2419 if (Deduced)
2420 return Flags;
2421 return std::nullopt;
2422}
2423
2424// We're trying to construct a SCEV of type `Type' with `Ops' as operands and
2425// `OldFlags' as can't-wrap behavior. Infer a more aggressive set of
2426// can't-overflow flags for the operation if possible.
2427static SCEV::NoWrapFlags
2430 SCEV::NoWrapFlags Flags) {
2431 using namespace std::placeholders;
2432
2433 using OBO = OverflowingBinaryOperator;
2434
2435 bool CanAnalyze =
2437 (void)CanAnalyze;
2438 assert(CanAnalyze && "don't call from other places!");
2439
2440 int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
2441 SCEV::NoWrapFlags SignOrUnsignWrap =
2442 ScalarEvolution::maskFlags(Flags, SignOrUnsignMask);
2443
2444 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
2445 auto IsKnownNonNegative = [&](const SCEV *S) {
2446 return SE->isKnownNonNegative(S);
2447 };
2448
2449 if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Ops, IsKnownNonNegative))
2450 Flags =
2451 ScalarEvolution::setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask);
2452
2453 SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, SignOrUnsignMask);
2454
2455 if (SignOrUnsignWrap != SignOrUnsignMask &&
2456 (Type == scAddExpr || Type == scMulExpr) && Ops.size() == 2 &&
2457 isa<SCEVConstant>(Ops[0])) {
2458
2459 auto Opcode = [&] {
2460 switch (Type) {
2461 case scAddExpr:
2462 return Instruction::Add;
2463 case scMulExpr:
2464 return Instruction::Mul;
2465 default:
2466 llvm_unreachable("Unexpected SCEV op.");
2467 }
2468 }();
2469
2470 const APInt &C = cast<SCEVConstant>(Ops[0])->getAPInt();
2471
2472 // (A <opcode> C) --> (A <opcode> C)<nsw> if the op doesn't sign overflow.
2473 if (!(SignOrUnsignWrap & SCEV::FlagNSW)) {
2475 Opcode, C, OBO::NoSignedWrap);
2476 if (NSWRegion.contains(SE->getSignedRange(Ops[1])))
2478 }
2479
2480 // (A <opcode> C) --> (A <opcode> C)<nuw> if the op doesn't unsign overflow.
2481 if (!(SignOrUnsignWrap & SCEV::FlagNUW)) {
2483 Opcode, C, OBO::NoUnsignedWrap);
2484 if (NUWRegion.contains(SE->getUnsignedRange(Ops[1])))
2486 }
2487 }
2488
2489 // <0,+,nonnegative><nw> is also nuw
2490 // TODO: Add corresponding nsw case
2492 !ScalarEvolution::hasFlags(Flags, SCEV::FlagNUW) && Ops.size() == 2 &&
2493 Ops[0]->isZero() && IsKnownNonNegative(Ops[1]))
2495
2496 // both (udiv X, Y) * Y and Y * (udiv X, Y) are always NUW
2498 Ops.size() == 2) {
2499 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Ops[0]))
2500 if (UDiv->getOperand(1) == Ops[1])
2502 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Ops[1]))
2503 if (UDiv->getOperand(1) == Ops[0])
2505 }
2506
2507 return Flags;
2508}
2509
2511 return isLoopInvariant(S, L) && properlyDominates(S, L->getHeader());
2512}
2513
2514/// Get a canonical add expression, or something simpler if possible.
2516 SCEV::NoWrapFlags OrigFlags,
2517 unsigned Depth) {
2518 assert(!(OrigFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
2519 "only nuw or nsw allowed");
2520 assert(!Ops.empty() && "Cannot get empty add!");
2521 if (Ops.size() == 1) return Ops[0];
2522#ifndef NDEBUG
2523 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
2524 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
2525 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
2526 "SCEVAddExpr operand types don't match!");
2527 unsigned NumPtrs = count_if(
2528 Ops, [](const SCEV *Op) { return Op->getType()->isPointerTy(); });
2529 assert(NumPtrs <= 1 && "add has at most one pointer operand");
2530#endif
2531
2532 const SCEV *Folded = constantFoldAndGroupOps(
2533 *this, LI, DT, Ops,
2534 [](const APInt &C1, const APInt &C2) { return C1 + C2; },
2535 [](const APInt &C) { return C.isZero(); }, // identity
2536 [](const APInt &C) { return false; }); // absorber
2537 if (Folded)
2538 return Folded;
2539
2540 unsigned Idx = isa<SCEVConstant>(Ops[0]) ? 1 : 0;
2541
2542 // Delay expensive flag strengthening until necessary.
2543 auto ComputeFlags = [this, OrigFlags](const ArrayRef<const SCEV *> Ops) {
2544 return StrengthenNoWrapFlags(this, scAddExpr, Ops, OrigFlags);
2545 };
2546
2547 // Limit recursion calls depth.
2549 return getOrCreateAddExpr(Ops, ComputeFlags(Ops));
2550
2551 if (SCEV *S = findExistingSCEVInCache(scAddExpr, Ops)) {
2552 // Don't strengthen flags if we have no new information.
2553 SCEVAddExpr *Add = static_cast<SCEVAddExpr *>(S);
2554 if (Add->getNoWrapFlags(OrigFlags) != OrigFlags)
2555 Add->setNoWrapFlags(ComputeFlags(Ops));
2556 return S;
2557 }
2558
2559 // Okay, check to see if the same value occurs in the operand list more than
2560 // once. If so, merge them together into an multiply expression. Since we
2561 // sorted the list, these values are required to be adjacent.
2562 Type *Ty = Ops[0]->getType();
2563 bool FoundMatch = false;
2564 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
2565 if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2
2566 // Scan ahead to count how many equal operands there are.
2567 unsigned Count = 2;
2568 while (i+Count != e && Ops[i+Count] == Ops[i])
2569 ++Count;
2570 // Merge the values into a multiply.
2571 const SCEV *Scale = getConstant(Ty, Count);
2572 const SCEV *Mul = getMulExpr(Scale, Ops[i], SCEV::FlagAnyWrap, Depth + 1);
2573 if (Ops.size() == Count)
2574 return Mul;
2575 Ops[i] = Mul;
2576 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+Count);
2577 --i; e -= Count - 1;
2578 FoundMatch = true;
2579 }
2580 if (FoundMatch)
2581 return getAddExpr(Ops, OrigFlags, Depth + 1);
2582
2583 // Check for truncates. If all the operands are truncated from the same
2584 // type, see if factoring out the truncate would permit the result to be
2585 // folded. eg., n*trunc(x) + m*trunc(y) --> trunc(trunc(m)*x + trunc(n)*y)
2586 // if the contents of the resulting outer trunc fold to something simple.
2587 auto FindTruncSrcType = [&]() -> Type * {
2588 // We're ultimately looking to fold an addrec of truncs and muls of only
2589 // constants and truncs, so if we find any other types of SCEV
2590 // as operands of the addrec then we bail and return nullptr here.
2591 // Otherwise, we return the type of the operand of a trunc that we find.
2592 if (auto *T = dyn_cast<SCEVTruncateExpr>(Ops[Idx]))
2593 return T->getOperand()->getType();
2594 if (const auto *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
2595 const auto *LastOp = Mul->getOperand(Mul->getNumOperands() - 1);
2596 if (const auto *T = dyn_cast<SCEVTruncateExpr>(LastOp))
2597 return T->getOperand()->getType();
2598 }
2599 return nullptr;
2600 };
2601 if (auto *SrcType = FindTruncSrcType()) {
2603 bool Ok = true;
2604 // Check all the operands to see if they can be represented in the
2605 // source type of the truncate.
2606 for (const SCEV *Op : Ops) {
2608 if (T->getOperand()->getType() != SrcType) {
2609 Ok = false;
2610 break;
2611 }
2612 LargeOps.push_back(T->getOperand());
2613 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Op)) {
2614 LargeOps.push_back(getAnyExtendExpr(C, SrcType));
2615 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Op)) {
2616 SmallVector<const SCEV *, 8> LargeMulOps;
2617 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
2618 if (const SCEVTruncateExpr *T =
2619 dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) {
2620 if (T->getOperand()->getType() != SrcType) {
2621 Ok = false;
2622 break;
2623 }
2624 LargeMulOps.push_back(T->getOperand());
2625 } else if (const auto *C = dyn_cast<SCEVConstant>(M->getOperand(j))) {
2626 LargeMulOps.push_back(getAnyExtendExpr(C, SrcType));
2627 } else {
2628 Ok = false;
2629 break;
2630 }
2631 }
2632 if (Ok)
2633 LargeOps.push_back(getMulExpr(LargeMulOps, SCEV::FlagAnyWrap, Depth + 1));
2634 } else {
2635 Ok = false;
2636 break;
2637 }
2638 }
2639 if (Ok) {
2640 // Evaluate the expression in the larger type.
2641 const SCEV *Fold = getAddExpr(LargeOps, SCEV::FlagAnyWrap, Depth + 1);
2642 // If it folds to something simple, use it. Otherwise, don't.
2643 if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold))
2644 return getTruncateExpr(Fold, Ty);
2645 }
2646 }
2647
2648 if (Ops.size() == 2) {
2649 // Check if we have an expression of the form ((X + C1) - C2), where C1 and
2650 // C2 can be folded in a way that allows retaining wrapping flags of (X +
2651 // C1).
2652 const SCEV *A = Ops[0];
2653 const SCEV *B = Ops[1];
2654 auto *AddExpr = dyn_cast<SCEVAddExpr>(B);
2655 auto *C = dyn_cast<SCEVConstant>(A);
2656 if (AddExpr && C && isa<SCEVConstant>(AddExpr->getOperand(0))) {
2657 auto C1 = cast<SCEVConstant>(AddExpr->getOperand(0))->getAPInt();
2658 auto C2 = C->getAPInt();
2659 SCEV::NoWrapFlags PreservedFlags = SCEV::FlagAnyWrap;
2660
2661 APInt ConstAdd = C1 + C2;
2662 auto AddFlags = AddExpr->getNoWrapFlags();
2663 // Adding a smaller constant is NUW if the original AddExpr was NUW.
2665 ConstAdd.ule(C1)) {
2666 PreservedFlags =
2668 }
2669
2670 // Adding a constant with the same sign and small magnitude is NSW, if the
2671 // original AddExpr was NSW.
2673 C1.isSignBitSet() == ConstAdd.isSignBitSet() &&
2674 ConstAdd.abs().ule(C1.abs())) {
2675 PreservedFlags =
2677 }
2678
2679 if (PreservedFlags != SCEV::FlagAnyWrap) {
2680 SmallVector<const SCEV *, 4> NewOps(AddExpr->operands());
2681 NewOps[0] = getConstant(ConstAdd);
2682 return getAddExpr(NewOps, PreservedFlags);
2683 }
2684 }
2685
2686 // Try to push the constant operand into a ZExt: A + zext (-A + B) -> zext
2687 // (B), if trunc (A) + -A + B does not unsigned-wrap.
2688 const SCEVAddExpr *InnerAdd;
2689 if (match(B, m_scev_ZExt(m_scev_Add(InnerAdd)))) {
2690 const SCEV *NarrowA = getTruncateExpr(A, InnerAdd->getType());
2691 if (NarrowA == getNegativeSCEV(InnerAdd->getOperand(0)) &&
2692 getZeroExtendExpr(NarrowA, B->getType()) == A &&
2693 hasFlags(StrengthenNoWrapFlags(this, scAddExpr, {NarrowA, InnerAdd},
2695 SCEV::FlagNUW)) {
2696 return getZeroExtendExpr(getAddExpr(NarrowA, InnerAdd), B->getType());
2697 }
2698 }
2699 }
2700
2701 // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y)
2702 const SCEV *Y;
2703 if (Ops.size() == 2 &&
2704 match(Ops[0],
2706 m_scev_URem(m_scev_Specific(Ops[1]), m_SCEV(Y), *this))))
2707 return getMulExpr(Y, getUDivExpr(Ops[1], Y));
2708
2709 // Skip past any other cast SCEVs.
2710 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
2711 ++Idx;
2712
2713 // If there are add operands they would be next.
2714 if (Idx < Ops.size()) {
2715 bool DeletedAdd = false;
2716 // If the original flags and all inlined SCEVAddExprs are NUW, use the
2717 // common NUW flag for expression after inlining. Other flags cannot be
2718 // preserved, because they may depend on the original order of operations.
2719 SCEV::NoWrapFlags CommonFlags = maskFlags(OrigFlags, SCEV::FlagNUW);
2720 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) {
2721 if (Ops.size() > AddOpsInlineThreshold ||
2722 Add->getNumOperands() > AddOpsInlineThreshold)
2723 break;
2724 // If we have an add, expand the add operands onto the end of the operands
2725 // list.
2726 Ops.erase(Ops.begin()+Idx);
2727 append_range(Ops, Add->operands());
2728 DeletedAdd = true;
2729 CommonFlags = maskFlags(CommonFlags, Add->getNoWrapFlags());
2730 }
2731
2732 // If we deleted at least one add, we added operands to the end of the list,
2733 // and they are not necessarily sorted. Recurse to resort and resimplify
2734 // any operands we just acquired.
2735 if (DeletedAdd)
2736 return getAddExpr(Ops, CommonFlags, Depth + 1);
2737 }
2738
2739 // Skip over the add expression until we get to a multiply.
2740 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
2741 ++Idx;
2742
2743 // Check to see if there are any folding opportunities present with
2744 // operands multiplied by constant values.
2745 if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) {
2749 APInt AccumulatedConstant(BitWidth, 0);
2750 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
2751 Ops, APInt(BitWidth, 1), *this)) {
2752 struct APIntCompare {
2753 bool operator()(const APInt &LHS, const APInt &RHS) const {
2754 return LHS.ult(RHS);
2755 }
2756 };
2757
2758 // Some interesting folding opportunity is present, so its worthwhile to
2759 // re-generate the operands list. Group the operands by constant scale,
2760 // to avoid multiplying by the same constant scale multiple times.
2761 std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists;
2762 for (const SCEV *NewOp : NewOps)
2763 MulOpLists[M.find(NewOp)->second].push_back(NewOp);
2764 // Re-generate the operands list.
2765 Ops.clear();
2766 if (AccumulatedConstant != 0)
2767 Ops.push_back(getConstant(AccumulatedConstant));
2768 for (auto &MulOp : MulOpLists) {
2769 if (MulOp.first == 1) {
2770 Ops.push_back(getAddExpr(MulOp.second, SCEV::FlagAnyWrap, Depth + 1));
2771 } else if (MulOp.first != 0) {
2772 Ops.push_back(getMulExpr(
2773 getConstant(MulOp.first),
2774 getAddExpr(MulOp.second, SCEV::FlagAnyWrap, Depth + 1),
2775 SCEV::FlagAnyWrap, Depth + 1));
2776 }
2777 }
2778 if (Ops.empty())
2779 return getZero(Ty);
2780 if (Ops.size() == 1)
2781 return Ops[0];
2782 return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
2783 }
2784 }
2785
2786 // If we are adding something to a multiply expression, make sure the
2787 // something is not already an operand of the multiply. If so, merge it into
2788 // the multiply.
2789 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) {
2790 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]);
2791 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
2792 const SCEV *MulOpSCEV = Mul->getOperand(MulOp);
2793 if (isa<SCEVConstant>(MulOpSCEV))
2794 continue;
2795 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp)
2796 if (MulOpSCEV == Ops[AddOp]) {
2797 // Fold W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1))
2798 const SCEV *InnerMul = Mul->getOperand(MulOp == 0);
2799 if (Mul->getNumOperands() != 2) {
2800 // If the multiply has more than two operands, we must get the
2801 // Y*Z term.
2803 Mul->operands().take_front(MulOp));
2804 append_range(MulOps, Mul->operands().drop_front(MulOp + 1));
2805 InnerMul = getMulExpr(MulOps, SCEV::FlagAnyWrap, Depth + 1);
2806 }
2807 SmallVector<const SCEV *, 2> TwoOps = {getOne(Ty), InnerMul};
2808 const SCEV *AddOne = getAddExpr(TwoOps, SCEV::FlagAnyWrap, Depth + 1);
2809 const SCEV *OuterMul = getMulExpr(AddOne, MulOpSCEV,
2811 if (Ops.size() == 2) return OuterMul;
2812 if (AddOp < Idx) {
2813 Ops.erase(Ops.begin()+AddOp);
2814 Ops.erase(Ops.begin()+Idx-1);
2815 } else {
2816 Ops.erase(Ops.begin()+Idx);
2817 Ops.erase(Ops.begin()+AddOp-1);
2818 }
2819 Ops.push_back(OuterMul);
2820 return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
2821 }
2822
2823 // Check this multiply against other multiplies being added together.
2824 for (unsigned OtherMulIdx = Idx+1;
2825 OtherMulIdx < Ops.size() && isa<SCEVMulExpr>(Ops[OtherMulIdx]);
2826 ++OtherMulIdx) {
2827 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[OtherMulIdx]);
2828 // If MulOp occurs in OtherMul, we can fold the two multiplies
2829 // together.
2830 for (unsigned OMulOp = 0, e = OtherMul->getNumOperands();
2831 OMulOp != e; ++OMulOp)
2832 if (OtherMul->getOperand(OMulOp) == MulOpSCEV) {
2833 // Fold X + (A*B*C) + (A*D*E) --> X + (A*(B*C+D*E))
2834 const SCEV *InnerMul1 = Mul->getOperand(MulOp == 0);
2835 if (Mul->getNumOperands() != 2) {
2837 Mul->operands().take_front(MulOp));
2838 append_range(MulOps, Mul->operands().drop_front(MulOp+1));
2839 InnerMul1 = getMulExpr(MulOps, SCEV::FlagAnyWrap, Depth + 1);
2840 }
2841 const SCEV *InnerMul2 = OtherMul->getOperand(OMulOp == 0);
2842 if (OtherMul->getNumOperands() != 2) {
2844 OtherMul->operands().take_front(OMulOp));
2845 append_range(MulOps, OtherMul->operands().drop_front(OMulOp+1));
2846 InnerMul2 = getMulExpr(MulOps, SCEV::FlagAnyWrap, Depth + 1);
2847 }
2848 SmallVector<const SCEV *, 2> TwoOps = {InnerMul1, InnerMul2};
2849 const SCEV *InnerMulSum =
2850 getAddExpr(TwoOps, SCEV::FlagAnyWrap, Depth + 1);
2851 const SCEV *OuterMul = getMulExpr(MulOpSCEV, InnerMulSum,
2853 if (Ops.size() == 2) return OuterMul;
2854 Ops.erase(Ops.begin()+Idx);
2855 Ops.erase(Ops.begin()+OtherMulIdx-1);
2856 Ops.push_back(OuterMul);
2857 return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
2858 }
2859 }
2860 }
2861 }
2862
2863 // If there are any add recurrences in the operands list, see if any other
2864 // added values are loop invariant. If so, we can fold them into the
2865 // recurrence.
2866 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
2867 ++Idx;
2868
2869 // Scan over all recurrences, trying to fold loop invariants into them.
2870 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
2871 // Scan all of the other operands to this add and add them to the vector if
2872 // they are loop invariant w.r.t. the recurrence.
2874 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
2875 const Loop *AddRecLoop = AddRec->getLoop();
2876 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
2877 if (isAvailableAtLoopEntry(Ops[i], AddRecLoop)) {
2878 LIOps.push_back(Ops[i]);
2879 Ops.erase(Ops.begin()+i);
2880 --i; --e;
2881 }
2882
2883 // If we found some loop invariants, fold them into the recurrence.
2884 if (!LIOps.empty()) {
2885 // Compute nowrap flags for the addition of the loop-invariant ops and
2886 // the addrec. Temporarily push it as an operand for that purpose. These
2887 // flags are valid in the scope of the addrec only.
2888 LIOps.push_back(AddRec);
2889 SCEV::NoWrapFlags Flags = ComputeFlags(LIOps);
2890 LIOps.pop_back();
2891
2892 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step}
2893 LIOps.push_back(AddRec->getStart());
2894
2895 SmallVector<const SCEV *, 4> AddRecOps(AddRec->operands());
2896
2897 // It is not in general safe to propagate flags valid on an add within
2898 // the addrec scope to one outside it. We must prove that the inner
2899 // scope is guaranteed to execute if the outer one does to be able to
2900 // safely propagate. We know the program is undefined if poison is
2901 // produced on the inner scoped addrec. We also know that *for this use*
2902 // the outer scoped add can't overflow (because of the flags we just
2903 // computed for the inner scoped add) without the program being undefined.
2904 // Proving that entry to the outer scope neccesitates entry to the inner
2905 // scope, thus proves the program undefined if the flags would be violated
2906 // in the outer scope.
2907 SCEV::NoWrapFlags AddFlags = Flags;
2908 if (AddFlags != SCEV::FlagAnyWrap) {
2909 auto *DefI = getDefiningScopeBound(LIOps);
2910 auto *ReachI = &*AddRecLoop->getHeader()->begin();
2911 if (!isGuaranteedToTransferExecutionTo(DefI, ReachI))
2912 AddFlags = SCEV::FlagAnyWrap;
2913 }
2914 AddRecOps[0] = getAddExpr(LIOps, AddFlags, Depth + 1);
2915
2916 // Build the new addrec. Propagate the NUW and NSW flags if both the
2917 // outer add and the inner addrec are guaranteed to have no overflow.
2918 // Always propagate NW.
2919 Flags = AddRec->getNoWrapFlags(setFlags(Flags, SCEV::FlagNW));
2920 const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, Flags);
2921
2922 // If all of the other operands were loop invariant, we are done.
2923 if (Ops.size() == 1) return NewRec;
2924
2925 // Otherwise, add the folded AddRec by the non-invariant parts.
2926 for (unsigned i = 0;; ++i)
2927 if (Ops[i] == AddRec) {
2928 Ops[i] = NewRec;
2929 break;
2930 }
2931 return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
2932 }
2933
2934 // Okay, if there weren't any loop invariants to be folded, check to see if
2935 // there are multiple AddRec's with the same loop induction variable being
2936 // added together. If so, we can fold them.
2937 for (unsigned OtherIdx = Idx+1;
2938 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
2939 ++OtherIdx) {
2940 // We expect the AddRecExpr's to be sorted in reverse dominance order,
2941 // so that the 1st found AddRecExpr is dominated by all others.
2942 assert(DT.dominates(
2943 cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(),
2944 AddRec->getLoop()->getHeader()) &&
2945 "AddRecExprs are not sorted in reverse dominance order?");
2946 if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) {
2947 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L>
2948 SmallVector<const SCEV *, 4> AddRecOps(AddRec->operands());
2949 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
2950 ++OtherIdx) {
2951 const auto *OtherAddRec = cast<SCEVAddRecExpr>(Ops[OtherIdx]);
2952 if (OtherAddRec->getLoop() == AddRecLoop) {
2953 for (unsigned i = 0, e = OtherAddRec->getNumOperands();
2954 i != e; ++i) {
2955 if (i >= AddRecOps.size()) {
2956 append_range(AddRecOps, OtherAddRec->operands().drop_front(i));
2957 break;
2958 }
2960 AddRecOps[i], OtherAddRec->getOperand(i)};
2961 AddRecOps[i] = getAddExpr(TwoOps, SCEV::FlagAnyWrap, Depth + 1);
2962 }
2963 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
2964 }
2965 }
2966 // Step size has changed, so we cannot guarantee no self-wraparound.
2967 Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagAnyWrap);
2968 return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
2969 }
2970 }
2971
2972 // Otherwise couldn't fold anything into this recurrence. Move onto the
2973 // next one.
2974 }
2975
2976 // Okay, it looks like we really DO need an add expr. Check to see if we
2977 // already have one, otherwise create a new one.
2978 return getOrCreateAddExpr(Ops, ComputeFlags(Ops));
2979}
2980
2981const SCEV *
2982ScalarEvolution::getOrCreateAddExpr(ArrayRef<const SCEV *> Ops,
2983 SCEV::NoWrapFlags Flags) {
2985 ID.AddInteger(scAddExpr);
2986 for (const SCEV *Op : Ops)
2987 ID.AddPointer(Op);
2988 void *IP = nullptr;
2989 SCEVAddExpr *S =
2990 static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
2991 if (!S) {
2992 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
2994 S = new (SCEVAllocator)
2995 SCEVAddExpr(ID.Intern(SCEVAllocator), O, Ops.size());
2996 UniqueSCEVs.InsertNode(S, IP);
2997 registerUser(S, Ops);
2998 }
2999 S->setNoWrapFlags(Flags);
3000 return S;
3001}
3002
3003const SCEV *
3004ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<const SCEV *> Ops,
3005 const Loop *L, SCEV::NoWrapFlags Flags) {
3006 FoldingSetNodeID ID;
3007 ID.AddInteger(scAddRecExpr);
3008 for (const SCEV *Op : Ops)
3009 ID.AddPointer(Op);
3010 ID.AddPointer(L);
3011 void *IP = nullptr;
3012 SCEVAddRecExpr *S =
3013 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
3014 if (!S) {
3015 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
3017 S = new (SCEVAllocator)
3018 SCEVAddRecExpr(ID.Intern(SCEVAllocator), O, Ops.size(), L);
3019 UniqueSCEVs.InsertNode(S, IP);
3020 LoopUsers[L].push_back(S);
3021 registerUser(S, Ops);
3022 }
3023 setNoWrapFlags(S, Flags);
3024 return S;
3025}
3026
3027const SCEV *
3028ScalarEvolution::getOrCreateMulExpr(ArrayRef<const SCEV *> Ops,
3029 SCEV::NoWrapFlags Flags) {
3030 FoldingSetNodeID ID;
3031 ID.AddInteger(scMulExpr);
3032 for (const SCEV *Op : Ops)
3033 ID.AddPointer(Op);
3034 void *IP = nullptr;
3035 SCEVMulExpr *S =
3036 static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
3037 if (!S) {
3038 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
3040 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator),
3041 O, Ops.size());
3042 UniqueSCEVs.InsertNode(S, IP);
3043 registerUser(S, Ops);
3044 }
3045 S->setNoWrapFlags(Flags);
3046 return S;
3047}
3048
3049static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
3050 uint64_t k = i*j;
3051 if (j > 1 && k / j != i) Overflow = true;
3052 return k;
3053}
3054
3055/// Compute the result of "n choose k", the binomial coefficient. If an
3056/// intermediate computation overflows, Overflow will be set and the return will
3057/// be garbage. Overflow is not cleared on absence of overflow.
3058static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
3059 // We use the multiplicative formula:
3060 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
3061 // At each iteration, we take the n-th term of the numeral and divide by the
3062 // (k-n)th term of the denominator. This division will always produce an
3063 // integral result, and helps reduce the chance of overflow in the
3064 // intermediate computations. However, we can still overflow even when the
3065 // final result would fit.
3066
3067 if (n == 0 || n == k) return 1;
3068 if (k > n) return 0;
3069
3070 if (k > n/2)
3071 k = n-k;
3072
3073 uint64_t r = 1;
3074 for (uint64_t i = 1; i <= k; ++i) {
3075 r = umul_ov(r, n-(i-1), Overflow);
3076 r /= i;
3077 }
3078 return r;
3079}
3080
3081/// Determine if any of the operands in this SCEV are a constant or if
3082/// any of the add or multiply expressions in this SCEV contain a constant.
3083static bool containsConstantInAddMulChain(const SCEV *StartExpr) {
3084 struct FindConstantInAddMulChain {
3085 bool FoundConstant = false;
3086
3087 bool follow(const SCEV *S) {
3088 FoundConstant |= isa<SCEVConstant>(S);
3089 return isa<SCEVAddExpr>(S) || isa<SCEVMulExpr>(S);
3090 }
3091
3092 bool isDone() const {
3093 return FoundConstant;
3094 }
3095 };
3096
3097 FindConstantInAddMulChain F;
3099 ST.visitAll(StartExpr);
3100 return F.FoundConstant;
3101}
3102
3103/// Get a canonical multiply expression, or something simpler if possible.
3105 SCEV::NoWrapFlags OrigFlags,
3106 unsigned Depth) {
3107 assert(OrigFlags == maskFlags(OrigFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
3108 "only nuw or nsw allowed");
3109 assert(!Ops.empty() && "Cannot get empty mul!");
3110 if (Ops.size() == 1) return Ops[0];
3111#ifndef NDEBUG
3112 Type *ETy = Ops[0]->getType();
3113 assert(!ETy->isPointerTy());
3114 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
3115 assert(Ops[i]->getType() == ETy &&
3116 "SCEVMulExpr operand types don't match!");
3117#endif
3118
3119 const SCEV *Folded = constantFoldAndGroupOps(
3120 *this, LI, DT, Ops,
3121 [](const APInt &C1, const APInt &C2) { return C1 * C2; },
3122 [](const APInt &C) { return C.isOne(); }, // identity
3123 [](const APInt &C) { return C.isZero(); }); // absorber
3124 if (Folded)
3125 return Folded;
3126
3127 // Delay expensive flag strengthening until necessary.
3128 auto ComputeFlags = [this, OrigFlags](const ArrayRef<const SCEV *> Ops) {
3129 return StrengthenNoWrapFlags(this, scMulExpr, Ops, OrigFlags);
3130 };
3131
3132 // Limit recursion calls depth.
3134 return getOrCreateMulExpr(Ops, ComputeFlags(Ops));
3135
3136 if (SCEV *S = findExistingSCEVInCache(scMulExpr, Ops)) {
3137 // Don't strengthen flags if we have no new information.
3138 SCEVMulExpr *Mul = static_cast<SCEVMulExpr *>(S);
3139 if (Mul->getNoWrapFlags(OrigFlags) != OrigFlags)
3140 Mul->setNoWrapFlags(ComputeFlags(Ops));
3141 return S;
3142 }
3143
3144 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
3145 if (Ops.size() == 2) {
3146 // C1*(C2+V) -> C1*C2 + C1*V
3147 // If any of Add's ops are Adds or Muls with a constant, apply this
3148 // transformation as well.
3149 //
3150 // TODO: There are some cases where this transformation is not
3151 // profitable; for example, Add = (C0 + X) * Y + Z. Maybe the scope of
3152 // this transformation should be narrowed down.
3153 const SCEV *Op0, *Op1;
3154 if (match(Ops[1], m_scev_Add(m_SCEV(Op0), m_SCEV(Op1))) &&
3156 const SCEV *LHS = getMulExpr(LHSC, Op0, SCEV::FlagAnyWrap, Depth + 1);
3157 const SCEV *RHS = getMulExpr(LHSC, Op1, SCEV::FlagAnyWrap, Depth + 1);
3158 return getAddExpr(LHS, RHS, SCEV::FlagAnyWrap, Depth + 1);
3159 }
3160
3161 if (Ops[0]->isAllOnesValue()) {
3162 // If we have a mul by -1 of an add, try distributing the -1 among the
3163 // add operands.
3164 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) {
3166 bool AnyFolded = false;
3167 for (const SCEV *AddOp : Add->operands()) {
3168 const SCEV *Mul = getMulExpr(Ops[0], AddOp, SCEV::FlagAnyWrap,
3169 Depth + 1);
3170 if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true;
3171 NewOps.push_back(Mul);
3172 }
3173 if (AnyFolded)
3174 return getAddExpr(NewOps, SCEV::FlagAnyWrap, Depth + 1);
3175 } else if (const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Ops[1])) {
3176 // Negation preserves a recurrence's no self-wrap property.
3178 for (const SCEV *AddRecOp : AddRec->operands())
3179 Operands.push_back(getMulExpr(Ops[0], AddRecOp, SCEV::FlagAnyWrap,
3180 Depth + 1));
3181 // Let M be the minimum representable signed value. AddRec with nsw
3182 // multiplied by -1 can have signed overflow if and only if it takes a
3183 // value of M: M * (-1) would stay M and (M + 1) * (-1) would be the
3184 // maximum signed value. In all other cases signed overflow is
3185 // impossible.
3186 auto FlagsMask = SCEV::FlagNW;
3187 if (hasFlags(AddRec->getNoWrapFlags(), SCEV::FlagNSW)) {
3188 auto MinInt =
3189 APInt::getSignedMinValue(getTypeSizeInBits(AddRec->getType()));
3190 if (getSignedRangeMin(AddRec) != MinInt)
3191 FlagsMask = setFlags(FlagsMask, SCEV::FlagNSW);
3192 }
3193 return getAddRecExpr(Operands, AddRec->getLoop(),
3194 AddRec->getNoWrapFlags(FlagsMask));
3195 }
3196 }
3197
3198 // Try to push the constant operand into a ZExt: C * zext (A + B) ->
3199 // zext (C*A + C*B) if trunc (C) * (A + B) does not unsigned-wrap.
3200 const SCEVAddExpr *InnerAdd;
3201 if (match(Ops[1], m_scev_ZExt(m_scev_Add(InnerAdd)))) {
3202 const SCEV *NarrowC = getTruncateExpr(LHSC, InnerAdd->getType());
3203 if (isa<SCEVConstant>(InnerAdd->getOperand(0)) &&
3204 getZeroExtendExpr(NarrowC, Ops[1]->getType()) == LHSC &&
3205 hasFlags(StrengthenNoWrapFlags(this, scMulExpr, {NarrowC, InnerAdd},
3207 SCEV::FlagNUW)) {
3208 auto *Res = getMulExpr(NarrowC, InnerAdd, SCEV::FlagNUW, Depth + 1);
3209 return getZeroExtendExpr(Res, Ops[1]->getType(), Depth + 1);
3210 };
3211 }
3212
3213 // Try to fold (C1 * D /u C2) -> C1/C2 * D, if C1 and C2 are powers-of-2,
3214 // D is a multiple of C2, and C1 is a multiple of C2. If C2 is a multiple
3215 // of C1, fold to (D /u (C2 /u C1)).
3216 const SCEV *D;
3217 APInt C1V = LHSC->getAPInt();
3218 // (C1 * D /u C2) == -1 * -C1 * D /u C2 when C1 != INT_MIN. Don't treat -1
3219 // as -1 * 1, as it won't enable additional folds.
3220 if (C1V.isNegative() && !C1V.isMinSignedValue() && !C1V.isAllOnes())
3221 C1V = C1V.abs();
3222 const SCEVConstant *C2;
3223 if (C1V.isPowerOf2() &&
3225 C2->getAPInt().isPowerOf2() &&
3226 C1V.logBase2() <= getMinTrailingZeros(D)) {
3227 const SCEV *NewMul = nullptr;
3228 if (C1V.uge(C2->getAPInt())) {
3229 NewMul = getMulExpr(getUDivExpr(getConstant(C1V), C2), D);
3230 } else if (C2->getAPInt().logBase2() <= getMinTrailingZeros(D)) {
3231 assert(C1V.ugt(1) && "C1 <= 1 should have been folded earlier");
3232 NewMul = getUDivExpr(D, getUDivExpr(C2, getConstant(C1V)));
3233 }
3234 if (NewMul)
3235 return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(NewMul);
3236 }
3237 }
3238 }
3239
3240 // Skip over the add expression until we get to a multiply.
3241 unsigned Idx = 0;
3242 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
3243 ++Idx;
3244
3245 // If there are mul operands inline them all into this expression.
3246 if (Idx < Ops.size()) {
3247 bool DeletedMul = false;
3248 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
3249 if (Ops.size() > MulOpsInlineThreshold)
3250 break;
3251 // If we have an mul, expand the mul operands onto the end of the
3252 // operands list.
3253 Ops.erase(Ops.begin()+Idx);
3254 append_range(Ops, Mul->operands());
3255 DeletedMul = true;
3256 }
3257
3258 // If we deleted at least one mul, we added operands to the end of the
3259 // list, and they are not necessarily sorted. Recurse to resort and
3260 // resimplify any operands we just acquired.
3261 if (DeletedMul)
3262 return getMulExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
3263 }
3264
3265 // If there are any add recurrences in the operands list, see if any other
3266 // added values are loop invariant. If so, we can fold them into the
3267 // recurrence.
3268 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
3269 ++Idx;
3270
3271 // Scan over all recurrences, trying to fold loop invariants into them.
3272 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
3273 // Scan all of the other operands to this mul and add them to the vector
3274 // if they are loop invariant w.r.t. the recurrence.
3276 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
3277 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
3278 if (isAvailableAtLoopEntry(Ops[i], AddRec->getLoop())) {
3279 LIOps.push_back(Ops[i]);
3280 Ops.erase(Ops.begin()+i);
3281 --i; --e;
3282 }
3283
3284 // If we found some loop invariants, fold them into the recurrence.
3285 if (!LIOps.empty()) {
3286 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step}
3288 NewOps.reserve(AddRec->getNumOperands());
3289 const SCEV *Scale = getMulExpr(LIOps, SCEV::FlagAnyWrap, Depth + 1);
3290
3291 // If both the mul and addrec are nuw, we can preserve nuw.
3292 // If both the mul and addrec are nsw, we can only preserve nsw if either
3293 // a) they are also nuw, or
3294 // b) all multiplications of addrec operands with scale are nsw.
3295 SCEV::NoWrapFlags Flags =
3296 AddRec->getNoWrapFlags(ComputeFlags({Scale, AddRec}));
3297
3298 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
3299 NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i),
3300 SCEV::FlagAnyWrap, Depth + 1));
3301
3302 if (hasFlags(Flags, SCEV::FlagNSW) && !hasFlags(Flags, SCEV::FlagNUW)) {
3304 Instruction::Mul, getSignedRange(Scale),
3306 if (!NSWRegion.contains(getSignedRange(AddRec->getOperand(i))))
3307 Flags = clearFlags(Flags, SCEV::FlagNSW);
3308 }
3309 }
3310
3311 const SCEV *NewRec = getAddRecExpr(NewOps, AddRec->getLoop(), Flags);
3312
3313 // If all of the other operands were loop invariant, we are done.
3314 if (Ops.size() == 1) return NewRec;
3315
3316 // Otherwise, multiply the folded AddRec by the non-invariant parts.
3317 for (unsigned i = 0;; ++i)
3318 if (Ops[i] == AddRec) {
3319 Ops[i] = NewRec;
3320 break;
3321 }
3322 return getMulExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
3323 }
3324
3325 // Okay, if there weren't any loop invariants to be folded, check to see
3326 // if there are multiple AddRec's with the same loop induction variable
3327 // being multiplied together. If so, we can fold them.
3328
3329 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
3330 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
3331 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
3332 // ]]],+,...up to x=2n}.
3333 // Note that the arguments to choose() are always integers with values
3334 // known at compile time, never SCEV objects.
3335 //
3336 // The implementation avoids pointless extra computations when the two
3337 // addrec's are of different length (mathematically, it's equivalent to
3338 // an infinite stream of zeros on the right).
3339 bool OpsModified = false;
3340 for (unsigned OtherIdx = Idx+1;
3341 OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
3342 ++OtherIdx) {
3343 const SCEVAddRecExpr *OtherAddRec =
3344 dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]);
3345 if (!OtherAddRec || OtherAddRec->getLoop() != AddRec->getLoop())
3346 continue;
3347
3348 // Limit max number of arguments to avoid creation of unreasonably big
3349 // SCEVAddRecs with very complex operands.
3350 if (AddRec->getNumOperands() + OtherAddRec->getNumOperands() - 1 >
3351 MaxAddRecSize || hasHugeExpression({AddRec, OtherAddRec}))
3352 continue;
3353
3354 bool Overflow = false;
3355 Type *Ty = AddRec->getType();
3356 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
3358 for (int x = 0, xe = AddRec->getNumOperands() +
3359 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
3361 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
3362 uint64_t Coeff1 = Choose(x, 2*x - y, Overflow);
3363 for (int z = std::max(y-x, y-(int)AddRec->getNumOperands()+1),
3364 ze = std::min(x+1, (int)OtherAddRec->getNumOperands());
3365 z < ze && !Overflow; ++z) {
3366 uint64_t Coeff2 = Choose(2*x - y, x-z, Overflow);
3367 uint64_t Coeff;
3368 if (LargerThan64Bits)
3369 Coeff = umul_ov(Coeff1, Coeff2, Overflow);
3370 else
3371 Coeff = Coeff1*Coeff2;
3372 const SCEV *CoeffTerm = getConstant(Ty, Coeff);
3373 const SCEV *Term1 = AddRec->getOperand(y-z);
3374 const SCEV *Term2 = OtherAddRec->getOperand(z);
3375 SumOps.push_back(getMulExpr(CoeffTerm, Term1, Term2,
3376 SCEV::FlagAnyWrap, Depth + 1));
3377 }
3378 }
3379 if (SumOps.empty())
3380 SumOps.push_back(getZero(Ty));
3381 AddRecOps.push_back(getAddExpr(SumOps, SCEV::FlagAnyWrap, Depth + 1));
3382 }
3383 if (!Overflow) {
3384 const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRec->getLoop(),
3386 if (Ops.size() == 2) return NewAddRec;
3387 Ops[Idx] = NewAddRec;
3388 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
3389 OpsModified = true;
3390 AddRec = dyn_cast<SCEVAddRecExpr>(NewAddRec);
3391 if (!AddRec)
3392 break;
3393 }
3394 }
3395 if (OpsModified)
3396 return getMulExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
3397
3398 // Otherwise couldn't fold anything into this recurrence. Move onto the
3399 // next one.
3400 }
3401
3402 // Okay, it looks like we really DO need an mul expr. Check to see if we
3403 // already have one, otherwise create a new one.
3404 return getOrCreateMulExpr(Ops, ComputeFlags(Ops));
3405}
3406
3407/// Represents an unsigned remainder expression based on unsigned division.
3409 const SCEV *RHS) {
3410 assert(getEffectiveSCEVType(LHS->getType()) ==
3411 getEffectiveSCEVType(RHS->getType()) &&
3412 "SCEVURemExpr operand types don't match!");
3413
3414 // Short-circuit easy cases
3415 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
3416 // If constant is one, the result is trivial
3417 if (RHSC->getValue()->isOne())
3418 return getZero(LHS->getType()); // X urem 1 --> 0
3419
3420 // If constant is a power of two, fold into a zext(trunc(LHS)).
3421 if (RHSC->getAPInt().isPowerOf2()) {
3422 Type *FullTy = LHS->getType();
3423 Type *TruncTy =
3424 IntegerType::get(getContext(), RHSC->getAPInt().logBase2());
3425 return getZeroExtendExpr(getTruncateExpr(LHS, TruncTy), FullTy);
3426 }
3427 }
3428
3429 // Fallback to %a == %x urem %y == %x -<nuw> ((%x udiv %y) *<nuw> %y)
3430 const SCEV *UDiv = getUDivExpr(LHS, RHS);
3431 const SCEV *Mult = getMulExpr(UDiv, RHS, SCEV::FlagNUW);
3432 return getMinusSCEV(LHS, Mult, SCEV::FlagNUW);
3433}
3434
3435/// Get a canonical unsigned division expression, or something simpler if
3436/// possible.
3438 const SCEV *RHS) {
3439 assert(!LHS->getType()->isPointerTy() &&
3440 "SCEVUDivExpr operand can't be pointer!");
3441 assert(LHS->getType() == RHS->getType() &&
3442 "SCEVUDivExpr operand types don't match!");
3443
3445 ID.AddInteger(scUDivExpr);
3446 ID.AddPointer(LHS);
3447 ID.AddPointer(RHS);
3448 void *IP = nullptr;
3449 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
3450 return S;
3451
3452 // 0 udiv Y == 0
3453 if (match(LHS, m_scev_Zero()))
3454 return LHS;
3455
3456 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
3457 if (RHSC->getValue()->isOne())
3458 return LHS; // X udiv 1 --> x
3459 // If the denominator is zero, the result of the udiv is undefined. Don't
3460 // try to analyze it, because the resolution chosen here may differ from
3461 // the resolution chosen in other parts of the compiler.
3462 if (!RHSC->getValue()->isZero()) {
3463 // Determine if the division can be folded into the operands of
3464 // its operands.
3465 // TODO: Generalize this to non-constants by using known-bits information.
3466 Type *Ty = LHS->getType();
3467 unsigned LZ = RHSC->getAPInt().countl_zero();
3468 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
3469 // For non-power-of-two values, effectively round the value up to the
3470 // nearest power of two.
3471 if (!RHSC->getAPInt().isPowerOf2())
3472 ++MaxShiftAmt;
3473 IntegerType *ExtTy =
3474 IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt);
3475 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS))
3476 if (const SCEVConstant *Step =
3477 dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) {
3478 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
3479 const APInt &StepInt = Step->getAPInt();
3480 const APInt &DivInt = RHSC->getAPInt();
3481 if (!StepInt.urem(DivInt) &&
3482 getZeroExtendExpr(AR, ExtTy) ==
3483 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
3484 getZeroExtendExpr(Step, ExtTy),
3485 AR->getLoop(), SCEV::FlagAnyWrap)) {
3487 for (const SCEV *Op : AR->operands())
3488 Operands.push_back(getUDivExpr(Op, RHS));
3489 return getAddRecExpr(Operands, AR->getLoop(), SCEV::FlagNW);
3490 }
3491 /// Get a canonical UDivExpr for a recurrence.
3492 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
3493 // We can currently only fold X%N if X is constant.
3494 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(AR->getStart());
3495 if (StartC && !DivInt.urem(StepInt) &&
3496 getZeroExtendExpr(AR, ExtTy) ==
3497 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
3498 getZeroExtendExpr(Step, ExtTy),
3499 AR->getLoop(), SCEV::FlagAnyWrap)) {
3500 const APInt &StartInt = StartC->getAPInt();
3501 const APInt &StartRem = StartInt.urem(StepInt);
3502 if (StartRem != 0) {
3503 const SCEV *NewLHS =
3504 getAddRecExpr(getConstant(StartInt - StartRem), Step,
3505 AR->getLoop(), SCEV::FlagNW);
3506 if (LHS != NewLHS) {
3507 LHS = NewLHS;
3508
3509 // Reset the ID to include the new LHS, and check if it is
3510 // already cached.
3511 ID.clear();
3512 ID.AddInteger(scUDivExpr);
3513 ID.AddPointer(LHS);
3514 ID.AddPointer(RHS);
3515 IP = nullptr;
3516 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
3517 return S;
3518 }
3519 }
3520 }
3521 }
3522 // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
3523 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) {
3525 for (const SCEV *Op : M->operands())
3526 Operands.push_back(getZeroExtendExpr(Op, ExtTy));
3527 if (getZeroExtendExpr(M, ExtTy) == getMulExpr(Operands))
3528 // Find an operand that's safely divisible.
3529 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
3530 const SCEV *Op = M->getOperand(i);
3531 const SCEV *Div = getUDivExpr(Op, RHSC);
3532 if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) {
3533 Operands = SmallVector<const SCEV *, 4>(M->operands());
3534 Operands[i] = Div;
3535 return getMulExpr(Operands);
3536 }
3537 }
3538 }
3539
3540 // (A/B)/C --> A/(B*C) if safe and B*C can be folded.
3541 if (const SCEVUDivExpr *OtherDiv = dyn_cast<SCEVUDivExpr>(LHS)) {
3542 if (auto *DivisorConstant =
3543 dyn_cast<SCEVConstant>(OtherDiv->getRHS())) {
3544 bool Overflow = false;
3545 APInt NewRHS =
3546 DivisorConstant->getAPInt().umul_ov(RHSC->getAPInt(), Overflow);
3547 if (Overflow) {
3548 return getConstant(RHSC->getType(), 0, false);
3549 }
3550 return getUDivExpr(OtherDiv->getLHS(), getConstant(NewRHS));
3551 }
3552 }
3553
3554 // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded.
3555 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) {
3557 for (const SCEV *Op : A->operands())
3558 Operands.push_back(getZeroExtendExpr(Op, ExtTy));
3559 if (getZeroExtendExpr(A, ExtTy) == getAddExpr(Operands)) {
3560 Operands.clear();
3561 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
3562 const SCEV *Op = getUDivExpr(A->getOperand(i), RHS);
3563 if (isa<SCEVUDivExpr>(Op) ||
3564 getMulExpr(Op, RHS) != A->getOperand(i))
3565 break;
3566 Operands.push_back(Op);
3567 }
3568 if (Operands.size() == A->getNumOperands())
3569 return getAddExpr(Operands);
3570 }
3571 }
3572
3573 // Fold if both operands are constant.
3574 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS))
3575 return getConstant(LHSC->getAPInt().udiv(RHSC->getAPInt()));
3576 }
3577 }
3578
3579 // ((-C + (C smax %x)) /u %x) evaluates to zero, for any positive constant C.
3580 const APInt *NegC, *C;
3581 if (match(LHS,
3584 NegC->isNegative() && !NegC->isMinSignedValue() && *C == -*NegC)
3585 return getZero(LHS->getType());
3586
3587 // TODO: Generalize to handle any common factors.
3588 // udiv (mul nuw a, vscale), (mul nuw b, vscale) --> udiv a, b
3589 const SCEV *NewLHS, *NewRHS;
3590 if (match(LHS, m_scev_c_NUWMul(m_SCEV(NewLHS), m_SCEVVScale())) &&
3591 match(RHS, m_scev_c_NUWMul(m_SCEV(NewRHS), m_SCEVVScale())))
3592 return getUDivExpr(NewLHS, NewRHS);
3593
3594 // The Insertion Point (IP) might be invalid by now (due to UniqueSCEVs
3595 // changes). Make sure we get a new one.
3596 IP = nullptr;
3597 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
3598 SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator),
3599 LHS, RHS);
3600 UniqueSCEVs.InsertNode(S, IP);
3601 registerUser(S, {LHS, RHS});
3602 return S;
3603}
3604
3605APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2) {
3606 APInt A = C1->getAPInt().abs();
3607 APInt B = C2->getAPInt().abs();
3608 uint32_t ABW = A.getBitWidth();
3609 uint32_t BBW = B.getBitWidth();
3610
3611 if (ABW > BBW)
3612 B = B.zext(ABW);
3613 else if (ABW < BBW)
3614 A = A.zext(BBW);
3615
3616 return APIntOps::GreatestCommonDivisor(std::move(A), std::move(B));
3617}
3618
3619/// Get a canonical unsigned division expression, or something simpler if
3620/// possible. There is no representation for an exact udiv in SCEV IR, but we
3621/// can attempt to remove factors from the LHS and RHS. We can't do this when
3622/// it's not exact because the udiv may be clearing bits.
3624 const SCEV *RHS) {
3625 // TODO: we could try to find factors in all sorts of things, but for now we
3626 // just deal with u/exact (multiply, constant). See SCEVDivision towards the
3627 // end of this file for inspiration.
3628
3630 if (!Mul || !Mul->hasNoUnsignedWrap())
3631 return getUDivExpr(LHS, RHS);
3632
3633 if (const SCEVConstant *RHSCst = dyn_cast<SCEVConstant>(RHS)) {
3634 // If the mulexpr multiplies by a constant, then that constant must be the
3635 // first element of the mulexpr.
3636 if (const auto *LHSCst = dyn_cast<SCEVConstant>(Mul->getOperand(0))) {
3637 if (LHSCst == RHSCst) {
3638 SmallVector<const SCEV *, 2> Operands(drop_begin(Mul->operands()));
3639 return getMulExpr(Operands);
3640 }
3641
3642 // We can't just assume that LHSCst divides RHSCst cleanly, it could be
3643 // that there's a factor provided by one of the other terms. We need to
3644 // check.
3645 APInt Factor = gcd(LHSCst, RHSCst);
3646 if (!Factor.isIntN(1)) {
3647 LHSCst =
3648 cast<SCEVConstant>(getConstant(LHSCst->getAPInt().udiv(Factor)));
3649 RHSCst =
3650 cast<SCEVConstant>(getConstant(RHSCst->getAPInt().udiv(Factor)));
3652 Operands.push_back(LHSCst);
3653 append_range(Operands, Mul->operands().drop_front());
3654 LHS = getMulExpr(Operands);
3655 RHS = RHSCst;
3657 if (!Mul)
3658 return getUDivExactExpr(LHS, RHS);
3659 }
3660 }
3661 }
3662
3663 for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
3664 if (Mul->getOperand(i) == RHS) {
3666 append_range(Operands, Mul->operands().take_front(i));
3667 append_range(Operands, Mul->operands().drop_front(i + 1));
3668 return getMulExpr(Operands);
3669 }
3670 }
3671
3672 return getUDivExpr(LHS, RHS);
3673}
3674
3675/// Get an add recurrence expression for the specified loop. Simplify the
3676/// expression as much as possible.
3677const SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start, const SCEV *Step,
3678 const Loop *L,
3679 SCEV::NoWrapFlags Flags) {
3681 Operands.push_back(Start);
3682 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step))
3683 if (StepChrec->getLoop() == L) {
3684 append_range(Operands, StepChrec->operands());
3685 return getAddRecExpr(Operands, L, maskFlags(Flags, SCEV::FlagNW));
3686 }
3687
3688 Operands.push_back(Step);
3689 return getAddRecExpr(Operands, L, Flags);
3690}
3691
3692/// Get an add recurrence expression for the specified loop. Simplify the
3693/// expression as much as possible.
3694const SCEV *
3696 const Loop *L, SCEV::NoWrapFlags Flags) {
3697 if (Operands.size() == 1) return Operands[0];
3698#ifndef NDEBUG
3699 Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
3700 for (const SCEV *Op : llvm::drop_begin(Operands)) {
3701 assert(getEffectiveSCEVType(Op->getType()) == ETy &&
3702 "SCEVAddRecExpr operand types don't match!");
3703 assert(!Op->getType()->isPointerTy() && "Step must be integer");
3704 }
3705 for (const SCEV *Op : Operands)
3707 "SCEVAddRecExpr operand is not available at loop entry!");
3708#endif
3709
3710 if (Operands.back()->isZero()) {
3711 Operands.pop_back();
3712 return getAddRecExpr(Operands, L, SCEV::FlagAnyWrap); // {X,+,0} --> X
3713 }
3714
3715 // It's tempting to want to call getConstantMaxBackedgeTakenCount count here and
3716 // use that information to infer NUW and NSW flags. However, computing a
3717 // BE count requires calling getAddRecExpr, so we may not yet have a
3718 // meaningful BE count at this point (and if we don't, we'd be stuck
3719 // with a SCEVCouldNotCompute as the cached BE count).
3720
3721 Flags = StrengthenNoWrapFlags(this, scAddRecExpr, Operands, Flags);
3722
3723 // Canonicalize nested AddRecs in by nesting them in order of loop depth.
3724 if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Operands[0])) {
3725 const Loop *NestedLoop = NestedAR->getLoop();
3726 if (L->contains(NestedLoop)
3727 ? (L->getLoopDepth() < NestedLoop->getLoopDepth())
3728 : (!NestedLoop->contains(L) &&
3729 DT.dominates(L->getHeader(), NestedLoop->getHeader()))) {
3730 SmallVector<const SCEV *, 4> NestedOperands(NestedAR->operands());
3731 Operands[0] = NestedAR->getStart();
3732 // AddRecs require their operands be loop-invariant with respect to their
3733 // loops. Don't perform this transformation if it would break this
3734 // requirement.
3735 bool AllInvariant = all_of(
3736 Operands, [&](const SCEV *Op) { return isLoopInvariant(Op, L); });
3737
3738 if (AllInvariant) {
3739 // Create a recurrence for the outer loop with the same step size.
3740 //
3741 // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the
3742 // inner recurrence has the same property.
3743 SCEV::NoWrapFlags OuterFlags =
3744 maskFlags(Flags, SCEV::FlagNW | NestedAR->getNoWrapFlags());
3745
3746 NestedOperands[0] = getAddRecExpr(Operands, L, OuterFlags);
3747 AllInvariant = all_of(NestedOperands, [&](const SCEV *Op) {
3748 return isLoopInvariant(Op, NestedLoop);
3749 });
3750
3751 if (AllInvariant) {
3752 // Ok, both add recurrences are valid after the transformation.
3753 //
3754 // The inner recurrence keeps its NW flag but only keeps NUW/NSW if
3755 // the outer recurrence has the same property.
3756 SCEV::NoWrapFlags InnerFlags =
3757 maskFlags(NestedAR->getNoWrapFlags(), SCEV::FlagNW | Flags);
3758 return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags);
3759 }
3760 }
3761 // Reset Operands to its original state.
3762 Operands[0] = NestedAR;
3763 }
3764 }
3765
3766 // Okay, it looks like we really DO need an addrec expr. Check to see if we
3767 // already have one, otherwise create a new one.
3768 return getOrCreateAddRecExpr(Operands, L, Flags);
3769}
3770
3771const SCEV *
3773 const SmallVectorImpl<const SCEV *> &IndexExprs) {
3774 const SCEV *BaseExpr = getSCEV(GEP->getPointerOperand());
3775 // getSCEV(Base)->getType() has the same address space as Base->getType()
3776 // because SCEV::getType() preserves the address space.
3777 Type *IntIdxTy = getEffectiveSCEVType(BaseExpr->getType());
3778 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
3779 if (NW != GEPNoWrapFlags::none()) {
3780 // We'd like to propagate flags from the IR to the corresponding SCEV nodes,
3781 // but to do that, we have to ensure that said flag is valid in the entire
3782 // defined scope of the SCEV.
3783 // TODO: non-instructions have global scope. We might be able to prove
3784 // some global scope cases
3785 auto *GEPI = dyn_cast<Instruction>(GEP);
3786 if (!GEPI || !isSCEVExprNeverPoison(GEPI))
3787 NW = GEPNoWrapFlags::none();
3788 }
3789
3791 if (NW.hasNoUnsignedSignedWrap())
3792 OffsetWrap = setFlags(OffsetWrap, SCEV::FlagNSW);
3793 if (NW.hasNoUnsignedWrap())
3794 OffsetWrap = setFlags(OffsetWrap, SCEV::FlagNUW);
3795
3796 Type *CurTy = GEP->getType();
3797 bool FirstIter = true;
3799 for (const SCEV *IndexExpr : IndexExprs) {
3800 // Compute the (potentially symbolic) offset in bytes for this index.
3801 if (StructType *STy = dyn_cast<StructType>(CurTy)) {
3802 // For a struct, add the member offset.
3803 ConstantInt *Index = cast<SCEVConstant>(IndexExpr)->getValue();
3804 unsigned FieldNo = Index->getZExtValue();
3805 const SCEV *FieldOffset = getOffsetOfExpr(IntIdxTy, STy, FieldNo);
3806 Offsets.push_back(FieldOffset);
3807
3808 // Update CurTy to the type of the field at Index.
3809 CurTy = STy->getTypeAtIndex(Index);
3810 } else {
3811 // Update CurTy to its element type.
3812 if (FirstIter) {
3813 assert(isa<PointerType>(CurTy) &&
3814 "The first index of a GEP indexes a pointer");
3815 CurTy = GEP->getSourceElementType();
3816 FirstIter = false;
3817 } else {
3819 }
3820 // For an array, add the element offset, explicitly scaled.
3821 const SCEV *ElementSize = getSizeOfExpr(IntIdxTy, CurTy);
3822 // Getelementptr indices are signed.
3823 IndexExpr = getTruncateOrSignExtend(IndexExpr, IntIdxTy);
3824
3825 // Multiply the index by the element size to compute the element offset.
3826 const SCEV *LocalOffset = getMulExpr(IndexExpr, ElementSize, OffsetWrap);
3827 Offsets.push_back(LocalOffset);
3828 }
3829 }
3830
3831 // Handle degenerate case of GEP without offsets.
3832 if (Offsets.empty())
3833 return BaseExpr;
3834
3835 // Add the offsets together, assuming nsw if inbounds.
3836 const SCEV *Offset = getAddExpr(Offsets, OffsetWrap);
3837 // Add the base address and the offset. We cannot use the nsw flag, as the
3838 // base address is unsigned. However, if we know that the offset is
3839 // non-negative, we can use nuw.
3840 bool NUW = NW.hasNoUnsignedWrap() ||
3843 auto *GEPExpr = getAddExpr(BaseExpr, Offset, BaseWrap);
3844 assert(BaseExpr->getType() == GEPExpr->getType() &&
3845 "GEP should not change type mid-flight.");
3846 return GEPExpr;
3847}
3848
3849SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
3852 ID.AddInteger(SCEVType);
3853 for (const SCEV *Op : Ops)
3854 ID.AddPointer(Op);
3855 void *IP = nullptr;
3856 return UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
3857}
3858
3859const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) {
3861 return getSMaxExpr(Op, getNegativeSCEV(Op, Flags));
3862}
3863
3866 assert(SCEVMinMaxExpr::isMinMaxType(Kind) && "Not a SCEVMinMaxExpr!");
3867 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
3868 if (Ops.size() == 1) return Ops[0];
3869#ifndef NDEBUG
3870 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
3871 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
3872 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
3873 "Operand types don't match!");
3874 assert(Ops[0]->getType()->isPointerTy() ==
3875 Ops[i]->getType()->isPointerTy() &&
3876 "min/max should be consistently pointerish");
3877 }
3878#endif
3879
3880 bool IsSigned = Kind == scSMaxExpr || Kind == scSMinExpr;
3881 bool IsMax = Kind == scSMaxExpr || Kind == scUMaxExpr;
3882
3883 const SCEV *Folded = constantFoldAndGroupOps(
3884 *this, LI, DT, Ops,
3885 [&](const APInt &C1, const APInt &C2) {
3886 switch (Kind) {
3887 case scSMaxExpr:
3888 return APIntOps::smax(C1, C2);
3889 case scSMinExpr:
3890 return APIntOps::smin(C1, C2);
3891 case scUMaxExpr:
3892 return APIntOps::umax(C1, C2);
3893 case scUMinExpr:
3894 return APIntOps::umin(C1, C2);
3895 default:
3896 llvm_unreachable("Unknown SCEV min/max opcode");
3897 }
3898 },
3899 [&](const APInt &C) {
3900 // identity
3901 if (IsMax)
3902 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3903 else
3904 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3905 },
3906 [&](const APInt &C) {
3907 // absorber
3908 if (IsMax)
3909 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3910 else
3911 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3912 });
3913 if (Folded)
3914 return Folded;
3915
3916 // Check if we have created the same expression before.
3917 if (const SCEV *S = findExistingSCEVInCache(Kind, Ops)) {
3918 return S;
3919 }
3920
3921 // Find the first operation of the same kind
3922 unsigned Idx = 0;
3923 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < Kind)
3924 ++Idx;
3925
3926 // Check to see if one of the operands is of the same kind. If so, expand its
3927 // operands onto our operand list, and recurse to simplify.
3928 if (Idx < Ops.size()) {
3929 bool DeletedAny = false;
3930 while (Ops[Idx]->getSCEVType() == Kind) {
3931 const SCEVMinMaxExpr *SMME = cast<SCEVMinMaxExpr>(Ops[Idx]);
3932 Ops.erase(Ops.begin()+Idx);
3933 append_range(Ops, SMME->operands());
3934 DeletedAny = true;
3935 }
3936
3937 if (DeletedAny)
3938 return getMinMaxExpr(Kind, Ops);
3939 }
3940
3941 // Okay, check to see if the same value occurs in the operand list twice. If
3942 // so, delete one. Since we sorted the list, these values are required to
3943 // be adjacent.
3948 llvm::CmpInst::Predicate FirstPred = IsMax ? GEPred : LEPred;
3949 llvm::CmpInst::Predicate SecondPred = IsMax ? LEPred : GEPred;
3950 for (unsigned i = 0, e = Ops.size() - 1; i != e; ++i) {
3951 if (Ops[i] == Ops[i + 1] ||
3952 isKnownViaNonRecursiveReasoning(FirstPred, Ops[i], Ops[i + 1])) {
3953 // X op Y op Y --> X op Y
3954 // X op Y --> X, if we know X, Y are ordered appropriately
3955 Ops.erase(Ops.begin() + i + 1, Ops.begin() + i + 2);
3956 --i;
3957 --e;
3958 } else if (isKnownViaNonRecursiveReasoning(SecondPred, Ops[i],
3959 Ops[i + 1])) {
3960 // X op Y --> Y, if we know X, Y are ordered appropriately
3961 Ops.erase(Ops.begin() + i, Ops.begin() + i + 1);
3962 --i;
3963 --e;
3964 }
3965 }
3966
3967 if (Ops.size() == 1) return Ops[0];
3968
3969 assert(!Ops.empty() && "Reduced smax down to nothing!");
3970
3971 // Okay, it looks like we really DO need an expr. Check to see if we
3972 // already have one, otherwise create a new one.
3974 ID.AddInteger(Kind);
3975 for (const SCEV *Op : Ops)
3976 ID.AddPointer(Op);
3977 void *IP = nullptr;
3978 const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
3979 if (ExistingSCEV)
3980 return ExistingSCEV;
3981 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
3983 SCEV *S = new (SCEVAllocator)
3984 SCEVMinMaxExpr(ID.Intern(SCEVAllocator), Kind, O, Ops.size());
3985
3986 UniqueSCEVs.InsertNode(S, IP);
3987 registerUser(S, Ops);
3988 return S;
3989}
3990
3991namespace {
3992
3993class SCEVSequentialMinMaxDeduplicatingVisitor final
3994 : public SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor,
3995 std::optional<const SCEV *>> {
3996 using RetVal = std::optional<const SCEV *>;
3998
3999 ScalarEvolution &SE;
4000 const SCEVTypes RootKind; // Must be a sequential min/max expression.
4001 const SCEVTypes NonSequentialRootKind; // Non-sequential variant of RootKind.
4003
4004 bool canRecurseInto(SCEVTypes Kind) const {
4005 // We can only recurse into the SCEV expression of the same effective type
4006 // as the type of our root SCEV expression.
4007 return RootKind == Kind || NonSequentialRootKind == Kind;
4008 };
4009
4010 RetVal visitAnyMinMaxExpr(const SCEV *S) {
4012 "Only for min/max expressions.");
4013 SCEVTypes Kind = S->getSCEVType();
4014
4015 if (!canRecurseInto(Kind))
4016 return S;
4017
4018 auto *NAry = cast<SCEVNAryExpr>(S);
4020 bool Changed = visit(Kind, NAry->operands(), NewOps);
4021
4022 if (!Changed)
4023 return S;
4024 if (NewOps.empty())
4025 return std::nullopt;
4026
4028 ? SE.getSequentialMinMaxExpr(Kind, NewOps)
4029 : SE.getMinMaxExpr(Kind, NewOps);
4030 }
4031
4032 RetVal visit(const SCEV *S) {
4033 // Has the whole operand been seen already?
4034 if (!SeenOps.insert(S).second)
4035 return std::nullopt;
4036 return Base::visit(S);
4037 }
4038
4039public:
4040 SCEVSequentialMinMaxDeduplicatingVisitor(ScalarEvolution &SE,
4041 SCEVTypes RootKind)
4042 : SE(SE), RootKind(RootKind),
4043 NonSequentialRootKind(
4044 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
4045 RootKind)) {}
4046
4047 bool /*Changed*/ visit(SCEVTypes Kind, ArrayRef<const SCEV *> OrigOps,
4048 SmallVectorImpl<const SCEV *> &NewOps) {
4049 bool Changed = false;
4051 Ops.reserve(OrigOps.size());
4052
4053 for (const SCEV *Op : OrigOps) {
4054 RetVal NewOp = visit(Op);
4055 if (NewOp != Op)
4056 Changed = true;
4057 if (NewOp)
4058 Ops.emplace_back(*NewOp);
4059 }
4060
4061 if (Changed)
4062 NewOps = std::move(Ops);
4063 return Changed;
4064 }
4065
4066 RetVal visitConstant(const SCEVConstant *Constant) { return Constant; }
4067
4068 RetVal visitVScale(const SCEVVScale *VScale) { return VScale; }
4069
4070 RetVal visitPtrToIntExpr(const SCEVPtrToIntExpr *Expr) { return Expr; }
4071
4072 RetVal visitTruncateExpr(const SCEVTruncateExpr *Expr) { return Expr; }
4073
4074 RetVal visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) { return Expr; }
4075
4076 RetVal visitSignExtendExpr(const SCEVSignExtendExpr *Expr) { return Expr; }
4077
4078 RetVal visitAddExpr(const SCEVAddExpr *Expr) { return Expr; }
4079
4080 RetVal visitMulExpr(const SCEVMulExpr *Expr) { return Expr; }
4081
4082 RetVal visitUDivExpr(const SCEVUDivExpr *Expr) { return Expr; }
4083
4084 RetVal visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
4085
4086 RetVal visitSMaxExpr(const SCEVSMaxExpr *Expr) {
4087 return visitAnyMinMaxExpr(Expr);
4088 }
4089
4090 RetVal visitUMaxExpr(const SCEVUMaxExpr *Expr) {
4091 return visitAnyMinMaxExpr(Expr);
4092 }
4093
4094 RetVal visitSMinExpr(const SCEVSMinExpr *Expr) {
4095 return visitAnyMinMaxExpr(Expr);
4096 }
4097
4098 RetVal visitUMinExpr(const SCEVUMinExpr *Expr) {
4099 return visitAnyMinMaxExpr(Expr);
4100 }
4101
4102 RetVal visitSequentialUMinExpr(const SCEVSequentialUMinExpr *Expr) {
4103 return visitAnyMinMaxExpr(Expr);
4104 }
4105
4106 RetVal visitUnknown(const SCEVUnknown *Expr) { return Expr; }
4107
4108 RetVal visitCouldNotCompute(const SCEVCouldNotCompute *Expr) { return Expr; }
4109};
4110
4111} // namespace
4112
4114 switch (Kind) {
4115 case scConstant:
4116 case scVScale:
4117 case scTruncate:
4118 case scZeroExtend:
4119 case scSignExtend:
4120 case scPtrToInt:
4121 case scAddExpr:
4122 case scMulExpr:
4123 case scUDivExpr:
4124 case scAddRecExpr:
4125 case scUMaxExpr:
4126 case scSMaxExpr:
4127 case scUMinExpr:
4128 case scSMinExpr:
4129 case scUnknown:
4130 // If any operand is poison, the whole expression is poison.
4131 return true;
4133 // FIXME: if the *first* operand is poison, the whole expression is poison.
4134 return false; // Pessimistically, say that it does not propagate poison.
4135 case scCouldNotCompute:
4136 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
4137 }
4138 llvm_unreachable("Unknown SCEV kind!");
4139}
4140
4141namespace {
4142// The only way poison may be introduced in a SCEV expression is from a
4143// poison SCEVUnknown (ConstantExprs are also represented as SCEVUnknown,
4144// not SCEVConstant). Notably, nowrap flags in SCEV nodes can *not*
4145// introduce poison -- they encode guaranteed, non-speculated knowledge.
4146//
4147// Additionally, all SCEV nodes propagate poison from inputs to outputs,
4148// with the notable exception of umin_seq, where only poison from the first
4149// operand is (unconditionally) propagated.
4150struct SCEVPoisonCollector {
4151 bool LookThroughMaybePoisonBlocking;
4152 SmallPtrSet<const SCEVUnknown *, 4> MaybePoison;
4153 SCEVPoisonCollector(bool LookThroughMaybePoisonBlocking)
4154 : LookThroughMaybePoisonBlocking(LookThroughMaybePoisonBlocking) {}
4155
4156 bool follow(const SCEV *S) {
4157 if (!LookThroughMaybePoisonBlocking &&
4159 return false;
4160
4161 if (auto *SU = dyn_cast<SCEVUnknown>(S)) {
4162 if (!isGuaranteedNotToBePoison(SU->getValue()))
4163 MaybePoison.insert(SU);
4164 }
4165 return true;
4166 }
4167 bool isDone() const { return false; }
4168};
4169} // namespace
4170
4171/// Return true if V is poison given that AssumedPoison is already poison.
4172static bool impliesPoison(const SCEV *AssumedPoison, const SCEV *S) {
4173 // First collect all SCEVs that might result in AssumedPoison to be poison.
4174 // We need to look through potentially poison-blocking operations here,
4175 // because we want to find all SCEVs that *might* result in poison, not only
4176 // those that are *required* to.
4177 SCEVPoisonCollector PC1(/* LookThroughMaybePoisonBlocking */ true);
4178 visitAll(AssumedPoison, PC1);
4179
4180 // AssumedPoison is never poison. As the assumption is false, the implication
4181 // is true. Don't bother walking the other SCEV in this case.
4182 if (PC1.MaybePoison.empty())
4183 return true;
4184
4185 // Collect all SCEVs in S that, if poison, *will* result in S being poison
4186 // as well. We cannot look through potentially poison-blocking operations
4187 // here, as their arguments only *may* make the result poison.
4188 SCEVPoisonCollector PC2(/* LookThroughMaybePoisonBlocking */ false);
4189 visitAll(S, PC2);
4190
4191 // Make sure that no matter which SCEV in PC1.MaybePoison is actually poison,
4192 // it will also make S poison by being part of PC2.MaybePoison.
4193 return llvm::set_is_subset(PC1.MaybePoison, PC2.MaybePoison);
4194}
4195
4197 SmallPtrSetImpl<const Value *> &Result, const SCEV *S) {
4198 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ false);
4199 visitAll(S, PC);
4200 for (const SCEVUnknown *SU : PC.MaybePoison)
4201 Result.insert(SU->getValue());
4202}
4203
4205 const SCEV *S, Instruction *I,
4206 SmallVectorImpl<Instruction *> &DropPoisonGeneratingInsts) {
4207 // If the instruction cannot be poison, it's always safe to reuse.
4209 return true;
4210
4211 // Otherwise, it is possible that I is more poisonous that S. Collect the
4212 // poison-contributors of S, and then check whether I has any additional
4213 // poison-contributors. Poison that is contributed through poison-generating
4214 // flags is handled by dropping those flags instead.
4216 getPoisonGeneratingValues(PoisonVals, S);
4217
4218 SmallVector<Value *> Worklist;
4220 Worklist.push_back(I);
4221 while (!Worklist.empty()) {
4222 Value *V = Worklist.pop_back_val();
4223 if (!Visited.insert(V).second)
4224 continue;
4225
4226 // Avoid walking large instruction graphs.
4227 if (Visited.size() > 16)
4228 return false;
4229
4230 // Either the value can't be poison, or the S would also be poison if it
4231 // is.
4232 if (PoisonVals.contains(V) || ::isGuaranteedNotToBePoison(V))
4233 continue;
4234
4235 auto *I = dyn_cast<Instruction>(V);
4236 if (!I)
4237 return false;
4238
4239 // Disjoint or instructions are interpreted as adds by SCEV. However, we
4240 // can't replace an arbitrary add with disjoint or, even if we drop the
4241 // flag. We would need to convert the or into an add.
4242 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(I))
4243 if (PDI->isDisjoint())
4244 return false;
4245
4246 // FIXME: Ignore vscale, even though it technically could be poison. Do this
4247 // because SCEV currently assumes it can't be poison. Remove this special
4248 // case once we proper model when vscale can be poison.
4249 if (auto *II = dyn_cast<IntrinsicInst>(I);
4250 II && II->getIntrinsicID() == Intrinsic::vscale)
4251 continue;
4252
4253 if (canCreatePoison(cast<Operator>(I), /*ConsiderFlagsAndMetadata*/ false))
4254 return false;
4255
4256 // If the instruction can't create poison, we can recurse to its operands.
4257 if (I->hasPoisonGeneratingAnnotations())
4258 DropPoisonGeneratingInsts.push_back(I);
4259
4260 llvm::append_range(Worklist, I->operands());
4261 }
4262 return true;
4263}
4264
4265const SCEV *
4268 assert(SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) &&
4269 "Not a SCEVSequentialMinMaxExpr!");
4270 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
4271 if (Ops.size() == 1)
4272 return Ops[0];
4273#ifndef NDEBUG
4274 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
4275 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4276 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
4277 "Operand types don't match!");
4278 assert(Ops[0]->getType()->isPointerTy() ==
4279 Ops[i]->getType()->isPointerTy() &&
4280 "min/max should be consistently pointerish");
4281 }
4282#endif
4283
4284 // Note that SCEVSequentialMinMaxExpr is *NOT* commutative,
4285 // so we can *NOT* do any kind of sorting of the expressions!
4286
4287 // Check if we have created the same expression before.
4288 if (const SCEV *S = findExistingSCEVInCache(Kind, Ops))
4289 return S;
4290
4291 // FIXME: there are *some* simplifications that we can do here.
4292
4293 // Keep only the first instance of an operand.
4294 {
4295 SCEVSequentialMinMaxDeduplicatingVisitor Deduplicator(*this, Kind);
4296 bool Changed = Deduplicator.visit(Kind, Ops, Ops);
4297 if (Changed)
4298 return getSequentialMinMaxExpr(Kind, Ops);
4299 }
4300
4301 // Check to see if one of the operands is of the same kind. If so, expand its
4302 // operands onto our operand list, and recurse to simplify.
4303 {
4304 unsigned Idx = 0;
4305 bool DeletedAny = false;
4306 while (Idx < Ops.size()) {
4307 if (Ops[Idx]->getSCEVType() != Kind) {
4308 ++Idx;
4309 continue;
4310 }
4311 const auto *SMME = cast<SCEVSequentialMinMaxExpr>(Ops[Idx]);
4312 Ops.erase(Ops.begin() + Idx);
4313 Ops.insert(Ops.begin() + Idx, SMME->operands().begin(),
4314 SMME->operands().end());
4315 DeletedAny = true;
4316 }
4317
4318 if (DeletedAny)
4319 return getSequentialMinMaxExpr(Kind, Ops);
4320 }
4321
4322 const SCEV *SaturationPoint;
4324 switch (Kind) {
4326 SaturationPoint = getZero(Ops[0]->getType());
4327 Pred = ICmpInst::ICMP_ULE;
4328 break;
4329 default:
4330 llvm_unreachable("Not a sequential min/max type.");
4331 }
4332
4333 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4334 if (!isGuaranteedNotToCauseUB(Ops[i]))
4335 continue;
4336 // We can replace %x umin_seq %y with %x umin %y if either:
4337 // * %y being poison implies %x is also poison.
4338 // * %x cannot be the saturating value (e.g. zero for umin).
4339 if (::impliesPoison(Ops[i], Ops[i - 1]) ||
4340 isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_NE, Ops[i - 1],
4341 SaturationPoint)) {
4342 SmallVector<const SCEV *> SeqOps = {Ops[i - 1], Ops[i]};
4343 Ops[i - 1] = getMinMaxExpr(
4345 SeqOps);
4346 Ops.erase(Ops.begin() + i);
4347 return getSequentialMinMaxExpr(Kind, Ops);
4348 }
4349 // Fold %x umin_seq %y to %x if %x ule %y.
4350 // TODO: We might be able to prove the predicate for a later operand.
4351 if (isKnownViaNonRecursiveReasoning(Pred, Ops[i - 1], Ops[i])) {
4352 Ops.erase(Ops.begin() + i);
4353 return getSequentialMinMaxExpr(Kind, Ops);
4354 }
4355 }
4356
4357 // Okay, it looks like we really DO need an expr. Check to see if we
4358 // already have one, otherwise create a new one.
4360 ID.AddInteger(Kind);
4361 for (const SCEV *Op : Ops)
4362 ID.AddPointer(Op);
4363 void *IP = nullptr;
4364 const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
4365 if (ExistingSCEV)
4366 return ExistingSCEV;
4367
4368 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
4370 SCEV *S = new (SCEVAllocator)
4371 SCEVSequentialMinMaxExpr(ID.Intern(SCEVAllocator), Kind, O, Ops.size());
4372
4373 UniqueSCEVs.InsertNode(S, IP);
4374 registerUser(S, Ops);
4375 return S;
4376}
4377
4378const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS, const SCEV *RHS) {
4379 SmallVector<const SCEV *, 2> Ops = {LHS, RHS};
4380 return getSMaxExpr(Ops);
4381}
4382
4386
4387const SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS, const SCEV *RHS) {
4388 SmallVector<const SCEV *, 2> Ops = {LHS, RHS};
4389 return getUMaxExpr(Ops);
4390}
4391
4395
4397 const SCEV *RHS) {
4398 SmallVector<const SCEV *, 2> Ops = { LHS, RHS };
4399 return getSMinExpr(Ops);
4400}
4401
4405
4406const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS, const SCEV *RHS,
4407 bool Sequential) {
4408 SmallVector<const SCEV *, 2> Ops = { LHS, RHS };
4409 return getUMinExpr(Ops, Sequential);
4410}
4411
4417
4418const SCEV *
4420 const SCEV *Res = getConstant(IntTy, Size.getKnownMinValue());
4421 if (Size.isScalable())
4422 Res = getMulExpr(Res, getVScale(IntTy));
4423 return Res;
4424}
4425
4427 return getSizeOfExpr(IntTy, getDataLayout().getTypeAllocSize(AllocTy));
4428}
4429
4431 return getSizeOfExpr(IntTy, getDataLayout().getTypeStoreSize(StoreTy));
4432}
4433
4435 StructType *STy,
4436 unsigned FieldNo) {
4437 // We can bypass creating a target-independent constant expression and then
4438 // folding it back into a ConstantInt. This is just a compile-time
4439 // optimization.
4440 const StructLayout *SL = getDataLayout().getStructLayout(STy);
4441 assert(!SL->getSizeInBits().isScalable() &&
4442 "Cannot get offset for structure containing scalable vector types");
4443 return getConstant(IntTy, SL->getElementOffset(FieldNo));
4444}
4445
4447 // Don't attempt to do anything other than create a SCEVUnknown object
4448 // here. createSCEV only calls getUnknown after checking for all other
4449 // interesting possibilities, and any other code that calls getUnknown
4450 // is doing so in order to hide a value from SCEV canonicalization.
4451
4453 ID.AddInteger(scUnknown);
4454 ID.AddPointer(V);
4455 void *IP = nullptr;
4456 if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) {
4457 assert(cast<SCEVUnknown>(S)->getValue() == V &&
4458 "Stale SCEVUnknown in uniquing map!");
4459 return S;
4460 }
4461 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(SCEVAllocator), V, this,
4462 FirstUnknown);
4463 FirstUnknown = cast<SCEVUnknown>(S);
4464 UniqueSCEVs.InsertNode(S, IP);
4465 return S;
4466}
4467
4468//===----------------------------------------------------------------------===//
4469// Basic SCEV Analysis and PHI Idiom Recognition Code
4470//
4471
4472/// Test if values of the given type are analyzable within the SCEV
4473/// framework. This primarily includes integer types, and it can optionally
4474/// include pointer types if the ScalarEvolution class has access to
4475/// target-specific information.
4477 // Integers and pointers are always SCEVable.
4478 return Ty->isIntOrPtrTy();
4479}
4480
4481/// Return the size in bits of the specified type, for which isSCEVable must
4482/// return true.
4484 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4485 if (Ty->isPointerTy())
4487 return getDataLayout().getTypeSizeInBits(Ty);
4488}
4489
4490/// Return a type with the same bitwidth as the given type and which represents
4491/// how SCEV will treat the given type, for which isSCEVable must return
4492/// true. For pointer types, this is the pointer index sized integer type.
4494 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4495
4496 if (Ty->isIntegerTy())
4497 return Ty;
4498
4499 // The only other support type is pointer.
4500 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
4501 return getDataLayout().getIndexType(Ty);
4502}
4503
4505 return getTypeSizeInBits(T1) >= getTypeSizeInBits(T2) ? T1 : T2;
4506}
4507
4509 const SCEV *B) {
4510 /// For a valid use point to exist, the defining scope of one operand
4511 /// must dominate the other.
4512 bool PreciseA, PreciseB;
4513 auto *ScopeA = getDefiningScopeBound({A}, PreciseA);
4514 auto *ScopeB = getDefiningScopeBound({B}, PreciseB);
4515 if (!PreciseA || !PreciseB)
4516 // Can't tell.
4517 return false;
4518 return (ScopeA == ScopeB) || DT.dominates(ScopeA, ScopeB) ||
4519 DT.dominates(ScopeB, ScopeA);
4520}
4521
4523 return CouldNotCompute.get();
4524}
4525
4526bool ScalarEvolution::checkValidity(const SCEV *S) const {
4527 bool ContainsNulls = SCEVExprContains(S, [](const SCEV *S) {
4528 auto *SU = dyn_cast<SCEVUnknown>(S);
4529 return SU && SU->getValue() == nullptr;
4530 });
4531
4532 return !ContainsNulls;
4533}
4534
4536 HasRecMapType::iterator I = HasRecMap.find(S);
4537 if (I != HasRecMap.end())
4538 return I->second;
4539
4540 bool FoundAddRec =
4541 SCEVExprContains(S, [](const SCEV *S) { return isa<SCEVAddRecExpr>(S); });
4542 HasRecMap.insert({S, FoundAddRec});
4543 return FoundAddRec;
4544}
4545
4546/// Return the ValueOffsetPair set for \p S. \p S can be represented
4547/// by the value and offset from any ValueOffsetPair in the set.
4548ArrayRef<Value *> ScalarEvolution::getSCEVValues(const SCEV *S) {
4549 ExprValueMapType::iterator SI = ExprValueMap.find_as(S);
4550 if (SI == ExprValueMap.end())
4551 return {};
4552 return SI->second.getArrayRef();
4553}
4554
4555/// Erase Value from ValueExprMap and ExprValueMap. ValueExprMap.erase(V)
4556/// cannot be used separately. eraseValueFromMap should be used to remove
4557/// V from ValueExprMap and ExprValueMap at the same time.
4558void ScalarEvolution::eraseValueFromMap(Value *V) {
4559 ValueExprMapType::iterator I = ValueExprMap.find_as(V);
4560 if (I != ValueExprMap.end()) {
4561 auto EVIt = ExprValueMap.find(I->second);
4562 bool Removed = EVIt->second.remove(V);
4563 (void) Removed;
4564 assert(Removed && "Value not in ExprValueMap?");
4565 ValueExprMap.erase(I);
4566 }
4567}
4568
4569void ScalarEvolution::insertValueToMap(Value *V, const SCEV *S) {
4570 // A recursive query may have already computed the SCEV. It should be
4571 // equivalent, but may not necessarily be exactly the same, e.g. due to lazily
4572 // inferred nowrap flags.
4573 auto It = ValueExprMap.find_as(V);
4574 if (It == ValueExprMap.end()) {
4575 ValueExprMap.insert({SCEVCallbackVH(V, this), S});
4576 ExprValueMap[S].insert(V);
4577 }
4578}
4579
4580/// Return an existing SCEV if it exists, otherwise analyze the expression and
4581/// create a new one.
4583 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4584
4585 if (const SCEV *S = getExistingSCEV(V))
4586 return S;
4587 return createSCEVIter(V);
4588}
4589
4591 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4592
4593 ValueExprMapType::iterator I = ValueExprMap.find_as(V);
4594 if (I != ValueExprMap.end()) {
4595 const SCEV *S = I->second;
4596 assert(checkValidity(S) &&
4597 "existing SCEV has not been properly invalidated");
4598 return S;
4599 }
4600 return nullptr;
4601}
4602
4603/// Return a SCEV corresponding to -V = -1*V
4605 SCEV::NoWrapFlags Flags) {
4606 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
4607 return getConstant(
4608 cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue())));
4609
4610 Type *Ty = V->getType();
4611 Ty = getEffectiveSCEVType(Ty);
4612 return getMulExpr(V, getMinusOne(Ty), Flags);
4613}
4614
4615/// If Expr computes ~A, return A else return nullptr
4616static const SCEV *MatchNotExpr(const SCEV *Expr) {
4617 const SCEV *MulOp;
4618 if (match(Expr, m_scev_Add(m_scev_AllOnes(),
4619 m_scev_Mul(m_scev_AllOnes(), m_SCEV(MulOp)))))
4620 return MulOp;
4621 return nullptr;
4622}
4623
4624/// Return a SCEV corresponding to ~V = -1-V
4626 assert(!V->getType()->isPointerTy() && "Can't negate pointer");
4627
4628 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
4629 return getConstant(
4630 cast<ConstantInt>(ConstantExpr::getNot(VC->getValue())));
4631
4632 // Fold ~(u|s)(min|max)(~x, ~y) to (u|s)(max|min)(x, y)
4633 if (const SCEVMinMaxExpr *MME = dyn_cast<SCEVMinMaxExpr>(V)) {
4634 auto MatchMinMaxNegation = [&](const SCEVMinMaxExpr *MME) {
4635 SmallVector<const SCEV *, 2> MatchedOperands;
4636 for (const SCEV *Operand : MME->operands()) {
4637 const SCEV *Matched = MatchNotExpr(Operand);
4638 if (!Matched)
4639 return (const SCEV *)nullptr;
4640 MatchedOperands.push_back(Matched);
4641 }
4642 return getMinMaxExpr(SCEVMinMaxExpr::negate(MME->getSCEVType()),
4643 MatchedOperands);
4644 };
4645 if (const SCEV *Replaced = MatchMinMaxNegation(MME))
4646 return Replaced;
4647 }
4648
4649 Type *Ty = V->getType();
4650 Ty = getEffectiveSCEVType(Ty);
4651 return getMinusSCEV(getMinusOne(Ty), V);
4652}
4653
4655 assert(P->getType()->isPointerTy());
4656
4657 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(P)) {
4658 // The base of an AddRec is the first operand.
4659 SmallVector<const SCEV *> Ops{AddRec->operands()};
4660 Ops[0] = removePointerBase(Ops[0]);
4661 // Don't try to transfer nowrap flags for now. We could in some cases
4662 // (for example, if pointer operand of the AddRec is a SCEVUnknown).
4663 return getAddRecExpr(Ops, AddRec->getLoop(), SCEV::FlagAnyWrap);
4664 }
4665 if (auto *Add = dyn_cast<SCEVAddExpr>(P)) {
4666 // The base of an Add is the pointer operand.
4667 SmallVector<const SCEV *> Ops{Add->operands()};
4668 const SCEV **PtrOp = nullptr;
4669 for (const SCEV *&AddOp : Ops) {
4670 if (AddOp->getType()->isPointerTy()) {
4671 assert(!PtrOp && "Cannot have multiple pointer ops");
4672 PtrOp = &AddOp;
4673 }
4674 }
4675 *PtrOp = removePointerBase(*PtrOp);
4676 // Don't try to transfer nowrap flags for now. We could in some cases
4677 // (for example, if the pointer operand of the Add is a SCEVUnknown).
4678 return getAddExpr(Ops);
4679 }
4680 // Any other expression must be a pointer base.
4681 return getZero(P->getType());
4682}
4683
4684const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS,
4685 SCEV::NoWrapFlags Flags,
4686 unsigned Depth) {
4687 // Fast path: X - X --> 0.
4688 if (LHS == RHS)
4689 return getZero(LHS->getType());
4690
4691 // If we subtract two pointers with different pointer bases, bail.
4692 // Eventually, we're going to add an assertion to getMulExpr that we
4693 // can't multiply by a pointer.
4694 if (RHS->getType()->isPointerTy()) {
4695 if (!LHS->getType()->isPointerTy() ||
4696 getPointerBase(LHS) != getPointerBase(RHS))
4697 return getCouldNotCompute();
4698 LHS = removePointerBase(LHS);
4699 RHS = removePointerBase(RHS);
4700 }
4701
4702 // We represent LHS - RHS as LHS + (-1)*RHS. This transformation
4703 // makes it so that we cannot make much use of NUW.
4704 auto AddFlags = SCEV::FlagAnyWrap;
4705 const bool RHSIsNotMinSigned =
4707 if (hasFlags(Flags, SCEV::FlagNSW)) {
4708 // Let M be the minimum representable signed value. Then (-1)*RHS
4709 // signed-wraps if and only if RHS is M. That can happen even for
4710 // a NSW subtraction because e.g. (-1)*M signed-wraps even though
4711 // -1 - M does not. So to transfer NSW from LHS - RHS to LHS +
4712 // (-1)*RHS, we need to prove that RHS != M.
4713 //
4714 // If LHS is non-negative and we know that LHS - RHS does not
4715 // signed-wrap, then RHS cannot be M. So we can rule out signed-wrap
4716 // either by proving that RHS > M or that LHS >= 0.
4717 if (RHSIsNotMinSigned || isKnownNonNegative(LHS)) {
4718 AddFlags = SCEV::FlagNSW;
4719 }
4720 }
4721
4722 // FIXME: Find a correct way to transfer NSW to (-1)*M when LHS -
4723 // RHS is NSW and LHS >= 0.
4724 //
4725 // The difficulty here is that the NSW flag may have been proven
4726 // relative to a loop that is to be found in a recurrence in LHS and
4727 // not in RHS. Applying NSW to (-1)*M may then let the NSW have a
4728 // larger scope than intended.
4729 auto NegFlags = RHSIsNotMinSigned ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
4730
4731 return getAddExpr(LHS, getNegativeSCEV(RHS, NegFlags), AddFlags, Depth);
4732}
4733
4735 unsigned Depth) {
4736 Type *SrcTy = V->getType();
4737 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4738 "Cannot truncate or zero extend with non-integer arguments!");
4739 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4740 return V; // No conversion
4741 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
4742 return getTruncateExpr(V, Ty, Depth);
4743 return getZeroExtendExpr(V, Ty, Depth);
4744}
4745
4747 unsigned Depth) {
4748 Type *SrcTy = V->getType();
4749 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4750 "Cannot truncate or zero extend with non-integer arguments!");
4751 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4752 return V; // No conversion
4753 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
4754 return getTruncateExpr(V, Ty, Depth);
4755 return getSignExtendExpr(V, Ty, Depth);
4756}
4757
4758const SCEV *
4760 Type *SrcTy = V->getType();
4761 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4762 "Cannot noop or zero extend with non-integer arguments!");
4764 "getNoopOrZeroExtend cannot truncate!");
4765 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4766 return V; // No conversion
4767 return getZeroExtendExpr(V, Ty);
4768}
4769
4770const SCEV *
4772 Type *SrcTy = V->getType();
4773 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4774 "Cannot noop or sign extend with non-integer arguments!");
4776 "getNoopOrSignExtend cannot truncate!");
4777 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4778 return V; // No conversion
4779 return getSignExtendExpr(V, Ty);
4780}
4781
4782const SCEV *
4784 Type *SrcTy = V->getType();
4785 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4786 "Cannot noop or any extend with non-integer arguments!");
4788 "getNoopOrAnyExtend cannot truncate!");
4789 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4790 return V; // No conversion
4791 return getAnyExtendExpr(V, Ty);
4792}
4793
4794const SCEV *
4796 Type *SrcTy = V->getType();
4797 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4798 "Cannot truncate or noop with non-integer arguments!");
4800 "getTruncateOrNoop cannot extend!");
4801 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4802 return V; // No conversion
4803 return getTruncateExpr(V, Ty);
4804}
4805
4807 const SCEV *RHS) {
4808 const SCEV *PromotedLHS = LHS;
4809 const SCEV *PromotedRHS = RHS;
4810
4811 if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType()))
4812 PromotedRHS = getZeroExtendExpr(RHS, LHS->getType());
4813 else
4814 PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType());
4815
4816 return getUMaxExpr(PromotedLHS, PromotedRHS);
4817}
4818
4820 const SCEV *RHS,
4821 bool Sequential) {
4822 SmallVector<const SCEV *, 2> Ops = { LHS, RHS };
4823 return getUMinFromMismatchedTypes(Ops, Sequential);
4824}
4825
4826const SCEV *
4828 bool Sequential) {
4829 assert(!Ops.empty() && "At least one operand must be!");
4830 // Trivial case.
4831 if (Ops.size() == 1)
4832 return Ops[0];
4833
4834 // Find the max type first.
4835 Type *MaxType = nullptr;
4836 for (const auto *S : Ops)
4837 if (MaxType)
4838 MaxType = getWiderType(MaxType, S->getType());
4839 else
4840 MaxType = S->getType();
4841 assert(MaxType && "Failed to find maximum type!");
4842
4843 // Extend all ops to max type.
4844 SmallVector<const SCEV *, 2> PromotedOps;
4845 for (const auto *S : Ops)
4846 PromotedOps.push_back(getNoopOrZeroExtend(S, MaxType));
4847
4848 // Generate umin.
4849 return getUMinExpr(PromotedOps, Sequential);
4850}
4851
4853 // A pointer operand may evaluate to a nonpointer expression, such as null.
4854 if (!V->getType()->isPointerTy())
4855 return V;
4856
4857 while (true) {
4858 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(V)) {
4859 V = AddRec->getStart();
4860 } else if (auto *Add = dyn_cast<SCEVAddExpr>(V)) {
4861 const SCEV *PtrOp = nullptr;
4862 for (const SCEV *AddOp : Add->operands()) {
4863 if (AddOp->getType()->isPointerTy()) {
4864 assert(!PtrOp && "Cannot have multiple pointer ops");
4865 PtrOp = AddOp;
4866 }
4867 }
4868 assert(PtrOp && "Must have pointer op");
4869 V = PtrOp;
4870 } else // Not something we can look further into.
4871 return V;
4872 }
4873}
4874
4875/// Push users of the given Instruction onto the given Worklist.
4879 // Push the def-use children onto the Worklist stack.
4880 for (User *U : I->users()) {
4881 auto *UserInsn = cast<Instruction>(U);
4882 if (Visited.insert(UserInsn).second)
4883 Worklist.push_back(UserInsn);
4884 }
4885}
4886
4887namespace {
4888
4889/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its start
4890/// expression in case its Loop is L. If it is not L then
4891/// if IgnoreOtherLoops is true then use AddRec itself
4892/// otherwise rewrite cannot be done.
4893/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4894class SCEVInitRewriter : public SCEVRewriteVisitor<SCEVInitRewriter> {
4895public:
4896 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
4897 bool IgnoreOtherLoops = true) {
4898 SCEVInitRewriter Rewriter(L, SE);
4899 const SCEV *Result = Rewriter.visit(S);
4900 if (Rewriter.hasSeenLoopVariantSCEVUnknown())
4901 return SE.getCouldNotCompute();
4902 return Rewriter.hasSeenOtherLoops() && !IgnoreOtherLoops
4903 ? SE.getCouldNotCompute()
4904 : Result;
4905 }
4906
4907 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4908 if (!SE.isLoopInvariant(Expr, L))
4909 SeenLoopVariantSCEVUnknown = true;
4910 return Expr;
4911 }
4912
4913 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4914 // Only re-write AddRecExprs for this loop.
4915 if (Expr->getLoop() == L)
4916 return Expr->getStart();
4917 SeenOtherLoops = true;
4918 return Expr;
4919 }
4920
4921 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4922
4923 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4924
4925private:
4926 explicit SCEVInitRewriter(const Loop *L, ScalarEvolution &SE)
4927 : SCEVRewriteVisitor(SE), L(L) {}
4928
4929 const Loop *L;
4930 bool SeenLoopVariantSCEVUnknown = false;
4931 bool SeenOtherLoops = false;
4932};
4933
4934/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its post
4935/// increment expression in case its Loop is L. If it is not L then
4936/// use AddRec itself.
4937/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4938class SCEVPostIncRewriter : public SCEVRewriteVisitor<SCEVPostIncRewriter> {
4939public:
4940 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE) {
4941 SCEVPostIncRewriter Rewriter(L, SE);
4942 const SCEV *Result = Rewriter.visit(S);
4943 return Rewriter.hasSeenLoopVariantSCEVUnknown()
4944 ? SE.getCouldNotCompute()
4945 : Result;
4946 }
4947
4948 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4949 if (!SE.isLoopInvariant(Expr, L))
4950 SeenLoopVariantSCEVUnknown = true;
4951 return Expr;
4952 }
4953
4954 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4955 // Only re-write AddRecExprs for this loop.
4956 if (Expr->getLoop() == L)
4957 return Expr->getPostIncExpr(SE);
4958 SeenOtherLoops = true;
4959 return Expr;
4960 }
4961
4962 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4963
4964 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4965
4966private:
4967 explicit SCEVPostIncRewriter(const Loop *L, ScalarEvolution &SE)
4968 : SCEVRewriteVisitor(SE), L(L) {}
4969
4970 const Loop *L;
4971 bool SeenLoopVariantSCEVUnknown = false;
4972 bool SeenOtherLoops = false;
4973};
4974
4975/// This class evaluates the compare condition by matching it against the
4976/// condition of loop latch. If there is a match we assume a true value
4977/// for the condition while building SCEV nodes.
4978class SCEVBackedgeConditionFolder
4979 : public SCEVRewriteVisitor<SCEVBackedgeConditionFolder> {
4980public:
4981 static const SCEV *rewrite(const SCEV *S, const Loop *L,
4982 ScalarEvolution &SE) {
4983 bool IsPosBECond = false;
4984 Value *BECond = nullptr;
4985 if (BasicBlock *Latch = L->getLoopLatch()) {
4986 BranchInst *BI = dyn_cast<BranchInst>(Latch->getTerminator());
4987 if (BI && BI->isConditional()) {
4988 assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
4989 "Both outgoing branches should not target same header!");
4990 BECond = BI->getCondition();
4991 IsPosBECond = BI->getSuccessor(0) == L->getHeader();
4992 } else {
4993 return S;
4994 }
4995 }
4996 SCEVBackedgeConditionFolder Rewriter(L, BECond, IsPosBECond, SE);
4997 return Rewriter.visit(S);
4998 }
4999
5000 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5001 const SCEV *Result = Expr;
5002 bool InvariantF = SE.isLoopInvariant(Expr, L);
5003
5004 if (!InvariantF) {
5006 switch (I->getOpcode()) {
5007 case Instruction::Select: {
5008 SelectInst *SI = cast<SelectInst>(I);
5009 std::optional<const SCEV *> Res =
5010 compareWithBackedgeCondition(SI->getCondition());
5011 if (Res) {
5012 bool IsOne = cast<SCEVConstant>(*Res)->getValue()->isOne();
5013 Result = SE.getSCEV(IsOne ? SI->getTrueValue() : SI->getFalseValue());
5014 }
5015 break;
5016 }
5017 default: {
5018 std::optional<const SCEV *> Res = compareWithBackedgeCondition(I);
5019 if (Res)
5020 Result = *Res;
5021 break;
5022 }
5023 }
5024 }
5025 return Result;
5026 }
5027
5028private:
5029 explicit SCEVBackedgeConditionFolder(const Loop *L, Value *BECond,
5030 bool IsPosBECond, ScalarEvolution &SE)
5031 : SCEVRewriteVisitor(SE), L(L), BackedgeCond(BECond),
5032 IsPositiveBECond(IsPosBECond) {}
5033
5034 std::optional<const SCEV *> compareWithBackedgeCondition(Value *IC);
5035
5036 const Loop *L;
5037 /// Loop back condition.
5038 Value *BackedgeCond = nullptr;
5039 /// Set to true if loop back is on positive branch condition.
5040 bool IsPositiveBECond;
5041};
5042
5043std::optional<const SCEV *>
5044SCEVBackedgeConditionFolder::compareWithBackedgeCondition(Value *IC) {
5045
5046 // If value matches the backedge condition for loop latch,
5047 // then return a constant evolution node based on loopback
5048 // branch taken.
5049 if (BackedgeCond == IC)
5050 return IsPositiveBECond ? SE.getOne(Type::getInt1Ty(SE.getContext()))
5052 return std::nullopt;
5053}
5054
5055class SCEVShiftRewriter : public SCEVRewriteVisitor<SCEVShiftRewriter> {
5056public:
5057 static const SCEV *rewrite(const SCEV *S, const Loop *L,
5058 ScalarEvolution &SE) {
5059 SCEVShiftRewriter Rewriter(L, SE);
5060 const SCEV *Result = Rewriter.visit(S);
5061 return Rewriter.isValid() ? Result : SE.getCouldNotCompute();
5062 }
5063
5064 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5065 // Only allow AddRecExprs for this loop.
5066 if (!SE.isLoopInvariant(Expr, L))
5067 Valid = false;
5068 return Expr;
5069 }
5070
5071 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
5072 if (Expr->getLoop() == L && Expr->isAffine())
5073 return SE.getMinusSCEV(Expr, Expr->getStepRecurrence(SE));
5074 Valid = false;
5075 return Expr;
5076 }
5077
5078 bool isValid() { return Valid; }
5079
5080private:
5081 explicit SCEVShiftRewriter(const Loop *L, ScalarEvolution &SE)
5082 : SCEVRewriteVisitor(SE), L(L) {}
5083
5084 const Loop *L;
5085 bool Valid = true;
5086};
5087
5088} // end anonymous namespace
5089
5091ScalarEvolution::proveNoWrapViaConstantRanges(const SCEVAddRecExpr *AR) {
5092 if (!AR->isAffine())
5093 return SCEV::FlagAnyWrap;
5094
5095 using OBO = OverflowingBinaryOperator;
5096
5098
5099 if (!AR->hasNoSelfWrap()) {
5100 const SCEV *BECount = getConstantMaxBackedgeTakenCount(AR->getLoop());
5101 if (const SCEVConstant *BECountMax = dyn_cast<SCEVConstant>(BECount)) {
5102 ConstantRange StepCR = getSignedRange(AR->getStepRecurrence(*this));
5103 const APInt &BECountAP = BECountMax->getAPInt();
5104 unsigned NoOverflowBitWidth =
5105 BECountAP.getActiveBits() + StepCR.getMinSignedBits();
5106 if (NoOverflowBitWidth <= getTypeSizeInBits(AR->getType()))
5108 }
5109 }
5110
5111 if (!AR->hasNoSignedWrap()) {
5112 ConstantRange AddRecRange = getSignedRange(AR);
5113 ConstantRange IncRange = getSignedRange(AR->getStepRecurrence(*this));
5114
5116 Instruction::Add, IncRange, OBO::NoSignedWrap);
5117 if (NSWRegion.contains(AddRecRange))
5119 }
5120
5121 if (!AR->hasNoUnsignedWrap()) {
5122 ConstantRange AddRecRange = getUnsignedRange(AR);
5123 ConstantRange IncRange = getUnsignedRange(AR->getStepRecurrence(*this));
5124
5126 Instruction::Add, IncRange, OBO::NoUnsignedWrap);
5127 if (NUWRegion.contains(AddRecRange))
5129 }
5130
5131 return Result;
5132}
5133
5135ScalarEvolution::proveNoSignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5137
5138 if (AR->hasNoSignedWrap())
5139 return Result;
5140
5141 if (!AR->isAffine())
5142 return Result;
5143
5144 // This function can be expensive, only try to prove NSW once per AddRec.
5145 if (!SignedWrapViaInductionTried.insert(AR).second)
5146 return Result;
5147
5148 const SCEV *Step = AR->getStepRecurrence(*this);
5149 const Loop *L = AR->getLoop();
5150
5151 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5152 // Note that this serves two purposes: It filters out loops that are
5153 // simply not analyzable, and it covers the case where this code is
5154 // being called from within backedge-taken count analysis, such that
5155 // attempting to ask for the backedge-taken count would likely result
5156 // in infinite recursion. In the later case, the analysis code will
5157 // cope with a conservative value, and it will take care to purge
5158 // that value once it has finished.
5159 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5160
5161 // Normally, in the cases we can prove no-overflow via a
5162 // backedge guarding condition, we can also compute a backedge
5163 // taken count for the loop. The exceptions are assumptions and
5164 // guards present in the loop -- SCEV is not great at exploiting
5165 // these to compute max backedge taken counts, but can still use
5166 // these to prove lack of overflow. Use this fact to avoid
5167 // doing extra work that may not pay off.
5168
5169 if (isa<SCEVCouldNotCompute>(MaxBECount) && !HasGuards &&
5170 AC.assumptions().empty())
5171 return Result;
5172
5173 // If the backedge is guarded by a comparison with the pre-inc value the
5174 // addrec is safe. Also, if the entry is guarded by a comparison with the
5175 // start value and the backedge is guarded by a comparison with the post-inc
5176 // value, the addrec is safe.
5178 const SCEV *OverflowLimit =
5179 getSignedOverflowLimitForStep(Step, &Pred, this);
5180 if (OverflowLimit &&
5181 (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) ||
5182 isKnownOnEveryIteration(Pred, AR, OverflowLimit))) {
5183 Result = setFlags(Result, SCEV::FlagNSW);
5184 }
5185 return Result;
5186}
5188ScalarEvolution::proveNoUnsignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5190
5191 if (AR->hasNoUnsignedWrap())
5192 return Result;
5193
5194 if (!AR->isAffine())
5195 return Result;
5196
5197 // This function can be expensive, only try to prove NUW once per AddRec.
5198 if (!UnsignedWrapViaInductionTried.insert(AR).second)
5199 return Result;
5200
5201 const SCEV *Step = AR->getStepRecurrence(*this);
5202 unsigned BitWidth = getTypeSizeInBits(AR->getType());
5203 const Loop *L = AR->getLoop();
5204
5205 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5206 // Note that this serves two purposes: It filters out loops that are
5207 // simply not analyzable, and it covers the case where this code is
5208 // being called from within backedge-taken count analysis, such that
5209 // attempting to ask for the backedge-taken count would likely result
5210 // in infinite recursion. In the later case, the analysis code will
5211 // cope with a conservative value, and it will take care to purge
5212 // that value once it has finished.
5213 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5214
5215 // Normally, in the cases we can prove no-overflow via a
5216 // backedge guarding condition, we can also compute a backedge
5217 // taken count for the loop. The exceptions are assumptions and
5218 // guards present in the loop -- SCEV is not great at exploiting
5219 // these to compute max backedge taken counts, but can still use
5220 // these to prove lack of overflow. Use this fact to avoid
5221 // doing extra work that may not pay off.
5222
5223 if (isa<SCEVCouldNotCompute>(MaxBECount) && !HasGuards &&
5224 AC.assumptions().empty())
5225 return Result;
5226
5227 // If the backedge is guarded by a comparison with the pre-inc value the
5228 // addrec is safe. Also, if the entry is guarded by a comparison with the
5229 // start value and the backedge is guarded by a comparison with the post-inc
5230 // value, the addrec is safe.
5231 if (isKnownPositive(Step)) {
5232 const SCEV *N = getConstant(APInt::getMinValue(BitWidth) -
5233 getUnsignedRangeMax(Step));
5236 Result = setFlags(Result, SCEV::FlagNUW);
5237 }
5238 }
5239
5240 return Result;
5241}
5242
5243namespace {
5244
5245/// Represents an abstract binary operation. This may exist as a
5246/// normal instruction or constant expression, or may have been
5247/// derived from an expression tree.
5248struct BinaryOp {
5249 unsigned Opcode;
5250 Value *LHS;
5251 Value *RHS;
5252 bool IsNSW = false;
5253 bool IsNUW = false;
5254
5255 /// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or
5256 /// constant expression.
5257 Operator *Op = nullptr;
5258
5259 explicit BinaryOp(Operator *Op)
5260 : Opcode(Op->getOpcode()), LHS(Op->getOperand(0)), RHS(Op->getOperand(1)),
5261 Op(Op) {
5262 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Op)) {
5263 IsNSW = OBO->hasNoSignedWrap();
5264 IsNUW = OBO->hasNoUnsignedWrap();
5265 }
5266 }
5267
5268 explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false,
5269 bool IsNUW = false)
5270 : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW) {}
5271};
5272
5273} // end anonymous namespace
5274
5275/// Try to map \p V into a BinaryOp, and return \c std::nullopt on failure.
5276static std::optional<BinaryOp> MatchBinaryOp(Value *V, const DataLayout &DL,
5277 AssumptionCache &AC,
5278 const DominatorTree &DT,
5279 const Instruction *CxtI) {
5280 auto *Op = dyn_cast<Operator>(V);
5281 if (!Op)
5282 return std::nullopt;
5283
5284 // Implementation detail: all the cleverness here should happen without
5285 // creating new SCEV expressions -- our caller knowns tricks to avoid creating
5286 // SCEV expressions when possible, and we should not break that.
5287
5288 switch (Op->getOpcode()) {
5289 case Instruction::Add:
5290 case Instruction::Sub:
5291 case Instruction::Mul:
5292 case Instruction::UDiv:
5293 case Instruction::URem:
5294 case Instruction::And:
5295 case Instruction::AShr:
5296 case Instruction::Shl:
5297 return BinaryOp(Op);
5298
5299 case Instruction::Or: {
5300 // Convert or disjoint into add nuw nsw.
5301 if (cast<PossiblyDisjointInst>(Op)->isDisjoint())
5302 return BinaryOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1),
5303 /*IsNSW=*/true, /*IsNUW=*/true);
5304 return BinaryOp(Op);
5305 }
5306
5307 case Instruction::Xor:
5308 if (auto *RHSC = dyn_cast<ConstantInt>(Op->getOperand(1)))
5309 // If the RHS of the xor is a signmask, then this is just an add.
5310 // Instcombine turns add of signmask into xor as a strength reduction step.
5311 if (RHSC->getValue().isSignMask())
5312 return BinaryOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1));
5313 // Binary `xor` is a bit-wise `add`.
5314 if (V->getType()->isIntegerTy(1))
5315 return BinaryOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1));
5316 return BinaryOp(Op);
5317
5318 case Instruction::LShr:
5319 // Turn logical shift right of a constant into a unsigned divide.
5320 if (ConstantInt *SA = dyn_cast<ConstantInt>(Op->getOperand(1))) {
5321 uint32_t BitWidth = cast<IntegerType>(Op->getType())->getBitWidth();
5322
5323 // If the shift count is not less than the bitwidth, the result of
5324 // the shift is undefined. Don't try to analyze it, because the
5325 // resolution chosen here may differ from the resolution chosen in
5326 // other parts of the compiler.
5327 if (SA->getValue().ult(BitWidth)) {
5328 Constant *X =
5329 ConstantInt::get(SA->getContext(),
5330 APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
5331 return BinaryOp(Instruction::UDiv, Op->getOperand(0), X);
5332 }
5333 }
5334 return BinaryOp(Op);
5335
5336 case Instruction::ExtractValue: {
5337 auto *EVI = cast<ExtractValueInst>(Op);
5338 if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0)
5339 break;
5340
5341 auto *WO = dyn_cast<WithOverflowInst>(EVI->getAggregateOperand());
5342 if (!WO)
5343 break;
5344
5345 Instruction::BinaryOps BinOp = WO->getBinaryOp();
5346 bool Signed = WO->isSigned();
5347 // TODO: Should add nuw/nsw flags for mul as well.
5348 if (BinOp == Instruction::Mul || !isOverflowIntrinsicNoWrap(WO, DT))
5349 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS());
5350
5351 // Now that we know that all uses of the arithmetic-result component of
5352 // CI are guarded by the overflow check, we can go ahead and pretend
5353 // that the arithmetic is non-overflowing.
5354 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS(),
5355 /* IsNSW = */ Signed, /* IsNUW = */ !Signed);
5356 }
5357
5358 default:
5359 break;
5360 }
5361
5362 // Recognise intrinsic loop.decrement.reg, and as this has exactly the same
5363 // semantics as a Sub, return a binary sub expression.
5364 if (auto *II = dyn_cast<IntrinsicInst>(V))
5365 if (II->getIntrinsicID() == Intrinsic::loop_decrement_reg)
5366 return BinaryOp(Instruction::Sub, II->getOperand(0), II->getOperand(1));
5367
5368 return std::nullopt;
5369}
5370
5371/// Helper function to createAddRecFromPHIWithCasts. We have a phi
5372/// node whose symbolic (unknown) SCEV is \p SymbolicPHI, which is updated via
5373/// the loop backedge by a SCEVAddExpr, possibly also with a few casts on the
5374/// way. This function checks if \p Op, an operand of this SCEVAddExpr,
5375/// follows one of the following patterns:
5376/// Op == (SExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5377/// Op == (ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5378/// If the SCEV expression of \p Op conforms with one of the expected patterns
5379/// we return the type of the truncation operation, and indicate whether the
5380/// truncated type should be treated as signed/unsigned by setting
5381/// \p Signed to true/false, respectively.
5382static Type *isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI,
5383 bool &Signed, ScalarEvolution &SE) {
5384 // The case where Op == SymbolicPHI (that is, with no type conversions on
5385 // the way) is handled by the regular add recurrence creating logic and
5386 // would have already been triggered in createAddRecForPHI. Reaching it here
5387 // means that createAddRecFromPHI had failed for this PHI before (e.g.,
5388 // because one of the other operands of the SCEVAddExpr updating this PHI is
5389 // not invariant).
5390 //
5391 // Here we look for the case where Op = (ext(trunc(SymbolicPHI))), and in
5392 // this case predicates that allow us to prove that Op == SymbolicPHI will
5393 // be added.
5394 if (Op == SymbolicPHI)
5395 return nullptr;
5396
5397 unsigned SourceBits = SE.getTypeSizeInBits(SymbolicPHI->getType());
5398 unsigned NewBits = SE.getTypeSizeInBits(Op->getType());
5399 if (SourceBits != NewBits)
5400 return nullptr;
5401
5402 if (match(Op, m_scev_SExt(m_scev_Trunc(m_scev_Specific(SymbolicPHI))))) {
5403 Signed = true;
5404 return cast<SCEVCastExpr>(Op)->getOperand()->getType();
5405 }
5406 if (match(Op, m_scev_ZExt(m_scev_Trunc(m_scev_Specific(SymbolicPHI))))) {
5407 Signed = false;
5408 return cast<SCEVCastExpr>(Op)->getOperand()->getType();
5409 }
5410 return nullptr;
5411}
5412
5413static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) {
5414 if (!PN->getType()->isIntegerTy())
5415 return nullptr;
5416 const Loop *L = LI.getLoopFor(PN->getParent());
5417 if (!L || L->getHeader() != PN->getParent())
5418 return nullptr;
5419 return L;
5420}
5421
5422// Analyze \p SymbolicPHI, a SCEV expression of a phi node, and check if the
5423// computation that updates the phi follows the following pattern:
5424// (SExt/ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy) + InvariantAccum
5425// which correspond to a phi->trunc->sext/zext->add->phi update chain.
5426// If so, try to see if it can be rewritten as an AddRecExpr under some
5427// Predicates. If successful, return them as a pair. Also cache the results
5428// of the analysis.
5429//
5430// Example usage scenario:
5431// Say the Rewriter is called for the following SCEV:
5432// 8 * ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5433// where:
5434// %X = phi i64 (%Start, %BEValue)
5435// It will visitMul->visitAdd->visitSExt->visitTrunc->visitUnknown(%X),
5436// and call this function with %SymbolicPHI = %X.
5437//
5438// The analysis will find that the value coming around the backedge has
5439// the following SCEV:
5440// BEValue = ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5441// Upon concluding that this matches the desired pattern, the function
5442// will return the pair {NewAddRec, SmallPredsVec} where:
5443// NewAddRec = {%Start,+,%Step}
5444// SmallPredsVec = {P1, P2, P3} as follows:
5445// P1(WrapPred): AR: {trunc(%Start),+,(trunc %Step)}<nsw> Flags: <nssw>
5446// P2(EqualPred): %Start == (sext i32 (trunc i64 %Start to i32) to i64)
5447// P3(EqualPred): %Step == (sext i32 (trunc i64 %Step to i32) to i64)
5448// The returned pair means that SymbolicPHI can be rewritten into NewAddRec
5449// under the predicates {P1,P2,P3}.
5450// This predicated rewrite will be cached in PredicatedSCEVRewrites:
5451// PredicatedSCEVRewrites[{%X,L}] = {NewAddRec, {P1,P2,P3)}
5452//
5453// TODO's:
5454//
5455// 1) Extend the Induction descriptor to also support inductions that involve
5456// casts: When needed (namely, when we are called in the context of the
5457// vectorizer induction analysis), a Set of cast instructions will be
5458// populated by this method, and provided back to isInductionPHI. This is
5459// needed to allow the vectorizer to properly record them to be ignored by
5460// the cost model and to avoid vectorizing them (otherwise these casts,
5461// which are redundant under the runtime overflow checks, will be
5462// vectorized, which can be costly).
5463//
5464// 2) Support additional induction/PHISCEV patterns: We also want to support
5465// inductions where the sext-trunc / zext-trunc operations (partly) occur
5466// after the induction update operation (the induction increment):
5467//
5468// (Trunc iy (SExt/ZExt ix (%SymbolicPHI + InvariantAccum) to iy) to ix)
5469// which correspond to a phi->add->trunc->sext/zext->phi update chain.
5470//
5471// (Trunc iy ((SExt/ZExt ix (%SymbolicPhi) to iy) + InvariantAccum) to ix)
5472// which correspond to a phi->trunc->add->sext/zext->phi update chain.
5473//
5474// 3) Outline common code with createAddRecFromPHI to avoid duplication.
5475std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5476ScalarEvolution::createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI) {
5478
5479 // *** Part1: Analyze if we have a phi-with-cast pattern for which we can
5480 // return an AddRec expression under some predicate.
5481
5482 auto *PN = cast<PHINode>(SymbolicPHI->getValue());
5483 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5484 assert(L && "Expecting an integer loop header phi");
5485
5486 // The loop may have multiple entrances or multiple exits; we can analyze
5487 // this phi as an addrec if it has a unique entry value and a unique
5488 // backedge value.
5489 Value *BEValueV = nullptr, *StartValueV = nullptr;
5490 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5491 Value *V = PN->getIncomingValue(i);
5492 if (L->contains(PN->getIncomingBlock(i))) {
5493 if (!BEValueV) {
5494 BEValueV = V;
5495 } else if (BEValueV != V) {
5496 BEValueV = nullptr;
5497 break;
5498 }
5499 } else if (!StartValueV) {
5500 StartValueV = V;
5501 } else if (StartValueV != V) {
5502 StartValueV = nullptr;
5503 break;
5504 }
5505 }
5506 if (!BEValueV || !StartValueV)
5507 return std::nullopt;
5508
5509 const SCEV *BEValue = getSCEV(BEValueV);
5510
5511 // If the value coming around the backedge is an add with the symbolic
5512 // value we just inserted, possibly with casts that we can ignore under
5513 // an appropriate runtime guard, then we found a simple induction variable!
5514 const auto *Add = dyn_cast<SCEVAddExpr>(BEValue);
5515 if (!Add)
5516 return std::nullopt;
5517
5518 // If there is a single occurrence of the symbolic value, possibly
5519 // casted, replace it with a recurrence.
5520 unsigned FoundIndex = Add->getNumOperands();
5521 Type *TruncTy = nullptr;
5522 bool Signed;
5523 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5524 if ((TruncTy =
5525 isSimpleCastedPHI(Add->getOperand(i), SymbolicPHI, Signed, *this)))
5526 if (FoundIndex == e) {
5527 FoundIndex = i;
5528 break;
5529 }
5530
5531 if (FoundIndex == Add->getNumOperands())
5532 return std::nullopt;
5533
5534 // Create an add with everything but the specified operand.
5536 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5537 if (i != FoundIndex)
5538 Ops.push_back(Add->getOperand(i));
5539 const SCEV *Accum = getAddExpr(Ops);
5540
5541 // The runtime checks will not be valid if the step amount is
5542 // varying inside the loop.
5543 if (!isLoopInvariant(Accum, L))
5544 return std::nullopt;
5545
5546 // *** Part2: Create the predicates
5547
5548 // Analysis was successful: we have a phi-with-cast pattern for which we
5549 // can return an AddRec expression under the following predicates:
5550 //
5551 // P1: A Wrap predicate that guarantees that Trunc(Start) + i*Trunc(Accum)
5552 // fits within the truncated type (does not overflow) for i = 0 to n-1.
5553 // P2: An Equal predicate that guarantees that
5554 // Start = (Ext ix (Trunc iy (Start) to ix) to iy)
5555 // P3: An Equal predicate that guarantees that
5556 // Accum = (Ext ix (Trunc iy (Accum) to ix) to iy)
5557 //
5558 // As we next prove, the above predicates guarantee that:
5559 // Start + i*Accum = (Ext ix (Trunc iy ( Start + i*Accum ) to ix) to iy)
5560 //
5561 //
5562 // More formally, we want to prove that:
5563 // Expr(i+1) = Start + (i+1) * Accum
5564 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5565 //
5566 // Given that:
5567 // 1) Expr(0) = Start
5568 // 2) Expr(1) = Start + Accum
5569 // = (Ext ix (Trunc iy (Start) to ix) to iy) + Accum :: from P2
5570 // 3) Induction hypothesis (step i):
5571 // Expr(i) = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum
5572 //
5573 // Proof:
5574 // Expr(i+1) =
5575 // = Start + (i+1)*Accum
5576 // = (Start + i*Accum) + Accum
5577 // = Expr(i) + Accum
5578 // = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum + Accum
5579 // :: from step i
5580 //
5581 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy) + Accum + Accum
5582 //
5583 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy)
5584 // + (Ext ix (Trunc iy (Accum) to ix) to iy)
5585 // + Accum :: from P3
5586 //
5587 // = (Ext ix (Trunc iy ((Start + (i-1)*Accum) + Accum) to ix) to iy)
5588 // + Accum :: from P1: Ext(x)+Ext(y)=>Ext(x+y)
5589 //
5590 // = (Ext ix (Trunc iy (Start + i*Accum) to ix) to iy) + Accum
5591 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5592 //
5593 // By induction, the same applies to all iterations 1<=i<n:
5594 //
5595
5596 // Create a truncated addrec for which we will add a no overflow check (P1).
5597 const SCEV *StartVal = getSCEV(StartValueV);
5598 const SCEV *PHISCEV =
5599 getAddRecExpr(getTruncateExpr(StartVal, TruncTy),
5600 getTruncateExpr(Accum, TruncTy), L, SCEV::FlagAnyWrap);
5601
5602 // PHISCEV can be either a SCEVConstant or a SCEVAddRecExpr.
5603 // ex: If truncated Accum is 0 and StartVal is a constant, then PHISCEV
5604 // will be constant.
5605 //
5606 // If PHISCEV is a constant, then P1 degenerates into P2 or P3, so we don't
5607 // add P1.
5608 if (const auto *AR = dyn_cast<SCEVAddRecExpr>(PHISCEV)) {
5612 const SCEVPredicate *AddRecPred = getWrapPredicate(AR, AddedFlags);
5613 Predicates.push_back(AddRecPred);
5614 }
5615
5616 // Create the Equal Predicates P2,P3:
5617
5618 // It is possible that the predicates P2 and/or P3 are computable at
5619 // compile time due to StartVal and/or Accum being constants.
5620 // If either one is, then we can check that now and escape if either P2
5621 // or P3 is false.
5622
5623 // Construct the extended SCEV: (Ext ix (Trunc iy (Expr) to ix) to iy)
5624 // for each of StartVal and Accum
5625 auto getExtendedExpr = [&](const SCEV *Expr,
5626 bool CreateSignExtend) -> const SCEV * {
5627 assert(isLoopInvariant(Expr, L) && "Expr is expected to be invariant");
5628 const SCEV *TruncatedExpr = getTruncateExpr(Expr, TruncTy);
5629 const SCEV *ExtendedExpr =
5630 CreateSignExtend ? getSignExtendExpr(TruncatedExpr, Expr->getType())
5631 : getZeroExtendExpr(TruncatedExpr, Expr->getType());
5632 return ExtendedExpr;
5633 };
5634
5635 // Given:
5636 // ExtendedExpr = (Ext ix (Trunc iy (Expr) to ix) to iy
5637 // = getExtendedExpr(Expr)
5638 // Determine whether the predicate P: Expr == ExtendedExpr
5639 // is known to be false at compile time
5640 auto PredIsKnownFalse = [&](const SCEV *Expr,
5641 const SCEV *ExtendedExpr) -> bool {
5642 return Expr != ExtendedExpr &&
5643 isKnownPredicate(ICmpInst::ICMP_NE, Expr, ExtendedExpr);
5644 };
5645
5646 const SCEV *StartExtended = getExtendedExpr(StartVal, Signed);
5647 if (PredIsKnownFalse(StartVal, StartExtended)) {
5648 LLVM_DEBUG(dbgs() << "P2 is compile-time false\n";);
5649 return std::nullopt;
5650 }
5651
5652 // The Step is always Signed (because the overflow checks are either
5653 // NSSW or NUSW)
5654 const SCEV *AccumExtended = getExtendedExpr(Accum, /*CreateSignExtend=*/true);
5655 if (PredIsKnownFalse(Accum, AccumExtended)) {
5656 LLVM_DEBUG(dbgs() << "P3 is compile-time false\n";);
5657 return std::nullopt;
5658 }
5659
5660 auto AppendPredicate = [&](const SCEV *Expr,
5661 const SCEV *ExtendedExpr) -> void {
5662 if (Expr != ExtendedExpr &&
5663 !isKnownPredicate(ICmpInst::ICMP_EQ, Expr, ExtendedExpr)) {
5664 const SCEVPredicate *Pred = getEqualPredicate(Expr, ExtendedExpr);
5665 LLVM_DEBUG(dbgs() << "Added Predicate: " << *Pred);
5666 Predicates.push_back(Pred);
5667 }
5668 };
5669
5670 AppendPredicate(StartVal, StartExtended);
5671 AppendPredicate(Accum, AccumExtended);
5672
5673 // *** Part3: Predicates are ready. Now go ahead and create the new addrec in
5674 // which the casts had been folded away. The caller can rewrite SymbolicPHI
5675 // into NewAR if it will also add the runtime overflow checks specified in
5676 // Predicates.
5677 auto *NewAR = getAddRecExpr(StartVal, Accum, L, SCEV::FlagAnyWrap);
5678
5679 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> PredRewrite =
5680 std::make_pair(NewAR, Predicates);
5681 // Remember the result of the analysis for this SCEV at this locayyytion.
5682 PredicatedSCEVRewrites[{SymbolicPHI, L}] = PredRewrite;
5683 return PredRewrite;
5684}
5685
5686std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5688 auto *PN = cast<PHINode>(SymbolicPHI->getValue());
5689 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5690 if (!L)
5691 return std::nullopt;
5692
5693 // Check to see if we already analyzed this PHI.
5694 auto I = PredicatedSCEVRewrites.find({SymbolicPHI, L});
5695 if (I != PredicatedSCEVRewrites.end()) {
5696 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> Rewrite =
5697 I->second;
5698 // Analysis was done before and failed to create an AddRec:
5699 if (Rewrite.first == SymbolicPHI)
5700 return std::nullopt;
5701 // Analysis was done before and succeeded to create an AddRec under
5702 // a predicate:
5703 assert(isa<SCEVAddRecExpr>(Rewrite.first) && "Expected an AddRec");
5704 assert(!(Rewrite.second).empty() && "Expected to find Predicates");
5705 return Rewrite;
5706 }
5707
5708 std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5709 Rewrite = createAddRecFromPHIWithCastsImpl(SymbolicPHI);
5710
5711 // Record in the cache that the analysis failed
5712 if (!Rewrite) {
5714 PredicatedSCEVRewrites[{SymbolicPHI, L}] = {SymbolicPHI, Predicates};
5715 return std::nullopt;
5716 }
5717
5718 return Rewrite;
5719}
5720
5721// FIXME: This utility is currently required because the Rewriter currently
5722// does not rewrite this expression:
5723// {0, +, (sext ix (trunc iy to ix) to iy)}
5724// into {0, +, %step},
5725// even when the following Equal predicate exists:
5726// "%step == (sext ix (trunc iy to ix) to iy)".
5728 const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2) const {
5729 if (AR1 == AR2)
5730 return true;
5731
5732 auto areExprsEqual = [&](const SCEV *Expr1, const SCEV *Expr2) -> bool {
5733 if (Expr1 != Expr2 &&
5734 !Preds->implies(SE.getEqualPredicate(Expr1, Expr2), SE) &&
5735 !Preds->implies(SE.getEqualPredicate(Expr2, Expr1), SE))
5736 return false;
5737 return true;
5738 };
5739
5740 if (!areExprsEqual(AR1->getStart(), AR2->getStart()) ||
5741 !areExprsEqual(AR1->getStepRecurrence(SE), AR2->getStepRecurrence(SE)))
5742 return false;
5743 return true;
5744}
5745
5746/// A helper function for createAddRecFromPHI to handle simple cases.
5747///
5748/// This function tries to find an AddRec expression for the simplest (yet most
5749/// common) cases: PN = PHI(Start, OP(Self, LoopInvariant)).
5750/// If it fails, createAddRecFromPHI will use a more general, but slow,
5751/// technique for finding the AddRec expression.
5752const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
5753 Value *BEValueV,
5754 Value *StartValueV) {
5755 const Loop *L = LI.getLoopFor(PN->getParent());
5756 assert(L && L->getHeader() == PN->getParent());
5757 assert(BEValueV && StartValueV);
5758
5759 auto BO = MatchBinaryOp(BEValueV, getDataLayout(), AC, DT, PN);
5760 if (!BO)
5761 return nullptr;
5762
5763 if (BO->Opcode != Instruction::Add)
5764 return nullptr;
5765
5766 const SCEV *Accum = nullptr;
5767 if (BO->LHS == PN && L->isLoopInvariant(BO->RHS))
5768 Accum = getSCEV(BO->RHS);
5769 else if (BO->RHS == PN && L->isLoopInvariant(BO->LHS))
5770 Accum = getSCEV(BO->LHS);
5771
5772 if (!Accum)
5773 return nullptr;
5774
5776 if (BO->IsNUW)
5777 Flags = setFlags(Flags, SCEV::FlagNUW);
5778 if (BO->IsNSW)
5779 Flags = setFlags(Flags, SCEV::FlagNSW);
5780
5781 const SCEV *StartVal = getSCEV(StartValueV);
5782 const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags);
5783 insertValueToMap(PN, PHISCEV);
5784
5785 if (auto *AR = dyn_cast<SCEVAddRecExpr>(PHISCEV)) {
5786 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR),
5788 proveNoWrapViaConstantRanges(AR)));
5789 }
5790
5791 // We can add Flags to the post-inc expression only if we
5792 // know that it is *undefined behavior* for BEValueV to
5793 // overflow.
5794 if (auto *BEInst = dyn_cast<Instruction>(BEValueV)) {
5795 assert(isLoopInvariant(Accum, L) &&
5796 "Accum is defined outside L, but is not invariant?");
5797 if (isAddRecNeverPoison(BEInst, L))
5798 (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, Flags);
5799 }
5800
5801 return PHISCEV;
5802}
5803
5804const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
5805 const Loop *L = LI.getLoopFor(PN->getParent());
5806 if (!L || L->getHeader() != PN->getParent())
5807 return nullptr;
5808
5809 // The loop may have multiple entrances or multiple exits; we can analyze
5810 // this phi as an addrec if it has a unique entry value and a unique
5811 // backedge value.
5812 Value *BEValueV = nullptr, *StartValueV = nullptr;
5813 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5814 Value *V = PN->getIncomingValue(i);
5815 if (L->contains(PN->getIncomingBlock(i))) {
5816 if (!BEValueV) {
5817 BEValueV = V;
5818 } else if (BEValueV != V) {
5819 BEValueV = nullptr;
5820 break;
5821 }
5822 } else if (!StartValueV) {
5823 StartValueV = V;
5824 } else if (StartValueV != V) {
5825 StartValueV = nullptr;
5826 break;
5827 }
5828 }
5829 if (!BEValueV || !StartValueV)
5830 return nullptr;
5831
5832 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
5833 "PHI node already processed?");
5834
5835 // First, try to find AddRec expression without creating a fictituos symbolic
5836 // value for PN.
5837 if (auto *S = createSimpleAffineAddRec(PN, BEValueV, StartValueV))
5838 return S;
5839
5840 // Handle PHI node value symbolically.
5841 const SCEV *SymbolicName = getUnknown(PN);
5842 insertValueToMap(PN, SymbolicName);
5843
5844 // Using this symbolic name for the PHI, analyze the value coming around
5845 // the back-edge.
5846 const SCEV *BEValue = getSCEV(BEValueV);
5847
5848 // NOTE: If BEValue is loop invariant, we know that the PHI node just
5849 // has a special value for the first iteration of the loop.
5850
5851 // If the value coming around the backedge is an add with the symbolic
5852 // value we just inserted, then we found a simple induction variable!
5853 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BEValue)) {
5854 // If there is a single occurrence of the symbolic value, replace it
5855 // with a recurrence.
5856 unsigned FoundIndex = Add->getNumOperands();
5857 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5858 if (Add->getOperand(i) == SymbolicName)
5859 if (FoundIndex == e) {
5860 FoundIndex = i;
5861 break;
5862 }
5863
5864 if (FoundIndex != Add->getNumOperands()) {
5865 // Create an add with everything but the specified operand.
5867 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5868 if (i != FoundIndex)
5869 Ops.push_back(SCEVBackedgeConditionFolder::rewrite(Add->getOperand(i),
5870 L, *this));
5871 const SCEV *Accum = getAddExpr(Ops);
5872
5873 // This is not a valid addrec if the step amount is varying each
5874 // loop iteration, but is not itself an addrec in this loop.
5875 if (isLoopInvariant(Accum, L) ||
5876 (isa<SCEVAddRecExpr>(Accum) &&
5877 cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) {
5879
5880 if (auto BO = MatchBinaryOp(BEValueV, getDataLayout(), AC, DT, PN)) {
5881 if (BO->Opcode == Instruction::Add && BO->LHS == PN) {
5882 if (BO->IsNUW)
5883 Flags = setFlags(Flags, SCEV::FlagNUW);
5884 if (BO->IsNSW)
5885 Flags = setFlags(Flags, SCEV::FlagNSW);
5886 }
5887 } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(BEValueV)) {
5888 if (GEP->getOperand(0) == PN) {
5889 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
5890 // If the increment has any nowrap flags, then we know the address
5891 // space cannot be wrapped around.
5892 if (NW != GEPNoWrapFlags::none())
5893 Flags = setFlags(Flags, SCEV::FlagNW);
5894 // If the GEP is nuw or nusw with non-negative offset, we know that
5895 // no unsigned wrap occurs. We cannot set the nsw flag as only the
5896 // offset is treated as signed, while the base is unsigned.
5897 if (NW.hasNoUnsignedWrap() ||
5899 Flags = setFlags(Flags, SCEV::FlagNUW);
5900 }
5901
5902 // We cannot transfer nuw and nsw flags from subtraction
5903 // operations -- sub nuw X, Y is not the same as add nuw X, -Y
5904 // for instance.
5905 }
5906
5907 const SCEV *StartVal = getSCEV(StartValueV);
5908 const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags);
5909
5910 // Okay, for the entire analysis of this edge we assumed the PHI
5911 // to be symbolic. We now need to go back and purge all of the
5912 // entries for the scalars that use the symbolic expression.
5913 forgetMemoizedResults(SymbolicName);
5914 insertValueToMap(PN, PHISCEV);
5915
5916 if (auto *AR = dyn_cast<SCEVAddRecExpr>(PHISCEV)) {
5917 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR),
5919 proveNoWrapViaConstantRanges(AR)));
5920 }
5921
5922 // We can add Flags to the post-inc expression only if we
5923 // know that it is *undefined behavior* for BEValueV to
5924 // overflow.
5925 if (auto *BEInst = dyn_cast<Instruction>(BEValueV))
5926 if (isLoopInvariant(Accum, L) && isAddRecNeverPoison(BEInst, L))
5927 (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, Flags);
5928
5929 return PHISCEV;
5930 }
5931 }
5932 } else {
5933 // Otherwise, this could be a loop like this:
5934 // i = 0; for (j = 1; ..; ++j) { .... i = j; }
5935 // In this case, j = {1,+,1} and BEValue is j.
5936 // Because the other in-value of i (0) fits the evolution of BEValue
5937 // i really is an addrec evolution.
5938 //
5939 // We can generalize this saying that i is the shifted value of BEValue
5940 // by one iteration:
5941 // PHI(f(0), f({1,+,1})) --> f({0,+,1})
5942
5943 // Do not allow refinement in rewriting of BEValue.
5944 const SCEV *Shifted = SCEVShiftRewriter::rewrite(BEValue, L, *this);
5945 const SCEV *Start = SCEVInitRewriter::rewrite(Shifted, L, *this, false);
5946 if (Shifted != getCouldNotCompute() && Start != getCouldNotCompute() &&
5947 isGuaranteedNotToCauseUB(Shifted) && ::impliesPoison(Shifted, Start)) {
5948 const SCEV *StartVal = getSCEV(StartValueV);
5949 if (Start == StartVal) {
5950 // Okay, for the entire analysis of this edge we assumed the PHI
5951 // to be symbolic. We now need to go back and purge all of the
5952 // entries for the scalars that use the symbolic expression.
5953 forgetMemoizedResults(SymbolicName);
5954 insertValueToMap(PN, Shifted);
5955 return Shifted;
5956 }
5957 }
5958 }
5959
5960 // Remove the temporary PHI node SCEV that has been inserted while intending
5961 // to create an AddRecExpr for this PHI node. We can not keep this temporary
5962 // as it will prevent later (possibly simpler) SCEV expressions to be added
5963 // to the ValueExprMap.
5964 eraseValueFromMap(PN);
5965
5966 return nullptr;
5967}
5968
5969// Try to match a control flow sequence that branches out at BI and merges back
5970// at Merge into a "C ? LHS : RHS" select pattern. Return true on a successful
5971// match.
5973 Value *&C, Value *&LHS, Value *&RHS) {
5974 C = BI->getCondition();
5975
5976 BasicBlockEdge LeftEdge(BI->getParent(), BI->getSuccessor(0));
5977 BasicBlockEdge RightEdge(BI->getParent(), BI->getSuccessor(1));
5978
5979 if (!LeftEdge.isSingleEdge())
5980 return false;
5981
5982 assert(RightEdge.isSingleEdge() && "Follows from LeftEdge.isSingleEdge()");
5983
5984 Use &LeftUse = Merge->getOperandUse(0);
5985 Use &RightUse = Merge->getOperandUse(1);
5986
5987 if (DT.dominates(LeftEdge, LeftUse) && DT.dominates(RightEdge, RightUse)) {
5988 LHS = LeftUse;
5989 RHS = RightUse;
5990 return true;
5991 }
5992
5993 if (DT.dominates(LeftEdge, RightUse) && DT.dominates(RightEdge, LeftUse)) {
5994 LHS = RightUse;
5995 RHS = LeftUse;
5996 return true;
5997 }
5998
5999 return false;
6000}
6001
6002const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
6003 auto IsReachable =
6004 [&](BasicBlock *BB) { return DT.isReachableFromEntry(BB); };
6005 if (PN->getNumIncomingValues() == 2 && all_of(PN->blocks(), IsReachable)) {
6006 // Try to match
6007 //
6008 // br %cond, label %left, label %right
6009 // left:
6010 // br label %merge
6011 // right:
6012 // br label %merge
6013 // merge:
6014 // V = phi [ %x, %left ], [ %y, %right ]
6015 //
6016 // as "select %cond, %x, %y"
6017
6018 BasicBlock *IDom = DT[PN->getParent()]->getIDom()->getBlock();
6019 assert(IDom && "At least the entry block should dominate PN");
6020
6021 auto *BI = dyn_cast<BranchInst>(IDom->getTerminator());
6022 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
6023
6024 if (BI && BI->isConditional() &&
6025 BrPHIToSelect(DT, BI, PN, Cond, LHS, RHS) &&
6028 return createNodeForSelectOrPHI(PN, Cond, LHS, RHS);
6029 }
6030
6031 return nullptr;
6032}
6033
6034/// Returns SCEV for the first operand of a phi if all phi operands have
6035/// identical opcodes and operands
6036/// eg.
6037/// a: %add = %a + %b
6038/// br %c
6039/// b: %add1 = %a + %b
6040/// br %c
6041/// c: %phi = phi [%add, a], [%add1, b]
6042/// scev(%phi) => scev(%add)
6043const SCEV *
6044ScalarEvolution::createNodeForPHIWithIdenticalOperands(PHINode *PN) {
6045 BinaryOperator *CommonInst = nullptr;
6046 // Check if instructions are identical.
6047 for (Value *Incoming : PN->incoming_values()) {
6048 auto *IncomingInst = dyn_cast<BinaryOperator>(Incoming);
6049 if (!IncomingInst)
6050 return nullptr;
6051 if (CommonInst) {
6052 if (!CommonInst->isIdenticalToWhenDefined(IncomingInst))
6053 return nullptr; // Not identical, give up
6054 } else {
6055 // Remember binary operator
6056 CommonInst = IncomingInst;
6057 }
6058 }
6059 if (!CommonInst)
6060 return nullptr;
6061
6062 // Check if SCEV exprs for instructions are identical.
6063 const SCEV *CommonSCEV = getSCEV(CommonInst);
6064 bool SCEVExprsIdentical =
6066 [this, CommonSCEV](Value *V) { return CommonSCEV == getSCEV(V); });
6067 return SCEVExprsIdentical ? CommonSCEV : nullptr;
6068}
6069
6070const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
6071 if (const SCEV *S = createAddRecFromPHI(PN))
6072 return S;
6073
6074 // We do not allow simplifying phi (undef, X) to X here, to avoid reusing the
6075 // phi node for X.
6076 if (Value *V = simplifyInstruction(
6077 PN, {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
6078 /*UseInstrInfo=*/true, /*CanUseUndef=*/false}))
6079 return getSCEV(V);
6080
6081 if (const SCEV *S = createNodeForPHIWithIdenticalOperands(PN))
6082 return S;
6083
6084 if (const SCEV *S = createNodeFromSelectLikePHI(PN))
6085 return S;
6086
6087 // If it's not a loop phi, we can't handle it yet.
6088 return getUnknown(PN);
6089}
6090
6091bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind,
6092 SCEVTypes RootKind) {
6093 struct FindClosure {
6094 const SCEV *OperandToFind;
6095 const SCEVTypes RootKind; // Must be a sequential min/max expression.
6096 const SCEVTypes NonSequentialRootKind; // Non-seq variant of RootKind.
6097
6098 bool Found = false;
6099
6100 bool canRecurseInto(SCEVTypes Kind) const {
6101 // We can only recurse into the SCEV expression of the same effective type
6102 // as the type of our root SCEV expression, and into zero-extensions.
6103 return RootKind == Kind || NonSequentialRootKind == Kind ||
6104 scZeroExtend == Kind;
6105 };
6106
6107 FindClosure(const SCEV *OperandToFind, SCEVTypes RootKind)
6108 : OperandToFind(OperandToFind), RootKind(RootKind),
6109 NonSequentialRootKind(
6111 RootKind)) {}
6112
6113 bool follow(const SCEV *S) {
6114 Found = S == OperandToFind;
6115
6116 return !isDone() && canRecurseInto(S->getSCEVType());
6117 }
6118
6119 bool isDone() const { return Found; }
6120 };
6121
6122 FindClosure FC(OperandToFind, RootKind);
6123 visitAll(Root, FC);
6124 return FC.Found;
6125}
6126
6127std::optional<const SCEV *>
6128ScalarEvolution::createNodeForSelectOrPHIInstWithICmpInstCond(Type *Ty,
6129 ICmpInst *Cond,
6130 Value *TrueVal,
6131 Value *FalseVal) {
6132 // Try to match some simple smax or umax patterns.
6133 auto *ICI = Cond;
6134
6135 Value *LHS = ICI->getOperand(0);
6136 Value *RHS = ICI->getOperand(1);
6137
6138 switch (ICI->getPredicate()) {
6139 case ICmpInst::ICMP_SLT:
6140 case ICmpInst::ICMP_SLE:
6141 case ICmpInst::ICMP_ULT:
6142 case ICmpInst::ICMP_ULE:
6143 std::swap(LHS, RHS);
6144 [[fallthrough]];
6145 case ICmpInst::ICMP_SGT:
6146 case ICmpInst::ICMP_SGE:
6147 case ICmpInst::ICMP_UGT:
6148 case ICmpInst::ICMP_UGE:
6149 // a > b ? a+x : b+x -> max(a, b)+x
6150 // a > b ? b+x : a+x -> min(a, b)+x
6152 bool Signed = ICI->isSigned();
6153 const SCEV *LA = getSCEV(TrueVal);
6154 const SCEV *RA = getSCEV(FalseVal);
6155 const SCEV *LS = getSCEV(LHS);
6156 const SCEV *RS = getSCEV(RHS);
6157 if (LA->getType()->isPointerTy()) {
6158 // FIXME: Handle cases where LS/RS are pointers not equal to LA/RA.
6159 // Need to make sure we can't produce weird expressions involving
6160 // negated pointers.
6161 if (LA == LS && RA == RS)
6162 return Signed ? getSMaxExpr(LS, RS) : getUMaxExpr(LS, RS);
6163 if (LA == RS && RA == LS)
6164 return Signed ? getSMinExpr(LS, RS) : getUMinExpr(LS, RS);
6165 }
6166 auto CoerceOperand = [&](const SCEV *Op) -> const SCEV * {
6167 if (Op->getType()->isPointerTy()) {
6170 return Op;
6171 }
6172 if (Signed)
6173 Op = getNoopOrSignExtend(Op, Ty);
6174 else
6175 Op = getNoopOrZeroExtend(Op, Ty);
6176 return Op;
6177 };
6178 LS = CoerceOperand(LS);
6179 RS = CoerceOperand(RS);
6181 break;
6182 const SCEV *LDiff = getMinusSCEV(LA, LS);
6183 const SCEV *RDiff = getMinusSCEV(RA, RS);
6184 if (LDiff == RDiff)
6185 return getAddExpr(Signed ? getSMaxExpr(LS, RS) : getUMaxExpr(LS, RS),
6186 LDiff);
6187 LDiff = getMinusSCEV(LA, RS);
6188 RDiff = getMinusSCEV(RA, LS);
6189 if (LDiff == RDiff)
6190 return getAddExpr(Signed ? getSMinExpr(LS, RS) : getUMinExpr(LS, RS),
6191 LDiff);
6192 }
6193 break;
6194 case ICmpInst::ICMP_NE:
6195 // x != 0 ? x+y : C+y -> x == 0 ? C+y : x+y
6196 std::swap(TrueVal, FalseVal);
6197 [[fallthrough]];
6198 case ICmpInst::ICMP_EQ:
6199 // x == 0 ? C+y : x+y -> umax(x, C)+y iff C u<= 1
6202 const SCEV *X = getNoopOrZeroExtend(getSCEV(LHS), Ty);
6203 const SCEV *TrueValExpr = getSCEV(TrueVal); // C+y
6204 const SCEV *FalseValExpr = getSCEV(FalseVal); // x+y
6205 const SCEV *Y = getMinusSCEV(FalseValExpr, X); // y = (x+y)-x
6206 const SCEV *C = getMinusSCEV(TrueValExpr, Y); // C = (C+y)-y
6207 if (isa<SCEVConstant>(C) && cast<SCEVConstant>(C)->getAPInt().ule(1))
6208 return getAddExpr(getUMaxExpr(X, C), Y);
6209 }
6210 // x == 0 ? 0 : umin (..., x, ...) -> umin_seq(x, umin (...))
6211 // x == 0 ? 0 : umin_seq(..., x, ...) -> umin_seq(x, umin_seq(...))
6212 // x == 0 ? 0 : umin (..., umin_seq(..., x, ...), ...)
6213 // -> umin_seq(x, umin (..., umin_seq(...), ...))
6215 isa<ConstantInt>(TrueVal) && cast<ConstantInt>(TrueVal)->isZero()) {
6216 const SCEV *X = getSCEV(LHS);
6217 while (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(X))
6218 X = ZExt->getOperand();
6219 if (getTypeSizeInBits(X->getType()) <= getTypeSizeInBits(Ty)) {
6220 const SCEV *FalseValExpr = getSCEV(FalseVal);
6221 if (SCEVMinMaxExprContains(FalseValExpr, X, scSequentialUMinExpr))
6222 return getUMinExpr(getNoopOrZeroExtend(X, Ty), FalseValExpr,
6223 /*Sequential=*/true);
6224 }
6225 }
6226 break;
6227 default:
6228 break;
6229 }
6230
6231 return std::nullopt;
6232}
6233
6234static std::optional<const SCEV *>
6236 const SCEV *TrueExpr, const SCEV *FalseExpr) {
6237 assert(CondExpr->getType()->isIntegerTy(1) &&
6238 TrueExpr->getType() == FalseExpr->getType() &&
6239 TrueExpr->getType()->isIntegerTy(1) &&
6240 "Unexpected operands of a select.");
6241
6242 // i1 cond ? i1 x : i1 C --> C + (i1 cond ? (i1 x - i1 C) : i1 0)
6243 // --> C + (umin_seq cond, x - C)
6244 //
6245 // i1 cond ? i1 C : i1 x --> C + (i1 cond ? i1 0 : (i1 x - i1 C))
6246 // --> C + (i1 ~cond ? (i1 x - i1 C) : i1 0)
6247 // --> C + (umin_seq ~cond, x - C)
6248
6249 // FIXME: while we can't legally model the case where both of the hands
6250 // are fully variable, we only require that the *difference* is constant.
6251 if (!isa<SCEVConstant>(TrueExpr) && !isa<SCEVConstant>(FalseExpr))
6252 return std::nullopt;
6253
6254 const SCEV *X, *C;
6255 if (isa<SCEVConstant>(TrueExpr)) {
6256 CondExpr = SE->getNotSCEV(CondExpr);
6257 X = FalseExpr;
6258 C = TrueExpr;
6259 } else {
6260 X = TrueExpr;
6261 C = FalseExpr;
6262 }
6263 return SE->getAddExpr(C, SE->getUMinExpr(CondExpr, SE->getMinusSCEV(X, C),
6264 /*Sequential=*/true));
6265}
6266
6267static std::optional<const SCEV *>
6269 Value *FalseVal) {
6270 if (!isa<ConstantInt>(TrueVal) && !isa<ConstantInt>(FalseVal))
6271 return std::nullopt;
6272
6273 const auto *SECond = SE->getSCEV(Cond);
6274 const auto *SETrue = SE->getSCEV(TrueVal);
6275 const auto *SEFalse = SE->getSCEV(FalseVal);
6276 return createNodeForSelectViaUMinSeq(SE, SECond, SETrue, SEFalse);
6277}
6278
6279const SCEV *ScalarEvolution::createNodeForSelectOrPHIViaUMinSeq(
6280 Value *V, Value *Cond, Value *TrueVal, Value *FalseVal) {
6281 assert(Cond->getType()->isIntegerTy(1) && "Select condition is not an i1?");
6282 assert(TrueVal->getType() == FalseVal->getType() &&
6283 V->getType() == TrueVal->getType() &&
6284 "Types of select hands and of the result must match.");
6285
6286 // For now, only deal with i1-typed `select`s.
6287 if (!V->getType()->isIntegerTy(1))
6288 return getUnknown(V);
6289
6290 if (std::optional<const SCEV *> S =
6291 createNodeForSelectViaUMinSeq(this, Cond, TrueVal, FalseVal))
6292 return *S;
6293
6294 return getUnknown(V);
6295}
6296
6297const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Value *V, Value *Cond,
6298 Value *TrueVal,
6299 Value *FalseVal) {
6300 // Handle "constant" branch or select. This can occur for instance when a
6301 // loop pass transforms an inner loop and moves on to process the outer loop.
6302 if (auto *CI = dyn_cast<ConstantInt>(Cond))
6303 return getSCEV(CI->isOne() ? TrueVal : FalseVal);
6304
6305 if (auto *I = dyn_cast<Instruction>(V)) {
6306 if (auto *ICI = dyn_cast<ICmpInst>(Cond)) {
6307 if (std::optional<const SCEV *> S =
6308 createNodeForSelectOrPHIInstWithICmpInstCond(I->getType(), ICI,
6309 TrueVal, FalseVal))
6310 return *S;
6311 }
6312 }
6313
6314 return createNodeForSelectOrPHIViaUMinSeq(V, Cond, TrueVal, FalseVal);
6315}
6316
6317/// Expand GEP instructions into add and multiply operations. This allows them
6318/// to be analyzed by regular SCEV code.
6319const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
6320 assert(GEP->getSourceElementType()->isSized() &&
6321 "GEP source element type must be sized");
6322
6324 for (Value *Index : GEP->indices())
6325 IndexExprs.push_back(getSCEV(Index));
6326 return getGEPExpr(GEP, IndexExprs);
6327}
6328
6329APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S,
6330 const Instruction *CtxI) {
6331 uint64_t BitWidth = getTypeSizeInBits(S->getType());
6332 auto GetShiftedByZeros = [BitWidth](uint32_t TrailingZeros) {
6333 return TrailingZeros >= BitWidth
6335 : APInt::getOneBitSet(BitWidth, TrailingZeros);
6336 };
6337 auto GetGCDMultiple = [this, CtxI](const SCEVNAryExpr *N) {
6338 // The result is GCD of all operands results.
6339 APInt Res = getConstantMultiple(N->getOperand(0), CtxI);
6340 for (unsigned I = 1, E = N->getNumOperands(); I < E && Res != 1; ++I)
6342 Res, getConstantMultiple(N->getOperand(I), CtxI));
6343 return Res;
6344 };
6345
6346 switch (S->getSCEVType()) {
6347 case scConstant:
6348 return cast<SCEVConstant>(S)->getAPInt();
6349 case scPtrToInt:
6350 return getConstantMultiple(cast<SCEVPtrToIntExpr>(S)->getOperand(), CtxI);
6351 case scUDivExpr:
6352 case scVScale:
6353 return APInt(BitWidth, 1);
6354 case scTruncate: {
6355 // Only multiples that are a power of 2 will hold after truncation.
6356 const SCEVTruncateExpr *T = cast<SCEVTruncateExpr>(S);
6357 uint32_t TZ = getMinTrailingZeros(T->getOperand(), CtxI);
6358 return GetShiftedByZeros(TZ);
6359 }
6360 case scZeroExtend: {
6361 const SCEVZeroExtendExpr *Z = cast<SCEVZeroExtendExpr>(S);
6362 return getConstantMultiple(Z->getOperand(), CtxI).zext(BitWidth);
6363 }
6364 case scSignExtend: {
6365 // Only multiples that are a power of 2 will hold after sext.
6366 const SCEVSignExtendExpr *E = cast<SCEVSignExtendExpr>(S);
6367 uint32_t TZ = getMinTrailingZeros(E->getOperand(), CtxI);
6368 return GetShiftedByZeros(TZ);
6369 }
6370 case scMulExpr: {
6371 const SCEVMulExpr *M = cast<SCEVMulExpr>(S);
6372 if (M->hasNoUnsignedWrap()) {
6373 // The result is the product of all operand results.
6374 APInt Res = getConstantMultiple(M->getOperand(0), CtxI);
6375 for (const SCEV *Operand : M->operands().drop_front())
6376 Res = Res * getConstantMultiple(Operand, CtxI);
6377 return Res;
6378 }
6379
6380 // If there are no wrap guarentees, find the trailing zeros, which is the
6381 // sum of trailing zeros for all its operands.
6382 uint32_t TZ = 0;
6383 for (const SCEV *Operand : M->operands())
6384 TZ += getMinTrailingZeros(Operand, CtxI);
6385 return GetShiftedByZeros(TZ);
6386 }
6387 case scAddExpr:
6388 case scAddRecExpr: {
6389 const SCEVNAryExpr *N = cast<SCEVNAryExpr>(S);
6390 if (N->hasNoUnsignedWrap())
6391 return GetGCDMultiple(N);
6392 // Find the trailing bits, which is the minimum of its operands.
6393 uint32_t TZ = getMinTrailingZeros(N->getOperand(0), CtxI);
6394 for (const SCEV *Operand : N->operands().drop_front())
6395 TZ = std::min(TZ, getMinTrailingZeros(Operand, CtxI));
6396 return GetShiftedByZeros(TZ);
6397 }
6398 case scUMaxExpr:
6399 case scSMaxExpr:
6400 case scUMinExpr:
6401 case scSMinExpr:
6403 return GetGCDMultiple(cast<SCEVNAryExpr>(S));
6404 case scUnknown: {
6405 // Ask ValueTracking for known bits. SCEVUnknown only become available at
6406 // the point their underlying IR instruction has been defined. If CtxI was
6407 // not provided, use:
6408 // * the first instruction in the entry block if it is an argument
6409 // * the instruction itself otherwise.
6410 const SCEVUnknown *U = cast<SCEVUnknown>(S);
6411 if (!CtxI) {
6412 if (isa<Argument>(U->getValue()))
6413 CtxI = &*F.getEntryBlock().begin();
6414 else if (auto *I = dyn_cast<Instruction>(U->getValue()))
6415 CtxI = I;
6416 }
6417 unsigned Known =
6418 computeKnownBits(U->getValue(), getDataLayout(), &AC, CtxI, &DT)
6419 .countMinTrailingZeros();
6420 return GetShiftedByZeros(Known);
6421 }
6422 case scCouldNotCompute:
6423 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6424 }
6425 llvm_unreachable("Unknown SCEV kind!");
6426}
6427
6429 const Instruction *CtxI) {
6430 // Skip looking up and updating the cache if there is a context instruction,
6431 // as the result will only be valid in the specified context.
6432 if (CtxI)
6433 return getConstantMultipleImpl(S, CtxI);
6434
6435 auto I = ConstantMultipleCache.find(S);
6436 if (I != ConstantMultipleCache.end())
6437 return I->second;
6438
6439 APInt Result = getConstantMultipleImpl(S, CtxI);
6440 auto InsertPair = ConstantMultipleCache.insert({S, Result});
6441 assert(InsertPair.second && "Should insert a new key");
6442 return InsertPair.first->second;
6443}
6444
6446 APInt Multiple = getConstantMultiple(S);
6447 return Multiple == 0 ? APInt(Multiple.getBitWidth(), 1) : Multiple;
6448}
6449
6451 const Instruction *CtxI) {
6452 return std::min(getConstantMultiple(S, CtxI).countTrailingZeros(),
6453 (unsigned)getTypeSizeInBits(S->getType()));
6454}
6455
6456/// Helper method to assign a range to V from metadata present in the IR.
6457static std::optional<ConstantRange> GetRangeFromMetadata(Value *V) {
6459 if (MDNode *MD = I->getMetadata(LLVMContext::MD_range))
6460 return getConstantRangeFromMetadata(*MD);
6461 if (const auto *CB = dyn_cast<CallBase>(V))
6462 if (std::optional<ConstantRange> Range = CB->getRange())
6463 return Range;
6464 }
6465 if (auto *A = dyn_cast<Argument>(V))
6466 if (std::optional<ConstantRange> Range = A->getRange())
6467 return Range;
6468
6469 return std::nullopt;
6470}
6471
6473 SCEV::NoWrapFlags Flags) {
6474 if (AddRec->getNoWrapFlags(Flags) != Flags) {
6475 AddRec->setNoWrapFlags(Flags);
6476 UnsignedRanges.erase(AddRec);
6477 SignedRanges.erase(AddRec);
6478 ConstantMultipleCache.erase(AddRec);
6479 }
6480}
6481
6482ConstantRange ScalarEvolution::
6483getRangeForUnknownRecurrence(const SCEVUnknown *U) {
6484 const DataLayout &DL = getDataLayout();
6485
6486 unsigned BitWidth = getTypeSizeInBits(U->getType());
6487 const ConstantRange FullSet(BitWidth, /*isFullSet=*/true);
6488
6489 // Match a simple recurrence of the form: <start, ShiftOp, Step>, and then
6490 // use information about the trip count to improve our available range. Note
6491 // that the trip count independent cases are already handled by known bits.
6492 // WARNING: The definition of recurrence used here is subtly different than
6493 // the one used by AddRec (and thus most of this file). Step is allowed to
6494 // be arbitrarily loop varying here, where AddRec allows only loop invariant
6495 // and other addrecs in the same loop (for non-affine addrecs). The code
6496 // below intentionally handles the case where step is not loop invariant.
6497 auto *P = dyn_cast<PHINode>(U->getValue());
6498 if (!P)
6499 return FullSet;
6500
6501 // Make sure that no Phi input comes from an unreachable block. Otherwise,
6502 // even the values that are not available in these blocks may come from them,
6503 // and this leads to false-positive recurrence test.
6504 for (auto *Pred : predecessors(P->getParent()))
6505 if (!DT.isReachableFromEntry(Pred))
6506 return FullSet;
6507
6508 BinaryOperator *BO;
6509 Value *Start, *Step;
6510 if (!matchSimpleRecurrence(P, BO, Start, Step))
6511 return FullSet;
6512
6513 // If we found a recurrence in reachable code, we must be in a loop. Note
6514 // that BO might be in some subloop of L, and that's completely okay.
6515 auto *L = LI.getLoopFor(P->getParent());
6516 assert(L && L->getHeader() == P->getParent());
6517 if (!L->contains(BO->getParent()))
6518 // NOTE: This bailout should be an assert instead. However, asserting
6519 // the condition here exposes a case where LoopFusion is querying SCEV
6520 // with malformed loop information during the midst of the transform.
6521 // There doesn't appear to be an obvious fix, so for the moment bailout
6522 // until the caller issue can be fixed. PR49566 tracks the bug.
6523 return FullSet;
6524
6525 // TODO: Extend to other opcodes such as mul, and div
6526 switch (BO->getOpcode()) {
6527 default:
6528 return FullSet;
6529 case Instruction::AShr:
6530 case Instruction::LShr:
6531 case Instruction::Shl:
6532 break;
6533 };
6534
6535 if (BO->getOperand(0) != P)
6536 // TODO: Handle the power function forms some day.
6537 return FullSet;
6538
6539 unsigned TC = getSmallConstantMaxTripCount(L);
6540 if (!TC || TC >= BitWidth)
6541 return FullSet;
6542
6543 auto KnownStart = computeKnownBits(Start, DL, &AC, nullptr, &DT);
6544 auto KnownStep = computeKnownBits(Step, DL, &AC, nullptr, &DT);
6545 assert(KnownStart.getBitWidth() == BitWidth &&
6546 KnownStep.getBitWidth() == BitWidth);
6547
6548 // Compute total shift amount, being careful of overflow and bitwidths.
6549 auto MaxShiftAmt = KnownStep.getMaxValue();
6550 APInt TCAP(BitWidth, TC-1);
6551 bool Overflow = false;
6552 auto TotalShift = MaxShiftAmt.umul_ov(TCAP, Overflow);
6553 if (Overflow)
6554 return FullSet;
6555
6556 switch (BO->getOpcode()) {
6557 default:
6558 llvm_unreachable("filtered out above");
6559 case Instruction::AShr: {
6560 // For each ashr, three cases:
6561 // shift = 0 => unchanged value
6562 // saturation => 0 or -1
6563 // other => a value closer to zero (of the same sign)
6564 // Thus, the end value is closer to zero than the start.
6565 auto KnownEnd = KnownBits::ashr(KnownStart,
6566 KnownBits::makeConstant(TotalShift));
6567 if (KnownStart.isNonNegative())
6568 // Analogous to lshr (simply not yet canonicalized)
6569 return ConstantRange::getNonEmpty(KnownEnd.getMinValue(),
6570 KnownStart.getMaxValue() + 1);
6571 if (KnownStart.isNegative())
6572 // End >=u Start && End <=s Start
6573 return ConstantRange::getNonEmpty(KnownStart.getMinValue(),
6574 KnownEnd.getMaxValue() + 1);
6575 break;
6576 }
6577 case Instruction::LShr: {
6578 // For each lshr, three cases:
6579 // shift = 0 => unchanged value
6580 // saturation => 0
6581 // other => a smaller positive number
6582 // Thus, the low end of the unsigned range is the last value produced.
6583 auto KnownEnd = KnownBits::lshr(KnownStart,
6584 KnownBits::makeConstant(TotalShift));
6585 return ConstantRange::getNonEmpty(KnownEnd.getMinValue(),
6586 KnownStart.getMaxValue() + 1);
6587 }
6588 case Instruction::Shl: {
6589 // Iff no bits are shifted out, value increases on every shift.
6590 auto KnownEnd = KnownBits::shl(KnownStart,
6591 KnownBits::makeConstant(TotalShift));
6592 if (TotalShift.ult(KnownStart.countMinLeadingZeros()))
6593 return ConstantRange(KnownStart.getMinValue(),
6594 KnownEnd.getMaxValue() + 1);
6595 break;
6596 }
6597 };
6598 return FullSet;
6599}
6600
6601const ConstantRange &
6602ScalarEvolution::getRangeRefIter(const SCEV *S,
6603 ScalarEvolution::RangeSignHint SignHint) {
6604 DenseMap<const SCEV *, ConstantRange> &Cache =
6605 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6606 : SignedRanges;
6608 SmallPtrSet<const SCEV *, 8> Seen;
6609
6610 // Add Expr to the worklist, if Expr is either an N-ary expression or a
6611 // SCEVUnknown PHI node.
6612 auto AddToWorklist = [&WorkList, &Seen, &Cache](const SCEV *Expr) {
6613 if (!Seen.insert(Expr).second)
6614 return;
6615 if (Cache.contains(Expr))
6616 return;
6617 switch (Expr->getSCEVType()) {
6618 case scUnknown:
6619 if (!isa<PHINode>(cast<SCEVUnknown>(Expr)->getValue()))
6620 break;
6621 [[fallthrough]];
6622 case scConstant:
6623 case scVScale:
6624 case scTruncate:
6625 case scZeroExtend:
6626 case scSignExtend:
6627 case scPtrToInt:
6628 case scAddExpr:
6629 case scMulExpr:
6630 case scUDivExpr:
6631 case scAddRecExpr:
6632 case scUMaxExpr:
6633 case scSMaxExpr:
6634 case scUMinExpr:
6635 case scSMinExpr:
6637 WorkList.push_back(Expr);
6638 break;
6639 case scCouldNotCompute:
6640 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6641 }
6642 };
6643 AddToWorklist(S);
6644
6645 // Build worklist by queuing operands of N-ary expressions and phi nodes.
6646 for (unsigned I = 0; I != WorkList.size(); ++I) {
6647 const SCEV *P = WorkList[I];
6648 auto *UnknownS = dyn_cast<SCEVUnknown>(P);
6649 // If it is not a `SCEVUnknown`, just recurse into operands.
6650 if (!UnknownS) {
6651 for (const SCEV *Op : P->operands())
6652 AddToWorklist(Op);
6653 continue;
6654 }
6655 // `SCEVUnknown`'s require special treatment.
6656 if (const PHINode *P = dyn_cast<PHINode>(UnknownS->getValue())) {
6657 if (!PendingPhiRangesIter.insert(P).second)
6658 continue;
6659 for (auto &Op : reverse(P->operands()))
6660 AddToWorklist(getSCEV(Op));
6661 }
6662 }
6663
6664 if (!WorkList.empty()) {
6665 // Use getRangeRef to compute ranges for items in the worklist in reverse
6666 // order. This will force ranges for earlier operands to be computed before
6667 // their users in most cases.
6668 for (const SCEV *P : reverse(drop_begin(WorkList))) {
6669 getRangeRef(P, SignHint);
6670
6671 if (auto *UnknownS = dyn_cast<SCEVUnknown>(P))
6672 if (const PHINode *P = dyn_cast<PHINode>(UnknownS->getValue()))
6673 PendingPhiRangesIter.erase(P);
6674 }
6675 }
6676
6677 return getRangeRef(S, SignHint, 0);
6678}
6679
6680/// Determine the range for a particular SCEV. If SignHint is
6681/// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges
6682/// with a "cleaner" unsigned (resp. signed) representation.
6683const ConstantRange &ScalarEvolution::getRangeRef(
6684 const SCEV *S, ScalarEvolution::RangeSignHint SignHint, unsigned Depth) {
6685 DenseMap<const SCEV *, ConstantRange> &Cache =
6686 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6687 : SignedRanges;
6689 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? ConstantRange::Unsigned
6691
6692 // See if we've computed this range already.
6694 if (I != Cache.end())
6695 return I->second;
6696
6697 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
6698 return setRange(C, SignHint, ConstantRange(C->getAPInt()));
6699
6700 // Switch to iteratively computing the range for S, if it is part of a deeply
6701 // nested expression.
6703 return getRangeRefIter(S, SignHint);
6704
6705 unsigned BitWidth = getTypeSizeInBits(S->getType());
6706 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
6707 using OBO = OverflowingBinaryOperator;
6708
6709 // If the value has known zeros, the maximum value will have those known zeros
6710 // as well.
6711 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
6712 APInt Multiple = getNonZeroConstantMultiple(S);
6713 APInt Remainder = APInt::getMaxValue(BitWidth).urem(Multiple);
6714 if (!Remainder.isZero())
6715 ConservativeResult =
6716 ConstantRange(APInt::getMinValue(BitWidth),
6717 APInt::getMaxValue(BitWidth) - Remainder + 1);
6718 }
6719 else {
6720 uint32_t TZ = getMinTrailingZeros(S);
6721 if (TZ != 0) {
6722 ConservativeResult = ConstantRange(
6724 APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1);
6725 }
6726 }
6727
6728 switch (S->getSCEVType()) {
6729 case scConstant:
6730 llvm_unreachable("Already handled above.");
6731 case scVScale:
6732 return setRange(S, SignHint, getVScaleRange(&F, BitWidth));
6733 case scTruncate: {
6734 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(S);
6735 ConstantRange X = getRangeRef(Trunc->getOperand(), SignHint, Depth + 1);
6736 return setRange(
6737 Trunc, SignHint,
6738 ConservativeResult.intersectWith(X.truncate(BitWidth), RangeType));
6739 }
6740 case scZeroExtend: {
6741 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(S);
6742 ConstantRange X = getRangeRef(ZExt->getOperand(), SignHint, Depth + 1);
6743 return setRange(
6744 ZExt, SignHint,
6745 ConservativeResult.intersectWith(X.zeroExtend(BitWidth), RangeType));
6746 }
6747 case scSignExtend: {
6748 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(S);
6749 ConstantRange X = getRangeRef(SExt->getOperand(), SignHint, Depth + 1);
6750 return setRange(
6751 SExt, SignHint,
6752 ConservativeResult.intersectWith(X.signExtend(BitWidth), RangeType));
6753 }
6754 case scPtrToInt: {
6755 const SCEVPtrToIntExpr *PtrToInt = cast<SCEVPtrToIntExpr>(S);
6756 ConstantRange X = getRangeRef(PtrToInt->getOperand(), SignHint, Depth + 1);
6757 return setRange(PtrToInt, SignHint, X);
6758 }
6759 case scAddExpr: {
6760 const SCEVAddExpr *Add = cast<SCEVAddExpr>(S);
6761 ConstantRange X = getRangeRef(Add->getOperand(0), SignHint, Depth + 1);
6762 unsigned WrapType = OBO::AnyWrap;
6763 if (Add->hasNoSignedWrap())
6764 WrapType |= OBO::NoSignedWrap;
6765 if (Add->hasNoUnsignedWrap())
6766 WrapType |= OBO::NoUnsignedWrap;
6767 for (const SCEV *Op : drop_begin(Add->operands()))
6768 X = X.addWithNoWrap(getRangeRef(Op, SignHint, Depth + 1), WrapType,
6769 RangeType);
6770 return setRange(Add, SignHint,
6771 ConservativeResult.intersectWith(X, RangeType));
6772 }
6773 case scMulExpr: {
6774 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(S);
6775 ConstantRange X = getRangeRef(Mul->getOperand(0), SignHint, Depth + 1);
6776 for (const SCEV *Op : drop_begin(Mul->operands()))
6777 X = X.multiply(getRangeRef(Op, SignHint, Depth + 1));
6778 return setRange(Mul, SignHint,
6779 ConservativeResult.intersectWith(X, RangeType));
6780 }
6781 case scUDivExpr: {
6782 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S);
6783 ConstantRange X = getRangeRef(UDiv->getLHS(), SignHint, Depth + 1);
6784 ConstantRange Y = getRangeRef(UDiv->getRHS(), SignHint, Depth + 1);
6785 return setRange(UDiv, SignHint,
6786 ConservativeResult.intersectWith(X.udiv(Y), RangeType));
6787 }
6788 case scAddRecExpr: {
6789 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(S);
6790 // If there's no unsigned wrap, the value will never be less than its
6791 // initial value.
6792 if (AddRec->hasNoUnsignedWrap()) {
6793 APInt UnsignedMinValue = getUnsignedRangeMin(AddRec->getStart());
6794 if (!UnsignedMinValue.isZero())
6795 ConservativeResult = ConservativeResult.intersectWith(
6796 ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), RangeType);
6797 }
6798
6799 // If there's no signed wrap, and all the operands except initial value have
6800 // the same sign or zero, the value won't ever be:
6801 // 1: smaller than initial value if operands are non negative,
6802 // 2: bigger than initial value if operands are non positive.
6803 // For both cases, value can not cross signed min/max boundary.
6804 if (AddRec->hasNoSignedWrap()) {
6805 bool AllNonNeg = true;
6806 bool AllNonPos = true;
6807 for (unsigned i = 1, e = AddRec->getNumOperands(); i != e; ++i) {
6808 if (!isKnownNonNegative(AddRec->getOperand(i)))
6809 AllNonNeg = false;
6810 if (!isKnownNonPositive(AddRec->getOperand(i)))
6811 AllNonPos = false;
6812 }
6813 if (AllNonNeg)
6814 ConservativeResult = ConservativeResult.intersectWith(
6817 RangeType);
6818 else if (AllNonPos)
6819 ConservativeResult = ConservativeResult.intersectWith(
6821 getSignedRangeMax(AddRec->getStart()) +
6822 1),
6823 RangeType);
6824 }
6825
6826 // TODO: non-affine addrec
6827 if (AddRec->isAffine()) {
6828 const SCEV *MaxBEScev =
6830 if (!isa<SCEVCouldNotCompute>(MaxBEScev)) {
6831 APInt MaxBECount = cast<SCEVConstant>(MaxBEScev)->getAPInt();
6832
6833 // Adjust MaxBECount to the same bitwidth as AddRec. We can truncate if
6834 // MaxBECount's active bits are all <= AddRec's bit width.
6835 if (MaxBECount.getBitWidth() > BitWidth &&
6836 MaxBECount.getActiveBits() <= BitWidth)
6837 MaxBECount = MaxBECount.trunc(BitWidth);
6838 else if (MaxBECount.getBitWidth() < BitWidth)
6839 MaxBECount = MaxBECount.zext(BitWidth);
6840
6841 if (MaxBECount.getBitWidth() == BitWidth) {
6842 auto RangeFromAffine = getRangeForAffineAR(
6843 AddRec->getStart(), AddRec->getStepRecurrence(*this), MaxBECount);
6844 ConservativeResult =
6845 ConservativeResult.intersectWith(RangeFromAffine, RangeType);
6846
6847 auto RangeFromFactoring = getRangeViaFactoring(
6848 AddRec->getStart(), AddRec->getStepRecurrence(*this), MaxBECount);
6849 ConservativeResult =
6850 ConservativeResult.intersectWith(RangeFromFactoring, RangeType);
6851 }
6852 }
6853
6854 // Now try symbolic BE count and more powerful methods.
6856 const SCEV *SymbolicMaxBECount =
6858 if (!isa<SCEVCouldNotCompute>(SymbolicMaxBECount) &&
6859 getTypeSizeInBits(MaxBEScev->getType()) <= BitWidth &&
6860 AddRec->hasNoSelfWrap()) {
6861 auto RangeFromAffineNew = getRangeForAffineNoSelfWrappingAR(
6862 AddRec, SymbolicMaxBECount, BitWidth, SignHint);
6863 ConservativeResult =
6864 ConservativeResult.intersectWith(RangeFromAffineNew, RangeType);
6865 }
6866 }
6867 }
6868
6869 return setRange(AddRec, SignHint, std::move(ConservativeResult));
6870 }
6871 case scUMaxExpr:
6872 case scSMaxExpr:
6873 case scUMinExpr:
6874 case scSMinExpr:
6875 case scSequentialUMinExpr: {
6877 switch (S->getSCEVType()) {
6878 case scUMaxExpr:
6879 ID = Intrinsic::umax;
6880 break;
6881 case scSMaxExpr:
6882 ID = Intrinsic::smax;
6883 break;
6884 case scUMinExpr:
6886 ID = Intrinsic::umin;
6887 break;
6888 case scSMinExpr:
6889 ID = Intrinsic::smin;
6890 break;
6891 default:
6892 llvm_unreachable("Unknown SCEVMinMaxExpr/SCEVSequentialMinMaxExpr.");
6893 }
6894
6895 const auto *NAry = cast<SCEVNAryExpr>(S);
6896 ConstantRange X = getRangeRef(NAry->getOperand(0), SignHint, Depth + 1);
6897 for (unsigned i = 1, e = NAry->getNumOperands(); i != e; ++i)
6898 X = X.intrinsic(
6899 ID, {X, getRangeRef(NAry->getOperand(i), SignHint, Depth + 1)});
6900 return setRange(S, SignHint,
6901 ConservativeResult.intersectWith(X, RangeType));
6902 }
6903 case scUnknown: {
6904 const SCEVUnknown *U = cast<SCEVUnknown>(S);
6905 Value *V = U->getValue();
6906
6907 // Check if the IR explicitly contains !range metadata.
6908 std::optional<ConstantRange> MDRange = GetRangeFromMetadata(V);
6909 if (MDRange)
6910 ConservativeResult =
6911 ConservativeResult.intersectWith(*MDRange, RangeType);
6912
6913 // Use facts about recurrences in the underlying IR. Note that add
6914 // recurrences are AddRecExprs and thus don't hit this path. This
6915 // primarily handles shift recurrences.
6916 auto CR = getRangeForUnknownRecurrence(U);
6917 ConservativeResult = ConservativeResult.intersectWith(CR);
6918
6919 // See if ValueTracking can give us a useful range.
6920 const DataLayout &DL = getDataLayout();
6921 KnownBits Known = computeKnownBits(V, DL, &AC, nullptr, &DT);
6922 if (Known.getBitWidth() != BitWidth)
6923 Known = Known.zextOrTrunc(BitWidth);
6924
6925 // ValueTracking may be able to compute a tighter result for the number of
6926 // sign bits than for the value of those sign bits.
6927 unsigned NS = ComputeNumSignBits(V, DL, &AC, nullptr, &DT);
6928 if (U->getType()->isPointerTy()) {
6929 // If the pointer size is larger than the index size type, this can cause
6930 // NS to be larger than BitWidth. So compensate for this.
6931 unsigned ptrSize = DL.getPointerTypeSizeInBits(U->getType());
6932 int ptrIdxDiff = ptrSize - BitWidth;
6933 if (ptrIdxDiff > 0 && ptrSize > BitWidth && NS > (unsigned)ptrIdxDiff)
6934 NS -= ptrIdxDiff;
6935 }
6936
6937 if (NS > 1) {
6938 // If we know any of the sign bits, we know all of the sign bits.
6939 if (!Known.Zero.getHiBits(NS).isZero())
6940 Known.Zero.setHighBits(NS);
6941 if (!Known.One.getHiBits(NS).isZero())
6942 Known.One.setHighBits(NS);
6943 }
6944
6945 if (Known.getMinValue() != Known.getMaxValue() + 1)
6946 ConservativeResult = ConservativeResult.intersectWith(
6947 ConstantRange(Known.getMinValue(), Known.getMaxValue() + 1),
6948 RangeType);
6949 if (NS > 1)
6950 ConservativeResult = ConservativeResult.intersectWith(
6951 ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1),
6952 APInt::getSignedMaxValue(BitWidth).ashr(NS - 1) + 1),
6953 RangeType);
6954
6955 if (U->getType()->isPointerTy() && SignHint == HINT_RANGE_UNSIGNED) {
6956 // Strengthen the range if the underlying IR value is a
6957 // global/alloca/heap allocation using the size of the object.
6958 bool CanBeNull, CanBeFreed;
6959 uint64_t DerefBytes =
6960 V->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
6961 if (DerefBytes > 1 && isUIntN(BitWidth, DerefBytes)) {
6962 // The highest address the object can start is DerefBytes bytes before
6963 // the end (unsigned max value). If this value is not a multiple of the
6964 // alignment, the last possible start value is the next lowest multiple
6965 // of the alignment. Note: The computations below cannot overflow,
6966 // because if they would there's no possible start address for the
6967 // object.
6968 APInt MaxVal =
6969 APInt::getMaxValue(BitWidth) - APInt(BitWidth, DerefBytes);
6970 uint64_t Align = U->getValue()->getPointerAlignment(DL).value();
6971 uint64_t Rem = MaxVal.urem(Align);
6972 MaxVal -= APInt(BitWidth, Rem);
6973 APInt MinVal = APInt::getZero(BitWidth);
6974 if (llvm::isKnownNonZero(V, DL))
6975 MinVal = Align;
6976 ConservativeResult = ConservativeResult.intersectWith(
6977 ConstantRange::getNonEmpty(MinVal, MaxVal + 1), RangeType);
6978 }
6979 }
6980
6981 // A range of Phi is a subset of union of all ranges of its input.
6982 if (PHINode *Phi = dyn_cast<PHINode>(V)) {
6983 // Make sure that we do not run over cycled Phis.
6984 if (PendingPhiRanges.insert(Phi).second) {
6985 ConstantRange RangeFromOps(BitWidth, /*isFullSet=*/false);
6986
6987 for (const auto &Op : Phi->operands()) {
6988 auto OpRange = getRangeRef(getSCEV(Op), SignHint, Depth + 1);
6989 RangeFromOps = RangeFromOps.unionWith(OpRange);
6990 // No point to continue if we already have a full set.
6991 if (RangeFromOps.isFullSet())
6992 break;
6993 }
6994 ConservativeResult =
6995 ConservativeResult.intersectWith(RangeFromOps, RangeType);
6996 bool Erased = PendingPhiRanges.erase(Phi);
6997 assert(Erased && "Failed to erase Phi properly?");
6998 (void)Erased;
6999 }
7000 }
7001
7002 // vscale can't be equal to zero
7003 if (const auto *II = dyn_cast<IntrinsicInst>(V))
7004 if (II->getIntrinsicID() == Intrinsic::vscale) {
7005 ConstantRange Disallowed = APInt::getZero(BitWidth);
7006 ConservativeResult = ConservativeResult.difference(Disallowed);
7007 }
7008
7009 return setRange(U, SignHint, std::move(ConservativeResult));
7010 }
7011 case scCouldNotCompute:
7012 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
7013 }
7014
7015 return setRange(S, SignHint, std::move(ConservativeResult));
7016}
7017
7018// Given a StartRange, Step and MaxBECount for an expression compute a range of
7019// values that the expression can take. Initially, the expression has a value
7020// from StartRange and then is changed by Step up to MaxBECount times. Signed
7021// argument defines if we treat Step as signed or unsigned.
7023 const ConstantRange &StartRange,
7024 const APInt &MaxBECount,
7025 bool Signed) {
7026 unsigned BitWidth = Step.getBitWidth();
7027 assert(BitWidth == StartRange.getBitWidth() &&
7028 BitWidth == MaxBECount.getBitWidth() && "mismatched bit widths");
7029 // If either Step or MaxBECount is 0, then the expression won't change, and we
7030 // just need to return the initial range.
7031 if (Step == 0 || MaxBECount == 0)
7032 return StartRange;
7033
7034 // If we don't know anything about the initial value (i.e. StartRange is
7035 // FullRange), then we don't know anything about the final range either.
7036 // Return FullRange.
7037 if (StartRange.isFullSet())
7038 return ConstantRange::getFull(BitWidth);
7039
7040 // If Step is signed and negative, then we use its absolute value, but we also
7041 // note that we're moving in the opposite direction.
7042 bool Descending = Signed && Step.isNegative();
7043
7044 if (Signed)
7045 // This is correct even for INT_SMIN. Let's look at i8 to illustrate this:
7046 // abs(INT_SMIN) = abs(-128) = abs(0x80) = -0x80 = 0x80 = 128.
7047 // This equations hold true due to the well-defined wrap-around behavior of
7048 // APInt.
7049 Step = Step.abs();
7050
7051 // Check if Offset is more than full span of BitWidth. If it is, the
7052 // expression is guaranteed to overflow.
7053 if (APInt::getMaxValue(StartRange.getBitWidth()).udiv(Step).ult(MaxBECount))
7054 return ConstantRange::getFull(BitWidth);
7055
7056 // Offset is by how much the expression can change. Checks above guarantee no
7057 // overflow here.
7058 APInt Offset = Step * MaxBECount;
7059
7060 // Minimum value of the final range will match the minimal value of StartRange
7061 // if the expression is increasing and will be decreased by Offset otherwise.
7062 // Maximum value of the final range will match the maximal value of StartRange
7063 // if the expression is decreasing and will be increased by Offset otherwise.
7064 APInt StartLower = StartRange.getLower();
7065 APInt StartUpper = StartRange.getUpper() - 1;
7066 APInt MovedBoundary = Descending ? (StartLower - std::move(Offset))
7067 : (StartUpper + std::move(Offset));
7068
7069 // It's possible that the new minimum/maximum value will fall into the initial
7070 // range (due to wrap around). This means that the expression can take any
7071 // value in this bitwidth, and we have to return full range.
7072 if (StartRange.contains(MovedBoundary))
7073 return ConstantRange::getFull(BitWidth);
7074
7075 APInt NewLower =
7076 Descending ? std::move(MovedBoundary) : std::move(StartLower);
7077 APInt NewUpper =
7078 Descending ? std::move(StartUpper) : std::move(MovedBoundary);
7079 NewUpper += 1;
7080
7081 // No overflow detected, return [StartLower, StartUpper + Offset + 1) range.
7082 return ConstantRange::getNonEmpty(std::move(NewLower), std::move(NewUpper));
7083}
7084
7085ConstantRange ScalarEvolution::getRangeForAffineAR(const SCEV *Start,
7086 const SCEV *Step,
7087 const APInt &MaxBECount) {
7088 assert(getTypeSizeInBits(Start->getType()) ==
7089 getTypeSizeInBits(Step->getType()) &&
7090 getTypeSizeInBits(Start->getType()) == MaxBECount.getBitWidth() &&
7091 "mismatched bit widths");
7092
7093 // First, consider step signed.
7094 ConstantRange StartSRange = getSignedRange(Start);
7095 ConstantRange StepSRange = getSignedRange(Step);
7096
7097 // If Step can be both positive and negative, we need to find ranges for the
7098 // maximum absolute step values in both directions and union them.
7099 ConstantRange SR = getRangeForAffineARHelper(
7100 StepSRange.getSignedMin(), StartSRange, MaxBECount, /* Signed = */ true);
7102 StartSRange, MaxBECount,
7103 /* Signed = */ true));
7104
7105 // Next, consider step unsigned.
7106 ConstantRange UR = getRangeForAffineARHelper(
7107 getUnsignedRangeMax(Step), getUnsignedRange(Start), MaxBECount,
7108 /* Signed = */ false);
7109
7110 // Finally, intersect signed and unsigned ranges.
7112}
7113
7114ConstantRange ScalarEvolution::getRangeForAffineNoSelfWrappingAR(
7115 const SCEVAddRecExpr *AddRec, const SCEV *MaxBECount, unsigned BitWidth,
7116 ScalarEvolution::RangeSignHint SignHint) {
7117 assert(AddRec->isAffine() && "Non-affine AddRecs are not suppored!\n");
7118 assert(AddRec->hasNoSelfWrap() &&
7119 "This only works for non-self-wrapping AddRecs!");
7120 const bool IsSigned = SignHint == HINT_RANGE_SIGNED;
7121 const SCEV *Step = AddRec->getStepRecurrence(*this);
7122 // Only deal with constant step to save compile time.
7123 if (!isa<SCEVConstant>(Step))
7124 return ConstantRange::getFull(BitWidth);
7125 // Let's make sure that we can prove that we do not self-wrap during
7126 // MaxBECount iterations. We need this because MaxBECount is a maximum
7127 // iteration count estimate, and we might infer nw from some exit for which we
7128 // do not know max exit count (or any other side reasoning).
7129 // TODO: Turn into assert at some point.
7130 if (getTypeSizeInBits(MaxBECount->getType()) >
7131 getTypeSizeInBits(AddRec->getType()))
7132 return ConstantRange::getFull(BitWidth);
7133 MaxBECount = getNoopOrZeroExtend(MaxBECount, AddRec->getType());
7134 const SCEV *RangeWidth = getMinusOne(AddRec->getType());
7135 const SCEV *StepAbs = getUMinExpr(Step, getNegativeSCEV(Step));
7136 const SCEV *MaxItersWithoutWrap = getUDivExpr(RangeWidth, StepAbs);
7137 if (!isKnownPredicateViaConstantRanges(ICmpInst::ICMP_ULE, MaxBECount,
7138 MaxItersWithoutWrap))
7139 return ConstantRange::getFull(BitWidth);
7140
7141 ICmpInst::Predicate LEPred =
7143 ICmpInst::Predicate GEPred =
7145 const SCEV *End = AddRec->evaluateAtIteration(MaxBECount, *this);
7146
7147 // We know that there is no self-wrap. Let's take Start and End values and
7148 // look at all intermediate values V1, V2, ..., Vn that IndVar takes during
7149 // the iteration. They either lie inside the range [Min(Start, End),
7150 // Max(Start, End)] or outside it:
7151 //
7152 // Case 1: RangeMin ... Start V1 ... VN End ... RangeMax;
7153 // Case 2: RangeMin Vk ... V1 Start ... End Vn ... Vk + 1 RangeMax;
7154 //
7155 // No self wrap flag guarantees that the intermediate values cannot be BOTH
7156 // outside and inside the range [Min(Start, End), Max(Start, End)]. Using that
7157 // knowledge, let's try to prove that we are dealing with Case 1. It is so if
7158 // Start <= End and step is positive, or Start >= End and step is negative.
7159 const SCEV *Start = applyLoopGuards(AddRec->getStart(), AddRec->getLoop());
7160 ConstantRange StartRange = getRangeRef(Start, SignHint);
7161 ConstantRange EndRange = getRangeRef(End, SignHint);
7162 ConstantRange RangeBetween = StartRange.unionWith(EndRange);
7163 // If they already cover full iteration space, we will know nothing useful
7164 // even if we prove what we want to prove.
7165 if (RangeBetween.isFullSet())
7166 return RangeBetween;
7167 // Only deal with ranges that do not wrap (i.e. RangeMin < RangeMax).
7168 bool IsWrappedSet = IsSigned ? RangeBetween.isSignWrappedSet()
7169 : RangeBetween.isWrappedSet();
7170 if (IsWrappedSet)
7171 return ConstantRange::getFull(BitWidth);
7172
7173 if (isKnownPositive(Step) &&
7174 isKnownPredicateViaConstantRanges(LEPred, Start, End))
7175 return RangeBetween;
7176 if (isKnownNegative(Step) &&
7177 isKnownPredicateViaConstantRanges(GEPred, Start, End))
7178 return RangeBetween;
7179 return ConstantRange::getFull(BitWidth);
7180}
7181
7182ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start,
7183 const SCEV *Step,
7184 const APInt &MaxBECount) {
7185 // RangeOf({C?A:B,+,C?P:Q}) == RangeOf(C?{A,+,P}:{B,+,Q})
7186 // == RangeOf({A,+,P}) union RangeOf({B,+,Q})
7187
7188 unsigned BitWidth = MaxBECount.getBitWidth();
7189 assert(getTypeSizeInBits(Start->getType()) == BitWidth &&
7190 getTypeSizeInBits(Step->getType()) == BitWidth &&
7191 "mismatched bit widths");
7192
7193 struct SelectPattern {
7194 Value *Condition = nullptr;
7195 APInt TrueValue;
7196 APInt FalseValue;
7197
7198 explicit SelectPattern(ScalarEvolution &SE, unsigned BitWidth,
7199 const SCEV *S) {
7200 std::optional<unsigned> CastOp;
7201 APInt Offset(BitWidth, 0);
7202
7204 "Should be!");
7205
7206 // Peel off a constant offset. In the future we could consider being
7207 // smarter here and handle {Start+Step,+,Step} too.
7208 const APInt *Off;
7209 if (match(S, m_scev_Add(m_scev_APInt(Off), m_SCEV(S))))
7210 Offset = *Off;
7211
7212 // Peel off a cast operation
7213 if (auto *SCast = dyn_cast<SCEVIntegralCastExpr>(S)) {
7214 CastOp = SCast->getSCEVType();
7215 S = SCast->getOperand();
7216 }
7217
7218 using namespace llvm::PatternMatch;
7219
7220 auto *SU = dyn_cast<SCEVUnknown>(S);
7221 const APInt *TrueVal, *FalseVal;
7222 if (!SU ||
7223 !match(SU->getValue(), m_Select(m_Value(Condition), m_APInt(TrueVal),
7224 m_APInt(FalseVal)))) {
7225 Condition = nullptr;
7226 return;
7227 }
7228
7229 TrueValue = *TrueVal;
7230 FalseValue = *FalseVal;
7231
7232 // Re-apply the cast we peeled off earlier
7233 if (CastOp)
7234 switch (*CastOp) {
7235 default:
7236 llvm_unreachable("Unknown SCEV cast type!");
7237
7238 case scTruncate:
7239 TrueValue = TrueValue.trunc(BitWidth);
7240 FalseValue = FalseValue.trunc(BitWidth);
7241 break;
7242 case scZeroExtend:
7243 TrueValue = TrueValue.zext(BitWidth);
7244 FalseValue = FalseValue.zext(BitWidth);
7245 break;
7246 case scSignExtend:
7247 TrueValue = TrueValue.sext(BitWidth);
7248 FalseValue = FalseValue.sext(BitWidth);
7249 break;
7250 }
7251
7252 // Re-apply the constant offset we peeled off earlier
7253 TrueValue += Offset;
7254 FalseValue += Offset;
7255 }
7256
7257 bool isRecognized() { return Condition != nullptr; }
7258 };
7259
7260 SelectPattern StartPattern(*this, BitWidth, Start);
7261 if (!StartPattern.isRecognized())
7262 return ConstantRange::getFull(BitWidth);
7263
7264 SelectPattern StepPattern(*this, BitWidth, Step);
7265 if (!StepPattern.isRecognized())
7266 return ConstantRange::getFull(BitWidth);
7267
7268 if (StartPattern.Condition != StepPattern.Condition) {
7269 // We don't handle this case today; but we could, by considering four
7270 // possibilities below instead of two. I'm not sure if there are cases where
7271 // that will help over what getRange already does, though.
7272 return ConstantRange::getFull(BitWidth);
7273 }
7274
7275 // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to
7276 // construct arbitrary general SCEV expressions here. This function is called
7277 // from deep in the call stack, and calling getSCEV (on a sext instruction,
7278 // say) can end up caching a suboptimal value.
7279
7280 // FIXME: without the explicit `this` receiver below, MSVC errors out with
7281 // C2352 and C2512 (otherwise it isn't needed).
7282
7283 const SCEV *TrueStart = this->getConstant(StartPattern.TrueValue);
7284 const SCEV *TrueStep = this->getConstant(StepPattern.TrueValue);
7285 const SCEV *FalseStart = this->getConstant(StartPattern.FalseValue);
7286 const SCEV *FalseStep = this->getConstant(StepPattern.FalseValue);
7287
7288 ConstantRange TrueRange =
7289 this->getRangeForAffineAR(TrueStart, TrueStep, MaxBECount);
7290 ConstantRange FalseRange =
7291 this->getRangeForAffineAR(FalseStart, FalseStep, MaxBECount);
7292
7293 return TrueRange.unionWith(FalseRange);
7294}
7295
7296SCEV::NoWrapFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) {
7297 if (isa<ConstantExpr>(V)) return SCEV::FlagAnyWrap;
7298 const BinaryOperator *BinOp = cast<BinaryOperator>(V);
7299
7300 // Return early if there are no flags to propagate to the SCEV.
7302 if (BinOp->hasNoUnsignedWrap())
7304 if (BinOp->hasNoSignedWrap())
7306 if (Flags == SCEV::FlagAnyWrap)
7307 return SCEV::FlagAnyWrap;
7308
7309 return isSCEVExprNeverPoison(BinOp) ? Flags : SCEV::FlagAnyWrap;
7310}
7311
7312const Instruction *
7313ScalarEvolution::getNonTrivialDefiningScopeBound(const SCEV *S) {
7314 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(S))
7315 return &*AddRec->getLoop()->getHeader()->begin();
7316 if (auto *U = dyn_cast<SCEVUnknown>(S))
7317 if (auto *I = dyn_cast<Instruction>(U->getValue()))
7318 return I;
7319 return nullptr;
7320}
7321
7322const Instruction *
7323ScalarEvolution::getDefiningScopeBound(ArrayRef<const SCEV *> Ops,
7324 bool &Precise) {
7325 Precise = true;
7326 // Do a bounded search of the def relation of the requested SCEVs.
7327 SmallPtrSet<const SCEV *, 16> Visited;
7329 auto pushOp = [&](const SCEV *S) {
7330 if (!Visited.insert(S).second)
7331 return;
7332 // Threshold of 30 here is arbitrary.
7333 if (Visited.size() > 30) {
7334 Precise = false;
7335 return;
7336 }
7337 Worklist.push_back(S);
7338 };
7339
7340 for (const auto *S : Ops)
7341 pushOp(S);
7342
7343 const Instruction *Bound = nullptr;
7344 while (!Worklist.empty()) {
7345 auto *S = Worklist.pop_back_val();
7346 if (auto *DefI = getNonTrivialDefiningScopeBound(S)) {
7347 if (!Bound || DT.dominates(Bound, DefI))
7348 Bound = DefI;
7349 } else {
7350 for (const auto *Op : S->operands())
7351 pushOp(Op);
7352 }
7353 }
7354 return Bound ? Bound : &*F.getEntryBlock().begin();
7355}
7356
7357const Instruction *
7358ScalarEvolution::getDefiningScopeBound(ArrayRef<const SCEV *> Ops) {
7359 bool Discard;
7360 return getDefiningScopeBound(Ops, Discard);
7361}
7362
7363bool ScalarEvolution::isGuaranteedToTransferExecutionTo(const Instruction *A,
7364 const Instruction *B) {
7365 if (A->getParent() == B->getParent() &&
7367 B->getIterator()))
7368 return true;
7369
7370 auto *BLoop = LI.getLoopFor(B->getParent());
7371 if (BLoop && BLoop->getHeader() == B->getParent() &&
7372 BLoop->getLoopPreheader() == A->getParent() &&
7374 A->getParent()->end()) &&
7375 isGuaranteedToTransferExecutionToSuccessor(B->getParent()->begin(),
7376 B->getIterator()))
7377 return true;
7378 return false;
7379}
7380
7381bool ScalarEvolution::isGuaranteedNotToBePoison(const SCEV *Op) {
7382 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ true);
7383 visitAll(Op, PC);
7384 return PC.MaybePoison.empty();
7385}
7386
7387bool ScalarEvolution::isGuaranteedNotToCauseUB(const SCEV *Op) {
7388 return !SCEVExprContains(Op, [this](const SCEV *S) {
7389 const SCEV *Op1;
7390 bool M = match(S, m_scev_UDiv(m_SCEV(), m_SCEV(Op1)));
7391 // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
7392 // is a non-zero constant, we have to assume the UDiv may be UB.
7393 return M && (!isKnownNonZero(Op1) || !isGuaranteedNotToBePoison(Op1));
7394 });
7395}
7396
7397bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) {
7398 // Only proceed if we can prove that I does not yield poison.
7400 return false;
7401
7402 // At this point we know that if I is executed, then it does not wrap
7403 // according to at least one of NSW or NUW. If I is not executed, then we do
7404 // not know if the calculation that I represents would wrap. Multiple
7405 // instructions can map to the same SCEV. If we apply NSW or NUW from I to
7406 // the SCEV, we must guarantee no wrapping for that SCEV also when it is
7407 // derived from other instructions that map to the same SCEV. We cannot make
7408 // that guarantee for cases where I is not executed. So we need to find a
7409 // upper bound on the defining scope for the SCEV, and prove that I is
7410 // executed every time we enter that scope. When the bounding scope is a
7411 // loop (the common case), this is equivalent to proving I executes on every
7412 // iteration of that loop.
7414 for (const Use &Op : I->operands()) {
7415 // I could be an extractvalue from a call to an overflow intrinsic.
7416 // TODO: We can do better here in some cases.
7417 if (isSCEVable(Op->getType()))
7418 SCEVOps.push_back(getSCEV(Op));
7419 }
7420 auto *DefI = getDefiningScopeBound(SCEVOps);
7421 return isGuaranteedToTransferExecutionTo(DefI, I);
7422}
7423
7424bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) {
7425 // If we know that \c I can never be poison period, then that's enough.
7426 if (isSCEVExprNeverPoison(I))
7427 return true;
7428
7429 // If the loop only has one exit, then we know that, if the loop is entered,
7430 // any instruction dominating that exit will be executed. If any such
7431 // instruction would result in UB, the addrec cannot be poison.
7432 //
7433 // This is basically the same reasoning as in isSCEVExprNeverPoison(), but
7434 // also handles uses outside the loop header (they just need to dominate the
7435 // single exit).
7436
7437 auto *ExitingBB = L->getExitingBlock();
7438 if (!ExitingBB || !loopHasNoAbnormalExits(L))
7439 return false;
7440
7441 SmallPtrSet<const Value *, 16> KnownPoison;
7443
7444 // We start by assuming \c I, the post-inc add recurrence, is poison. Only
7445 // things that are known to be poison under that assumption go on the
7446 // Worklist.
7447 KnownPoison.insert(I);
7448 Worklist.push_back(I);
7449
7450 while (!Worklist.empty()) {
7451 const Instruction *Poison = Worklist.pop_back_val();
7452
7453 for (const Use &U : Poison->uses()) {
7454 const Instruction *PoisonUser = cast<Instruction>(U.getUser());
7455 if (mustTriggerUB(PoisonUser, KnownPoison) &&
7456 DT.dominates(PoisonUser->getParent(), ExitingBB))
7457 return true;
7458
7459 if (propagatesPoison(U) && L->contains(PoisonUser))
7460 if (KnownPoison.insert(PoisonUser).second)
7461 Worklist.push_back(PoisonUser);
7462 }
7463 }
7464
7465 return false;
7466}
7467
7468ScalarEvolution::LoopProperties
7469ScalarEvolution::getLoopProperties(const Loop *L) {
7470 using LoopProperties = ScalarEvolution::LoopProperties;
7471
7472 auto Itr = LoopPropertiesCache.find(L);
7473 if (Itr == LoopPropertiesCache.end()) {
7474 auto HasSideEffects = [](Instruction *I) {
7475 if (auto *SI = dyn_cast<StoreInst>(I))
7476 return !SI->isSimple();
7477
7478 if (I->mayThrow())
7479 return true;
7480
7481 // Non-volatile memset / memcpy do not count as side-effect for forward
7482 // progress.
7483 if (isa<MemIntrinsic>(I) && !I->isVolatile())
7484 return false;
7485
7486 return I->mayWriteToMemory();
7487 };
7488
7489 LoopProperties LP = {/* HasNoAbnormalExits */ true,
7490 /*HasNoSideEffects*/ true};
7491
7492 for (auto *BB : L->getBlocks())
7493 for (auto &I : *BB) {
7495 LP.HasNoAbnormalExits = false;
7496 if (HasSideEffects(&I))
7497 LP.HasNoSideEffects = false;
7498 if (!LP.HasNoAbnormalExits && !LP.HasNoSideEffects)
7499 break; // We're already as pessimistic as we can get.
7500 }
7501
7502 auto InsertPair = LoopPropertiesCache.insert({L, LP});
7503 assert(InsertPair.second && "We just checked!");
7504 Itr = InsertPair.first;
7505 }
7506
7507 return Itr->second;
7508}
7509
7511 // A mustprogress loop without side effects must be finite.
7512 // TODO: The check used here is very conservative. It's only *specific*
7513 // side effects which are well defined in infinite loops.
7514 return isFinite(L) || (isMustProgress(L) && loopHasNoSideEffects(L));
7515}
7516
7517const SCEV *ScalarEvolution::createSCEVIter(Value *V) {
7518 // Worklist item with a Value and a bool indicating whether all operands have
7519 // been visited already.
7522
7523 Stack.emplace_back(V, true);
7524 Stack.emplace_back(V, false);
7525 while (!Stack.empty()) {
7526 auto E = Stack.pop_back_val();
7527 Value *CurV = E.getPointer();
7528
7529 if (getExistingSCEV(CurV))
7530 continue;
7531
7533 const SCEV *CreatedSCEV = nullptr;
7534 // If all operands have been visited already, create the SCEV.
7535 if (E.getInt()) {
7536 CreatedSCEV = createSCEV(CurV);
7537 } else {
7538 // Otherwise get the operands we need to create SCEV's for before creating
7539 // the SCEV for CurV. If the SCEV for CurV can be constructed trivially,
7540 // just use it.
7541 CreatedSCEV = getOperandsToCreate(CurV, Ops);
7542 }
7543
7544 if (CreatedSCEV) {
7545 insertValueToMap(CurV, CreatedSCEV);
7546 } else {
7547 // Queue CurV for SCEV creation, followed by its's operands which need to
7548 // be constructed first.
7549 Stack.emplace_back(CurV, true);
7550 for (Value *Op : Ops)
7551 Stack.emplace_back(Op, false);
7552 }
7553 }
7554
7555 return getExistingSCEV(V);
7556}
7557
7558const SCEV *
7559ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
7560 if (!isSCEVable(V->getType()))
7561 return getUnknown(V);
7562
7563 if (Instruction *I = dyn_cast<Instruction>(V)) {
7564 // Don't attempt to analyze instructions in blocks that aren't
7565 // reachable. Such instructions don't matter, and they aren't required
7566 // to obey basic rules for definitions dominating uses which this
7567 // analysis depends on.
7568 if (!DT.isReachableFromEntry(I->getParent()))
7569 return getUnknown(PoisonValue::get(V->getType()));
7570 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(V))
7571 return getConstant(CI);
7572 else if (isa<GlobalAlias>(V))
7573 return getUnknown(V);
7574 else if (!isa<ConstantExpr>(V))
7575 return getUnknown(V);
7576
7578 if (auto BO =
7580 bool IsConstArg = isa<ConstantInt>(BO->RHS);
7581 switch (BO->Opcode) {
7582 case Instruction::Add:
7583 case Instruction::Mul: {
7584 // For additions and multiplications, traverse add/mul chains for which we
7585 // can potentially create a single SCEV, to reduce the number of
7586 // get{Add,Mul}Expr calls.
7587 do {
7588 if (BO->Op) {
7589 if (BO->Op != V && getExistingSCEV(BO->Op)) {
7590 Ops.push_back(BO->Op);
7591 break;
7592 }
7593 }
7594 Ops.push_back(BO->RHS);
7595 auto NewBO = MatchBinaryOp(BO->LHS, getDataLayout(), AC, DT,
7597 if (!NewBO ||
7598 (BO->Opcode == Instruction::Add &&
7599 (NewBO->Opcode != Instruction::Add &&
7600 NewBO->Opcode != Instruction::Sub)) ||
7601 (BO->Opcode == Instruction::Mul &&
7602 NewBO->Opcode != Instruction::Mul)) {
7603 Ops.push_back(BO->LHS);
7604 break;
7605 }
7606 // CreateSCEV calls getNoWrapFlagsFromUB, which under certain conditions
7607 // requires a SCEV for the LHS.
7608 if (BO->Op && (BO->IsNSW || BO->IsNUW)) {
7609 auto *I = dyn_cast<Instruction>(BO->Op);
7610 if (I && programUndefinedIfPoison(I)) {
7611 Ops.push_back(BO->LHS);
7612 break;
7613 }
7614 }
7615 BO = NewBO;
7616 } while (true);
7617 return nullptr;
7618 }
7619 case Instruction::Sub:
7620 case Instruction::UDiv:
7621 case Instruction::URem:
7622 break;
7623 case Instruction::AShr:
7624 case Instruction::Shl:
7625 case Instruction::Xor:
7626 if (!IsConstArg)
7627 return nullptr;
7628 break;
7629 case Instruction::And:
7630 case Instruction::Or:
7631 if (!IsConstArg && !BO->LHS->getType()->isIntegerTy(1))
7632 return nullptr;
7633 break;
7634 case Instruction::LShr:
7635 return getUnknown(V);
7636 default:
7637 llvm_unreachable("Unhandled binop");
7638 break;
7639 }
7640
7641 Ops.push_back(BO->LHS);
7642 Ops.push_back(BO->RHS);
7643 return nullptr;
7644 }
7645
7646 switch (U->getOpcode()) {
7647 case Instruction::Trunc:
7648 case Instruction::ZExt:
7649 case Instruction::SExt:
7650 case Instruction::PtrToInt:
7651 Ops.push_back(U->getOperand(0));
7652 return nullptr;
7653
7654 case Instruction::BitCast:
7655 if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType())) {
7656 Ops.push_back(U->getOperand(0));
7657 return nullptr;
7658 }
7659 return getUnknown(V);
7660
7661 case Instruction::SDiv:
7662 case Instruction::SRem:
7663 Ops.push_back(U->getOperand(0));
7664 Ops.push_back(U->getOperand(1));
7665 return nullptr;
7666
7667 case Instruction::GetElementPtr:
7668 assert(cast<GEPOperator>(U)->getSourceElementType()->isSized() &&
7669 "GEP source element type must be sized");
7670 llvm::append_range(Ops, U->operands());
7671 return nullptr;
7672
7673 case Instruction::IntToPtr:
7674 return getUnknown(V);
7675
7676 case Instruction::PHI:
7677 // Keep constructing SCEVs' for phis recursively for now.
7678 return nullptr;
7679
7680 case Instruction::Select: {
7681 // Check if U is a select that can be simplified to a SCEVUnknown.
7682 auto CanSimplifyToUnknown = [this, U]() {
7683 if (U->getType()->isIntegerTy(1) || isa<ConstantInt>(U->getOperand(0)))
7684 return false;
7685
7686 auto *ICI = dyn_cast<ICmpInst>(U->getOperand(0));
7687 if (!ICI)
7688 return false;
7689 Value *LHS = ICI->getOperand(0);
7690 Value *RHS = ICI->getOperand(1);
7691 if (ICI->getPredicate() == CmpInst::ICMP_EQ ||
7692 ICI->getPredicate() == CmpInst::ICMP_NE) {
7694 return true;
7695 } else if (getTypeSizeInBits(LHS->getType()) >
7696 getTypeSizeInBits(U->getType()))
7697 return true;
7698 return false;
7699 };
7700 if (CanSimplifyToUnknown())
7701 return getUnknown(U);
7702
7703 llvm::append_range(Ops, U->operands());
7704 return nullptr;
7705 break;
7706 }
7707 case Instruction::Call:
7708 case Instruction::Invoke:
7709 if (Value *RV = cast<CallBase>(U)->getReturnedArgOperand()) {
7710 Ops.push_back(RV);
7711 return nullptr;
7712 }
7713
7714 if (auto *II = dyn_cast<IntrinsicInst>(U)) {
7715 switch (II->getIntrinsicID()) {
7716 case Intrinsic::abs:
7717 Ops.push_back(II->getArgOperand(0));
7718 return nullptr;
7719 case Intrinsic::umax:
7720 case Intrinsic::umin:
7721 case Intrinsic::smax:
7722 case Intrinsic::smin:
7723 case Intrinsic::usub_sat:
7724 case Intrinsic::uadd_sat:
7725 Ops.push_back(II->getArgOperand(0));
7726 Ops.push_back(II->getArgOperand(1));
7727 return nullptr;
7728 case Intrinsic::start_loop_iterations:
7729 case Intrinsic::annotation:
7730 case Intrinsic::ptr_annotation:
7731 Ops.push_back(II->getArgOperand(0));
7732 return nullptr;
7733 default:
7734 break;
7735 }
7736 }
7737 break;
7738 }
7739
7740 return nullptr;
7741}
7742
7743const SCEV *ScalarEvolution::createSCEV(Value *V) {
7744 if (!isSCEVable(V->getType()))
7745 return getUnknown(V);
7746
7747 if (Instruction *I = dyn_cast<Instruction>(V)) {
7748 // Don't attempt to analyze instructions in blocks that aren't
7749 // reachable. Such instructions don't matter, and they aren't required
7750 // to obey basic rules for definitions dominating uses which this
7751 // analysis depends on.
7752 if (!DT.isReachableFromEntry(I->getParent()))
7753 return getUnknown(PoisonValue::get(V->getType()));
7754 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(V))
7755 return getConstant(CI);
7756 else if (isa<GlobalAlias>(V))
7757 return getUnknown(V);
7758 else if (!isa<ConstantExpr>(V))
7759 return getUnknown(V);
7760
7761 const SCEV *LHS;
7762 const SCEV *RHS;
7763
7765 if (auto BO =
7767 switch (BO->Opcode) {
7768 case Instruction::Add: {
7769 // The simple thing to do would be to just call getSCEV on both operands
7770 // and call getAddExpr with the result. However if we're looking at a
7771 // bunch of things all added together, this can be quite inefficient,
7772 // because it leads to N-1 getAddExpr calls for N ultimate operands.
7773 // Instead, gather up all the operands and make a single getAddExpr call.
7774 // LLVM IR canonical form means we need only traverse the left operands.
7776 do {
7777 if (BO->Op) {
7778 if (auto *OpSCEV = getExistingSCEV(BO->Op)) {
7779 AddOps.push_back(OpSCEV);
7780 break;
7781 }
7782
7783 // If a NUW or NSW flag can be applied to the SCEV for this
7784 // addition, then compute the SCEV for this addition by itself
7785 // with a separate call to getAddExpr. We need to do that
7786 // instead of pushing the operands of the addition onto AddOps,
7787 // since the flags are only known to apply to this particular
7788 // addition - they may not apply to other additions that can be
7789 // formed with operands from AddOps.
7790 const SCEV *RHS = getSCEV(BO->RHS);
7791 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(BO->Op);
7792 if (Flags != SCEV::FlagAnyWrap) {
7793 const SCEV *LHS = getSCEV(BO->LHS);
7794 if (BO->Opcode == Instruction::Sub)
7795 AddOps.push_back(getMinusSCEV(LHS, RHS, Flags));
7796 else
7797 AddOps.push_back(getAddExpr(LHS, RHS, Flags));
7798 break;
7799 }
7800 }
7801
7802 if (BO->Opcode == Instruction::Sub)
7803 AddOps.push_back(getNegativeSCEV(getSCEV(BO->RHS)));
7804 else
7805 AddOps.push_back(getSCEV(BO->RHS));
7806
7807 auto NewBO = MatchBinaryOp(BO->LHS, getDataLayout(), AC, DT,
7809 if (!NewBO || (NewBO->Opcode != Instruction::Add &&
7810 NewBO->Opcode != Instruction::Sub)) {
7811 AddOps.push_back(getSCEV(BO->LHS));
7812 break;
7813 }
7814 BO = NewBO;
7815 } while (true);
7816
7817 return getAddExpr(AddOps);
7818 }
7819
7820 case Instruction::Mul: {
7822 do {
7823 if (BO->Op) {
7824 if (auto *OpSCEV = getExistingSCEV(BO->Op)) {
7825 MulOps.push_back(OpSCEV);
7826 break;
7827 }
7828
7829 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(BO->Op);
7830 if (Flags != SCEV::FlagAnyWrap) {
7831 LHS = getSCEV(BO->LHS);
7832 RHS = getSCEV(BO->RHS);
7833 MulOps.push_back(getMulExpr(LHS, RHS, Flags));
7834 break;
7835 }
7836 }
7837
7838 MulOps.push_back(getSCEV(BO->RHS));
7839 auto NewBO = MatchBinaryOp(BO->LHS, getDataLayout(), AC, DT,
7841 if (!NewBO || NewBO->Opcode != Instruction::Mul) {
7842 MulOps.push_back(getSCEV(BO->LHS));
7843 break;
7844 }
7845 BO = NewBO;
7846 } while (true);
7847
7848 return getMulExpr(MulOps);
7849 }
7850 case Instruction::UDiv:
7851 LHS = getSCEV(BO->LHS);
7852 RHS = getSCEV(BO->RHS);
7853 return getUDivExpr(LHS, RHS);
7854 case Instruction::URem:
7855 LHS = getSCEV(BO->LHS);
7856 RHS = getSCEV(BO->RHS);
7857 return getURemExpr(LHS, RHS);
7858 case Instruction::Sub: {
7860 if (BO->Op)
7861 Flags = getNoWrapFlagsFromUB(BO->Op);
7862 LHS = getSCEV(BO->LHS);
7863 RHS = getSCEV(BO->RHS);
7864 return getMinusSCEV(LHS, RHS, Flags);
7865 }
7866 case Instruction::And:
7867 // For an expression like x&255 that merely masks off the high bits,
7868 // use zext(trunc(x)) as the SCEV expression.
7869 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) {
7870 if (CI->isZero())
7871 return getSCEV(BO->RHS);
7872 if (CI->isMinusOne())
7873 return getSCEV(BO->LHS);
7874 const APInt &A = CI->getValue();
7875
7876 // Instcombine's ShrinkDemandedConstant may strip bits out of
7877 // constants, obscuring what would otherwise be a low-bits mask.
7878 // Use computeKnownBits to compute what ShrinkDemandedConstant
7879 // knew about to reconstruct a low-bits mask value.
7880 unsigned LZ = A.countl_zero();
7881 unsigned TZ = A.countr_zero();
7882 unsigned BitWidth = A.getBitWidth();
7883 KnownBits Known(BitWidth);
7884 computeKnownBits(BO->LHS, Known, getDataLayout(), &AC, nullptr, &DT);
7885
7886 APInt EffectiveMask =
7887 APInt::getLowBitsSet(BitWidth, BitWidth - LZ - TZ).shl(TZ);
7888 if ((LZ != 0 || TZ != 0) && !((~A & ~Known.Zero) & EffectiveMask)) {
7889 const SCEV *MulCount = getConstant(APInt::getOneBitSet(BitWidth, TZ));
7890 const SCEV *LHS = getSCEV(BO->LHS);
7891 const SCEV *ShiftedLHS = nullptr;
7892 if (auto *LHSMul = dyn_cast<SCEVMulExpr>(LHS)) {
7893 if (auto *OpC = dyn_cast<SCEVConstant>(LHSMul->getOperand(0))) {
7894 // For an expression like (x * 8) & 8, simplify the multiply.
7895 unsigned MulZeros = OpC->getAPInt().countr_zero();
7896 unsigned GCD = std::min(MulZeros, TZ);
7897 APInt DivAmt = APInt::getOneBitSet(BitWidth, TZ - GCD);
7899 MulOps.push_back(getConstant(OpC->getAPInt().ashr(GCD)));
7900 append_range(MulOps, LHSMul->operands().drop_front());
7901 auto *NewMul = getMulExpr(MulOps, LHSMul->getNoWrapFlags());
7902 ShiftedLHS = getUDivExpr(NewMul, getConstant(DivAmt));
7903 }
7904 }
7905 if (!ShiftedLHS)
7906 ShiftedLHS = getUDivExpr(LHS, MulCount);
7907 return getMulExpr(
7909 getTruncateExpr(ShiftedLHS,
7910 IntegerType::get(getContext(), BitWidth - LZ - TZ)),
7911 BO->LHS->getType()),
7912 MulCount);
7913 }
7914 }
7915 // Binary `and` is a bit-wise `umin`.
7916 if (BO->LHS->getType()->isIntegerTy(1)) {
7917 LHS = getSCEV(BO->LHS);
7918 RHS = getSCEV(BO->RHS);
7919 return getUMinExpr(LHS, RHS);
7920 }
7921 break;
7922
7923 case Instruction::Or:
7924 // Binary `or` is a bit-wise `umax`.
7925 if (BO->LHS->getType()->isIntegerTy(1)) {
7926 LHS = getSCEV(BO->LHS);
7927 RHS = getSCEV(BO->RHS);
7928 return getUMaxExpr(LHS, RHS);
7929 }
7930 break;
7931
7932 case Instruction::Xor:
7933 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) {
7934 // If the RHS of xor is -1, then this is a not operation.
7935 if (CI->isMinusOne())
7936 return getNotSCEV(getSCEV(BO->LHS));
7937
7938 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
7939 // This is a variant of the check for xor with -1, and it handles
7940 // the case where instcombine has trimmed non-demanded bits out
7941 // of an xor with -1.
7942 if (auto *LBO = dyn_cast<BinaryOperator>(BO->LHS))
7943 if (ConstantInt *LCI = dyn_cast<ConstantInt>(LBO->getOperand(1)))
7944 if (LBO->getOpcode() == Instruction::And &&
7945 LCI->getValue() == CI->getValue())
7946 if (const SCEVZeroExtendExpr *Z =
7948 Type *UTy = BO->LHS->getType();
7949 const SCEV *Z0 = Z->getOperand();
7950 Type *Z0Ty = Z0->getType();
7951 unsigned Z0TySize = getTypeSizeInBits(Z0Ty);
7952
7953 // If C is a low-bits mask, the zero extend is serving to
7954 // mask off the high bits. Complement the operand and
7955 // re-apply the zext.
7956 if (CI->getValue().isMask(Z0TySize))
7957 return getZeroExtendExpr(getNotSCEV(Z0), UTy);
7958
7959 // If C is a single bit, it may be in the sign-bit position
7960 // before the zero-extend. In this case, represent the xor
7961 // using an add, which is equivalent, and re-apply the zext.
7962 APInt Trunc = CI->getValue().trunc(Z0TySize);
7963 if (Trunc.zext(getTypeSizeInBits(UTy)) == CI->getValue() &&
7964 Trunc.isSignMask())
7965 return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)),
7966 UTy);
7967 }
7968 }
7969 break;
7970
7971 case Instruction::Shl:
7972 // Turn shift left of a constant amount into a multiply.
7973 if (ConstantInt *SA = dyn_cast<ConstantInt>(BO->RHS)) {
7974 uint32_t BitWidth = cast<IntegerType>(SA->getType())->getBitWidth();
7975
7976 // If the shift count is not less than the bitwidth, the result of
7977 // the shift is undefined. Don't try to analyze it, because the
7978 // resolution chosen here may differ from the resolution chosen in
7979 // other parts of the compiler.
7980 if (SA->getValue().uge(BitWidth))
7981 break;
7982
7983 // We can safely preserve the nuw flag in all cases. It's also safe to
7984 // turn a nuw nsw shl into a nuw nsw mul. However, nsw in isolation
7985 // requires special handling. It can be preserved as long as we're not
7986 // left shifting by bitwidth - 1.
7987 auto Flags = SCEV::FlagAnyWrap;
7988 if (BO->Op) {
7989 auto MulFlags = getNoWrapFlagsFromUB(BO->Op);
7990 if ((MulFlags & SCEV::FlagNSW) &&
7991 ((MulFlags & SCEV::FlagNUW) || SA->getValue().ult(BitWidth - 1)))
7993 if (MulFlags & SCEV::FlagNUW)
7995 }
7996
7997 ConstantInt *X = ConstantInt::get(
7998 getContext(), APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
7999 return getMulExpr(getSCEV(BO->LHS), getConstant(X), Flags);
8000 }
8001 break;
8002
8003 case Instruction::AShr:
8004 // AShr X, C, where C is a constant.
8005 ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS);
8006 if (!CI)
8007 break;
8008
8009 Type *OuterTy = BO->LHS->getType();
8010 uint64_t BitWidth = getTypeSizeInBits(OuterTy);
8011 // If the shift count is not less than the bitwidth, the result of
8012 // the shift is undefined. Don't try to analyze it, because the
8013 // resolution chosen here may differ from the resolution chosen in
8014 // other parts of the compiler.
8015 if (CI->getValue().uge(BitWidth))
8016 break;
8017
8018 if (CI->isZero())
8019 return getSCEV(BO->LHS); // shift by zero --> noop
8020
8021 uint64_t AShrAmt = CI->getZExtValue();
8022 Type *TruncTy = IntegerType::get(getContext(), BitWidth - AShrAmt);
8023
8024 Operator *L = dyn_cast<Operator>(BO->LHS);
8025 const SCEV *AddTruncateExpr = nullptr;
8026 ConstantInt *ShlAmtCI = nullptr;
8027 const SCEV *AddConstant = nullptr;
8028
8029 if (L && L->getOpcode() == Instruction::Add) {
8030 // X = Shl A, n
8031 // Y = Add X, c
8032 // Z = AShr Y, m
8033 // n, c and m are constants.
8034
8035 Operator *LShift = dyn_cast<Operator>(L->getOperand(0));
8036 ConstantInt *AddOperandCI = dyn_cast<ConstantInt>(L->getOperand(1));
8037 if (LShift && LShift->getOpcode() == Instruction::Shl) {
8038 if (AddOperandCI) {
8039 const SCEV *ShlOp0SCEV = getSCEV(LShift->getOperand(0));
8040 ShlAmtCI = dyn_cast<ConstantInt>(LShift->getOperand(1));
8041 // since we truncate to TruncTy, the AddConstant should be of the
8042 // same type, so create a new Constant with type same as TruncTy.
8043 // Also, the Add constant should be shifted right by AShr amount.
8044 APInt AddOperand = AddOperandCI->getValue().ashr(AShrAmt);
8045 AddConstant = getConstant(AddOperand.trunc(BitWidth - AShrAmt));
8046 // we model the expression as sext(add(trunc(A), c << n)), since the
8047 // sext(trunc) part is already handled below, we create a
8048 // AddExpr(TruncExp) which will be used later.
8049 AddTruncateExpr = getTruncateExpr(ShlOp0SCEV, TruncTy);
8050 }
8051 }
8052 } else if (L && L->getOpcode() == Instruction::Shl) {
8053 // X = Shl A, n
8054 // Y = AShr X, m
8055 // Both n and m are constant.
8056
8057 const SCEV *ShlOp0SCEV = getSCEV(L->getOperand(0));
8058 ShlAmtCI = dyn_cast<ConstantInt>(L->getOperand(1));
8059 AddTruncateExpr = getTruncateExpr(ShlOp0SCEV, TruncTy);
8060 }
8061
8062 if (AddTruncateExpr && ShlAmtCI) {
8063 // We can merge the two given cases into a single SCEV statement,
8064 // incase n = m, the mul expression will be 2^0, so it gets resolved to
8065 // a simpler case. The following code handles the two cases:
8066 //
8067 // 1) For a two-shift sext-inreg, i.e. n = m,
8068 // use sext(trunc(x)) as the SCEV expression.
8069 //
8070 // 2) When n > m, use sext(mul(trunc(x), 2^(n-m)))) as the SCEV
8071 // expression. We already checked that ShlAmt < BitWidth, so
8072 // the multiplier, 1 << (ShlAmt - AShrAmt), fits into TruncTy as
8073 // ShlAmt - AShrAmt < Amt.
8074 const APInt &ShlAmt = ShlAmtCI->getValue();
8075 if (ShlAmt.ult(BitWidth) && ShlAmt.uge(AShrAmt)) {
8076 APInt Mul = APInt::getOneBitSet(BitWidth - AShrAmt,
8077 ShlAmtCI->getZExtValue() - AShrAmt);
8078 const SCEV *CompositeExpr =
8079 getMulExpr(AddTruncateExpr, getConstant(Mul));
8080 if (L->getOpcode() != Instruction::Shl)
8081 CompositeExpr = getAddExpr(CompositeExpr, AddConstant);
8082
8083 return getSignExtendExpr(CompositeExpr, OuterTy);
8084 }
8085 }
8086 break;
8087 }
8088 }
8089
8090 switch (U->getOpcode()) {
8091 case Instruction::Trunc:
8092 return getTruncateExpr(getSCEV(U->getOperand(0)), U->getType());
8093
8094 case Instruction::ZExt:
8095 return getZeroExtendExpr(getSCEV(U->getOperand(0)), U->getType());
8096
8097 case Instruction::SExt:
8098 if (auto BO = MatchBinaryOp(U->getOperand(0), getDataLayout(), AC, DT,
8100 // The NSW flag of a subtract does not always survive the conversion to
8101 // A + (-1)*B. By pushing sign extension onto its operands we are much
8102 // more likely to preserve NSW and allow later AddRec optimisations.
8103 //
8104 // NOTE: This is effectively duplicating this logic from getSignExtend:
8105 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
8106 // but by that point the NSW information has potentially been lost.
8107 if (BO->Opcode == Instruction::Sub && BO->IsNSW) {
8108 Type *Ty = U->getType();
8109 auto *V1 = getSignExtendExpr(getSCEV(BO->LHS), Ty);
8110 auto *V2 = getSignExtendExpr(getSCEV(BO->RHS), Ty);
8111 return getMinusSCEV(V1, V2, SCEV::FlagNSW);
8112 }
8113 }
8114 return getSignExtendExpr(getSCEV(U->getOperand(0)), U->getType());
8115
8116 case Instruction::BitCast:
8117 // BitCasts are no-op casts so we just eliminate the cast.
8118 if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType()))
8119 return getSCEV(U->getOperand(0));
8120 break;
8121
8122 case Instruction::PtrToInt: {
8123 // Pointer to integer cast is straight-forward, so do model it.
8124 const SCEV *Op = getSCEV(U->getOperand(0));
8125 Type *DstIntTy = U->getType();
8126 // But only if effective SCEV (integer) type is wide enough to represent
8127 // all possible pointer values.
8128 const SCEV *IntOp = getPtrToIntExpr(Op, DstIntTy);
8129 if (isa<SCEVCouldNotCompute>(IntOp))
8130 return getUnknown(V);
8131 return IntOp;
8132 }
8133 case Instruction::IntToPtr:
8134 // Just don't deal with inttoptr casts.
8135 return getUnknown(V);
8136
8137 case Instruction::SDiv:
8138 // If both operands are non-negative, this is just an udiv.
8139 if (isKnownNonNegative(getSCEV(U->getOperand(0))) &&
8140 isKnownNonNegative(getSCEV(U->getOperand(1))))
8141 return getUDivExpr(getSCEV(U->getOperand(0)), getSCEV(U->getOperand(1)));
8142 break;
8143
8144 case Instruction::SRem:
8145 // If both operands are non-negative, this is just an urem.
8146 if (isKnownNonNegative(getSCEV(U->getOperand(0))) &&
8147 isKnownNonNegative(getSCEV(U->getOperand(1))))
8148 return getURemExpr(getSCEV(U->getOperand(0)), getSCEV(U->getOperand(1)));
8149 break;
8150
8151 case Instruction::GetElementPtr:
8152 return createNodeForGEP(cast<GEPOperator>(U));
8153
8154 case Instruction::PHI:
8155 return createNodeForPHI(cast<PHINode>(U));
8156
8157 case Instruction::Select:
8158 return createNodeForSelectOrPHI(U, U->getOperand(0), U->getOperand(1),
8159 U->getOperand(2));
8160
8161 case Instruction::Call:
8162 case Instruction::Invoke:
8163 if (Value *RV = cast<CallBase>(U)->getReturnedArgOperand())
8164 return getSCEV(RV);
8165
8166 if (auto *II = dyn_cast<IntrinsicInst>(U)) {
8167 switch (II->getIntrinsicID()) {
8168 case Intrinsic::abs:
8169 return getAbsExpr(
8170 getSCEV(II->getArgOperand(0)),
8171 /*IsNSW=*/cast<ConstantInt>(II->getArgOperand(1))->isOne());
8172 case Intrinsic::umax:
8173 LHS = getSCEV(II->getArgOperand(0));
8174 RHS = getSCEV(II->getArgOperand(1));
8175 return getUMaxExpr(LHS, RHS);
8176 case Intrinsic::umin:
8177 LHS = getSCEV(II->getArgOperand(0));
8178 RHS = getSCEV(II->getArgOperand(1));
8179 return getUMinExpr(LHS, RHS);
8180 case Intrinsic::smax:
8181 LHS = getSCEV(II->getArgOperand(0));
8182 RHS = getSCEV(II->getArgOperand(1));
8183 return getSMaxExpr(LHS, RHS);
8184 case Intrinsic::smin:
8185 LHS = getSCEV(II->getArgOperand(0));
8186 RHS = getSCEV(II->getArgOperand(1));
8187 return getSMinExpr(LHS, RHS);
8188 case Intrinsic::usub_sat: {
8189 const SCEV *X = getSCEV(II->getArgOperand(0));
8190 const SCEV *Y = getSCEV(II->getArgOperand(1));
8191 const SCEV *ClampedY = getUMinExpr(X, Y);
8192 return getMinusSCEV(X, ClampedY, SCEV::FlagNUW);
8193 }
8194 case Intrinsic::uadd_sat: {
8195 const SCEV *X = getSCEV(II->getArgOperand(0));
8196 const SCEV *Y = getSCEV(II->getArgOperand(1));
8197 const SCEV *ClampedX = getUMinExpr(X, getNotSCEV(Y));
8198 return getAddExpr(ClampedX, Y, SCEV::FlagNUW);
8199 }
8200 case Intrinsic::start_loop_iterations:
8201 case Intrinsic::annotation:
8202 case Intrinsic::ptr_annotation:
8203 // A start_loop_iterations or llvm.annotation or llvm.prt.annotation is
8204 // just eqivalent to the first operand for SCEV purposes.
8205 return getSCEV(II->getArgOperand(0));
8206 case Intrinsic::vscale:
8207 return getVScale(II->getType());
8208 default:
8209 break;
8210 }
8211 }
8212 break;
8213 }
8214
8215 return getUnknown(V);
8216}
8217
8218//===----------------------------------------------------------------------===//
8219// Iteration Count Computation Code
8220//
8221
8223 if (isa<SCEVCouldNotCompute>(ExitCount))
8224 return getCouldNotCompute();
8225
8226 auto *ExitCountType = ExitCount->getType();
8227 assert(ExitCountType->isIntegerTy());
8228 auto *EvalTy = Type::getIntNTy(ExitCountType->getContext(),
8229 1 + ExitCountType->getScalarSizeInBits());
8230 return getTripCountFromExitCount(ExitCount, EvalTy, nullptr);
8231}
8232
8234 Type *EvalTy,
8235 const Loop *L) {
8236 if (isa<SCEVCouldNotCompute>(ExitCount))
8237 return getCouldNotCompute();
8238
8239 unsigned ExitCountSize = getTypeSizeInBits(ExitCount->getType());
8240 unsigned EvalSize = EvalTy->getPrimitiveSizeInBits();
8241
8242 auto CanAddOneWithoutOverflow = [&]() {
8243 ConstantRange ExitCountRange =
8244 getRangeRef(ExitCount, RangeSignHint::HINT_RANGE_UNSIGNED);
8245 if (!ExitCountRange.contains(APInt::getMaxValue(ExitCountSize)))
8246 return true;
8247
8248 return L && isLoopEntryGuardedByCond(L, ICmpInst::ICMP_NE, ExitCount,
8249 getMinusOne(ExitCount->getType()));
8250 };
8251
8252 // If we need to zero extend the backedge count, check if we can add one to
8253 // it prior to zero extending without overflow. Provided this is safe, it
8254 // allows better simplification of the +1.
8255 if (EvalSize > ExitCountSize && CanAddOneWithoutOverflow())
8256 return getZeroExtendExpr(
8257 getAddExpr(ExitCount, getOne(ExitCount->getType())), EvalTy);
8258
8259 // Get the total trip count from the count by adding 1. This may wrap.
8260 return getAddExpr(getTruncateOrZeroExtend(ExitCount, EvalTy), getOne(EvalTy));
8261}
8262
8263static unsigned getConstantTripCount(const SCEVConstant *ExitCount) {
8264 if (!ExitCount)
8265 return 0;
8266
8267 ConstantInt *ExitConst = ExitCount->getValue();
8268
8269 // Guard against huge trip counts.
8270 if (ExitConst->getValue().getActiveBits() > 32)
8271 return 0;
8272
8273 // In case of integer overflow, this returns 0, which is correct.
8274 return ((unsigned)ExitConst->getZExtValue()) + 1;
8275}
8276
8278 auto *ExitCount = dyn_cast<SCEVConstant>(getBackedgeTakenCount(L, Exact));
8279 return getConstantTripCount(ExitCount);
8280}
8281
8282unsigned
8284 const BasicBlock *ExitingBlock) {
8285 assert(ExitingBlock && "Must pass a non-null exiting block!");
8286 assert(L->isLoopExiting(ExitingBlock) &&
8287 "Exiting block must actually branch out of the loop!");
8288 const SCEVConstant *ExitCount =
8289 dyn_cast<SCEVConstant>(getExitCount(L, ExitingBlock));
8290 return getConstantTripCount(ExitCount);
8291}
8292
8294 const Loop *L, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8295
8296 const auto *MaxExitCount =
8297 Predicates ? getPredicatedConstantMaxBackedgeTakenCount(L, *Predicates)
8299 return getConstantTripCount(dyn_cast<SCEVConstant>(MaxExitCount));
8300}
8301
8303 SmallVector<BasicBlock *, 8> ExitingBlocks;
8304 L->getExitingBlocks(ExitingBlocks);
8305
8306 std::optional<unsigned> Res;
8307 for (auto *ExitingBB : ExitingBlocks) {
8308 unsigned Multiple = getSmallConstantTripMultiple(L, ExitingBB);
8309 if (!Res)
8310 Res = Multiple;
8311 Res = std::gcd(*Res, Multiple);
8312 }
8313 return Res.value_or(1);
8314}
8315
8317 const SCEV *ExitCount) {
8318 if (isa<SCEVCouldNotCompute>(ExitCount))
8319 return 1;
8320
8321 // Get the trip count
8322 const SCEV *TCExpr = getTripCountFromExitCount(applyLoopGuards(ExitCount, L));
8323
8324 APInt Multiple = getNonZeroConstantMultiple(TCExpr);
8325 // If a trip multiple is huge (>=2^32), the trip count is still divisible by
8326 // the greatest power of 2 divisor less than 2^32.
8327 return Multiple.getActiveBits() > 32
8328 ? 1U << std::min(31U, Multiple.countTrailingZeros())
8329 : (unsigned)Multiple.getZExtValue();
8330}
8331
8332/// Returns the largest constant divisor of the trip count of this loop as a
8333/// normal unsigned value, if possible. This means that the actual trip count is
8334/// always a multiple of the returned value (don't forget the trip count could
8335/// very well be zero as well!).
8336///
8337/// Returns 1 if the trip count is unknown or not guaranteed to be the
8338/// multiple of a constant (which is also the case if the trip count is simply
8339/// constant, use getSmallConstantTripCount for that case), Will also return 1
8340/// if the trip count is very large (>= 2^32).
8341///
8342/// As explained in the comments for getSmallConstantTripCount, this assumes
8343/// that control exits the loop via ExitingBlock.
8344unsigned
8346 const BasicBlock *ExitingBlock) {
8347 assert(ExitingBlock && "Must pass a non-null exiting block!");
8348 assert(L->isLoopExiting(ExitingBlock) &&
8349 "Exiting block must actually branch out of the loop!");
8350 const SCEV *ExitCount = getExitCount(L, ExitingBlock);
8351 return getSmallConstantTripMultiple(L, ExitCount);
8352}
8353
8355 const BasicBlock *ExitingBlock,
8356 ExitCountKind Kind) {
8357 switch (Kind) {
8358 case Exact:
8359 return getBackedgeTakenInfo(L).getExact(ExitingBlock, this);
8360 case SymbolicMaximum:
8361 return getBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, this);
8362 case ConstantMaximum:
8363 return getBackedgeTakenInfo(L).getConstantMax(ExitingBlock, this);
8364 };
8365 llvm_unreachable("Invalid ExitCountKind!");
8366}
8367
8369 const Loop *L, const BasicBlock *ExitingBlock,
8371 switch (Kind) {
8372 case Exact:
8373 return getPredicatedBackedgeTakenInfo(L).getExact(ExitingBlock, this,
8374 Predicates);
8375 case SymbolicMaximum:
8376 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, this,
8377 Predicates);
8378 case ConstantMaximum:
8379 return getPredicatedBackedgeTakenInfo(L).getConstantMax(ExitingBlock, this,
8380 Predicates);
8381 };
8382 llvm_unreachable("Invalid ExitCountKind!");
8383}
8384
8387 return getPredicatedBackedgeTakenInfo(L).getExact(L, this, &Preds);
8388}
8389
8391 ExitCountKind Kind) {
8392 switch (Kind) {
8393 case Exact:
8394 return getBackedgeTakenInfo(L).getExact(L, this);
8395 case ConstantMaximum:
8396 return getBackedgeTakenInfo(L).getConstantMax(this);
8397 case SymbolicMaximum:
8398 return getBackedgeTakenInfo(L).getSymbolicMax(L, this);
8399 };
8400 llvm_unreachable("Invalid ExitCountKind!");
8401}
8402
8405 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(L, this, &Preds);
8406}
8407
8410 return getPredicatedBackedgeTakenInfo(L).getConstantMax(this, &Preds);
8411}
8412
8414 return getBackedgeTakenInfo(L).isConstantMaxOrZero(this);
8415}
8416
8417/// Push PHI nodes in the header of the given loop onto the given Worklist.
8418static void PushLoopPHIs(const Loop *L,
8421 BasicBlock *Header = L->getHeader();
8422
8423 // Push all Loop-header PHIs onto the Worklist stack.
8424 for (PHINode &PN : Header->phis())
8425 if (Visited.insert(&PN).second)
8426 Worklist.push_back(&PN);
8427}
8428
8429ScalarEvolution::BackedgeTakenInfo &
8430ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) {
8431 auto &BTI = getBackedgeTakenInfo(L);
8432 if (BTI.hasFullInfo())
8433 return BTI;
8434
8435 auto Pair = PredicatedBackedgeTakenCounts.try_emplace(L);
8436
8437 if (!Pair.second)
8438 return Pair.first->second;
8439
8440 BackedgeTakenInfo Result =
8441 computeBackedgeTakenCount(L, /*AllowPredicates=*/true);
8442
8443 return PredicatedBackedgeTakenCounts.find(L)->second = std::move(Result);
8444}
8445
8446ScalarEvolution::BackedgeTakenInfo &
8447ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
8448 // Initially insert an invalid entry for this loop. If the insertion
8449 // succeeds, proceed to actually compute a backedge-taken count and
8450 // update the value. The temporary CouldNotCompute value tells SCEV
8451 // code elsewhere that it shouldn't attempt to request a new
8452 // backedge-taken count, which could result in infinite recursion.
8453 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
8454 BackedgeTakenCounts.try_emplace(L);
8455 if (!Pair.second)
8456 return Pair.first->second;
8457
8458 // computeBackedgeTakenCount may allocate memory for its result. Inserting it
8459 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
8460 // must be cleared in this scope.
8461 BackedgeTakenInfo Result = computeBackedgeTakenCount(L);
8462
8463 // Now that we know more about the trip count for this loop, forget any
8464 // existing SCEV values for PHI nodes in this loop since they are only
8465 // conservative estimates made without the benefit of trip count
8466 // information. This invalidation is not necessary for correctness, and is
8467 // only done to produce more precise results.
8468 if (Result.hasAnyInfo()) {
8469 // Invalidate any expression using an addrec in this loop.
8471 auto LoopUsersIt = LoopUsers.find(L);
8472 if (LoopUsersIt != LoopUsers.end())
8473 append_range(ToForget, LoopUsersIt->second);
8474 forgetMemoizedResults(ToForget);
8475
8476 // Invalidate constant-evolved loop header phis.
8477 for (PHINode &PN : L->getHeader()->phis())
8478 ConstantEvolutionLoopExitValue.erase(&PN);
8479 }
8480
8481 // Re-lookup the insert position, since the call to
8482 // computeBackedgeTakenCount above could result in a
8483 // recusive call to getBackedgeTakenInfo (on a different
8484 // loop), which would invalidate the iterator computed
8485 // earlier.
8486 return BackedgeTakenCounts.find(L)->second = std::move(Result);
8487}
8488
8490 // This method is intended to forget all info about loops. It should
8491 // invalidate caches as if the following happened:
8492 // - The trip counts of all loops have changed arbitrarily
8493 // - Every llvm::Value has been updated in place to produce a different
8494 // result.
8495 BackedgeTakenCounts.clear();
8496 PredicatedBackedgeTakenCounts.clear();
8497 BECountUsers.clear();
8498 LoopPropertiesCache.clear();
8499 ConstantEvolutionLoopExitValue.clear();
8500 ValueExprMap.clear();
8501 ValuesAtScopes.clear();
8502 ValuesAtScopesUsers.clear();
8503 LoopDispositions.clear();
8504 BlockDispositions.clear();
8505 UnsignedRanges.clear();
8506 SignedRanges.clear();
8507 ExprValueMap.clear();
8508 HasRecMap.clear();
8509 ConstantMultipleCache.clear();
8510 PredicatedSCEVRewrites.clear();
8511 FoldCache.clear();
8512 FoldCacheUser.clear();
8513}
8514void ScalarEvolution::visitAndClearUsers(
8518 while (!Worklist.empty()) {
8519 Instruction *I = Worklist.pop_back_val();
8520 if (!isSCEVable(I->getType()) && !isa<WithOverflowInst>(I))
8521 continue;
8522
8524 ValueExprMap.find_as(static_cast<Value *>(I));
8525 if (It != ValueExprMap.end()) {
8526 eraseValueFromMap(It->first);
8527 ToForget.push_back(It->second);
8528 if (PHINode *PN = dyn_cast<PHINode>(I))
8529 ConstantEvolutionLoopExitValue.erase(PN);
8530 }
8531
8532 PushDefUseChildren(I, Worklist, Visited);
8533 }
8534}
8535
8537 SmallVector<const Loop *, 16> LoopWorklist(1, L);
8541
8542 // Iterate over all the loops and sub-loops to drop SCEV information.
8543 while (!LoopWorklist.empty()) {
8544 auto *CurrL = LoopWorklist.pop_back_val();
8545
8546 // Drop any stored trip count value.
8547 forgetBackedgeTakenCounts(CurrL, /* Predicated */ false);
8548 forgetBackedgeTakenCounts(CurrL, /* Predicated */ true);
8549
8550 // Drop information about predicated SCEV rewrites for this loop.
8551 for (auto I = PredicatedSCEVRewrites.begin();
8552 I != PredicatedSCEVRewrites.end();) {
8553 std::pair<const SCEV *, const Loop *> Entry = I->first;
8554 if (Entry.second == CurrL)
8555 PredicatedSCEVRewrites.erase(I++);
8556 else
8557 ++I;
8558 }
8559
8560 auto LoopUsersItr = LoopUsers.find(CurrL);
8561 if (LoopUsersItr != LoopUsers.end())
8562 llvm::append_range(ToForget, LoopUsersItr->second);
8563
8564 // Drop information about expressions based on loop-header PHIs.
8565 PushLoopPHIs(CurrL, Worklist, Visited);
8566 visitAndClearUsers(Worklist, Visited, ToForget);
8567
8568 LoopPropertiesCache.erase(CurrL);
8569 // Forget all contained loops too, to avoid dangling entries in the
8570 // ValuesAtScopes map.
8571 LoopWorklist.append(CurrL->begin(), CurrL->end());
8572 }
8573 forgetMemoizedResults(ToForget);
8574}
8575
8577 forgetLoop(L->getOutermostLoop());
8578}
8579
8582 if (!I) return;
8583
8584 // Drop information about expressions based on loop-header PHIs.
8588 Worklist.push_back(I);
8589 Visited.insert(I);
8590 visitAndClearUsers(Worklist, Visited, ToForget);
8591
8592 forgetMemoizedResults(ToForget);
8593}
8594
8596 if (!isSCEVable(V->getType()))
8597 return;
8598
8599 // If SCEV looked through a trivial LCSSA phi node, we might have SCEV's
8600 // directly using a SCEVUnknown/SCEVAddRec defined in the loop. After an
8601 // extra predecessor is added, this is no longer valid. Find all Unknowns and
8602 // AddRecs defined in the loop and invalidate any SCEV's making use of them.
8603 if (const SCEV *S = getExistingSCEV(V)) {
8604 struct InvalidationRootCollector {
8605 Loop *L;
8607
8608 InvalidationRootCollector(Loop *L) : L(L) {}
8609
8610 bool follow(const SCEV *S) {
8611 if (auto *SU = dyn_cast<SCEVUnknown>(S)) {
8612 if (auto *I = dyn_cast<Instruction>(SU->getValue()))
8613 if (L->contains(I))
8614 Roots.push_back(S);
8615 } else if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(S)) {
8616 if (L->contains(AddRec->getLoop()))
8617 Roots.push_back(S);
8618 }
8619 return true;
8620 }
8621 bool isDone() const { return false; }
8622 };
8623
8624 InvalidationRootCollector C(L);
8625 visitAll(S, C);
8626 forgetMemoizedResults(C.Roots);
8627 }
8628
8629 // Also perform the normal invalidation.
8630 forgetValue(V);
8631}
8632
8633void ScalarEvolution::forgetLoopDispositions() { LoopDispositions.clear(); }
8634
8636 // Unless a specific value is passed to invalidation, completely clear both
8637 // caches.
8638 if (!V) {
8639 BlockDispositions.clear();
8640 LoopDispositions.clear();
8641 return;
8642 }
8643
8644 if (!isSCEVable(V->getType()))
8645 return;
8646
8647 const SCEV *S = getExistingSCEV(V);
8648 if (!S)
8649 return;
8650
8651 // Invalidate the block and loop dispositions cached for S. Dispositions of
8652 // S's users may change if S's disposition changes (i.e. a user may change to
8653 // loop-invariant, if S changes to loop invariant), so also invalidate
8654 // dispositions of S's users recursively.
8655 SmallVector<const SCEV *, 8> Worklist = {S};
8657 while (!Worklist.empty()) {
8658 const SCEV *Curr = Worklist.pop_back_val();
8659 bool LoopDispoRemoved = LoopDispositions.erase(Curr);
8660 bool BlockDispoRemoved = BlockDispositions.erase(Curr);
8661 if (!LoopDispoRemoved && !BlockDispoRemoved)
8662 continue;
8663 auto Users = SCEVUsers.find(Curr);
8664 if (Users != SCEVUsers.end())
8665 for (const auto *User : Users->second)
8666 if (Seen.insert(User).second)
8667 Worklist.push_back(User);
8668 }
8669}
8670
8671/// Get the exact loop backedge taken count considering all loop exits. A
8672/// computable result can only be returned for loops with all exiting blocks
8673/// dominating the latch. howFarToZero assumes that the limit of each loop test
8674/// is never skipped. This is a valid assumption as long as the loop exits via
8675/// that test. For precise results, it is the caller's responsibility to specify
8676/// the relevant loop exiting block using getExact(ExitingBlock, SE).
8677const SCEV *ScalarEvolution::BackedgeTakenInfo::getExact(
8678 const Loop *L, ScalarEvolution *SE,
8680 // If any exits were not computable, the loop is not computable.
8681 if (!isComplete() || ExitNotTaken.empty())
8682 return SE->getCouldNotCompute();
8683
8684 const BasicBlock *Latch = L->getLoopLatch();
8685 // All exiting blocks we have collected must dominate the only backedge.
8686 if (!Latch)
8687 return SE->getCouldNotCompute();
8688
8689 // All exiting blocks we have gathered dominate loop's latch, so exact trip
8690 // count is simply a minimum out of all these calculated exit counts.
8692 for (const auto &ENT : ExitNotTaken) {
8693 const SCEV *BECount = ENT.ExactNotTaken;
8694 assert(BECount != SE->getCouldNotCompute() && "Bad exit SCEV!");
8695 assert(SE->DT.dominates(ENT.ExitingBlock, Latch) &&
8696 "We should only have known counts for exiting blocks that dominate "
8697 "latch!");
8698
8699 Ops.push_back(BECount);
8700
8701 if (Preds)
8702 append_range(*Preds, ENT.Predicates);
8703
8704 assert((Preds || ENT.hasAlwaysTruePredicate()) &&
8705 "Predicate should be always true!");
8706 }
8707
8708 // If an earlier exit exits on the first iteration (exit count zero), then
8709 // a later poison exit count should not propagate into the result. This are
8710 // exactly the semantics provided by umin_seq.
8711 return SE->getUMinFromMismatchedTypes(Ops, /* Sequential */ true);
8712}
8713
8714const ScalarEvolution::ExitNotTakenInfo *
8715ScalarEvolution::BackedgeTakenInfo::getExitNotTaken(
8716 const BasicBlock *ExitingBlock,
8717 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8718 for (const auto &ENT : ExitNotTaken)
8719 if (ENT.ExitingBlock == ExitingBlock) {
8720 if (ENT.hasAlwaysTruePredicate())
8721 return &ENT;
8722 else if (Predicates) {
8723 append_range(*Predicates, ENT.Predicates);
8724 return &ENT;
8725 }
8726 }
8727
8728 return nullptr;
8729}
8730
8731/// getConstantMax - Get the constant max backedge taken count for the loop.
8732const SCEV *ScalarEvolution::BackedgeTakenInfo::getConstantMax(
8733 ScalarEvolution *SE,
8734 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8735 if (!getConstantMax())
8736 return SE->getCouldNotCompute();
8737
8738 for (const auto &ENT : ExitNotTaken)
8739 if (!ENT.hasAlwaysTruePredicate()) {
8740 if (!Predicates)
8741 return SE->getCouldNotCompute();
8742 append_range(*Predicates, ENT.Predicates);
8743 }
8744
8745 assert((isa<SCEVCouldNotCompute>(getConstantMax()) ||
8746 isa<SCEVConstant>(getConstantMax())) &&
8747 "No point in having a non-constant max backedge taken count!");
8748 return getConstantMax();
8749}
8750
8751const SCEV *ScalarEvolution::BackedgeTakenInfo::getSymbolicMax(
8752 const Loop *L, ScalarEvolution *SE,
8753 SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8754 if (!SymbolicMax) {
8755 // Form an expression for the maximum exit count possible for this loop. We
8756 // merge the max and exact information to approximate a version of
8757 // getConstantMaxBackedgeTakenCount which isn't restricted to just
8758 // constants.
8760
8761 for (const auto &ENT : ExitNotTaken) {
8762 const SCEV *ExitCount = ENT.SymbolicMaxNotTaken;
8763 if (!isa<SCEVCouldNotCompute>(ExitCount)) {
8764 assert(SE->DT.dominates(ENT.ExitingBlock, L->getLoopLatch()) &&
8765 "We should only have known counts for exiting blocks that "
8766 "dominate latch!");
8767 ExitCounts.push_back(ExitCount);
8768 if (Predicates)
8769 append_range(*Predicates, ENT.Predicates);
8770
8771 assert((Predicates || ENT.hasAlwaysTruePredicate()) &&
8772 "Predicate should be always true!");
8773 }
8774 }
8775 if (ExitCounts.empty())
8776 SymbolicMax = SE->getCouldNotCompute();
8777 else
8778 SymbolicMax =
8779 SE->getUMinFromMismatchedTypes(ExitCounts, /*Sequential*/ true);
8780 }
8781 return SymbolicMax;
8782}
8783
8784bool ScalarEvolution::BackedgeTakenInfo::isConstantMaxOrZero(
8785 ScalarEvolution *SE) const {
8786 auto PredicateNotAlwaysTrue = [](const ExitNotTakenInfo &ENT) {
8787 return !ENT.hasAlwaysTruePredicate();
8788 };
8789 return MaxOrZero && !any_of(ExitNotTaken, PredicateNotAlwaysTrue);
8790}
8791
8794
8796 const SCEV *E, const SCEV *ConstantMaxNotTaken,
8797 const SCEV *SymbolicMaxNotTaken, bool MaxOrZero,
8801 // If we prove the max count is zero, so is the symbolic bound. This happens
8802 // in practice due to differences in a) how context sensitive we've chosen
8803 // to be and b) how we reason about bounds implied by UB.
8804 if (ConstantMaxNotTaken->isZero()) {
8805 this->ExactNotTaken = E = ConstantMaxNotTaken;
8806 this->SymbolicMaxNotTaken = SymbolicMaxNotTaken = ConstantMaxNotTaken;
8807 }
8808
8811 "Exact is not allowed to be less precise than Constant Max");
8814 "Exact is not allowed to be less precise than Symbolic Max");
8817 "Symbolic Max is not allowed to be less precise than Constant Max");
8820 "No point in having a non-constant max backedge taken count!");
8822 for (const auto PredList : PredLists)
8823 for (const auto *P : PredList) {
8824 if (SeenPreds.contains(P))
8825 continue;
8826 assert(!isa<SCEVUnionPredicate>(P) && "Only add leaf predicates here!");
8827 SeenPreds.insert(P);
8828 Predicates.push_back(P);
8829 }
8830 assert((isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) &&
8831 "Backedge count should be int");
8833 !ConstantMaxNotTaken->getType()->isPointerTy()) &&
8834 "Max backedge count should be int");
8835}
8836
8844
8845/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
8846/// computable exit into a persistent ExitNotTakenInfo array.
8847ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
8849 bool IsComplete, const SCEV *ConstantMax, bool MaxOrZero)
8850 : ConstantMax(ConstantMax), IsComplete(IsComplete), MaxOrZero(MaxOrZero) {
8851 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8852
8853 ExitNotTaken.reserve(ExitCounts.size());
8854 std::transform(ExitCounts.begin(), ExitCounts.end(),
8855 std::back_inserter(ExitNotTaken),
8856 [&](const EdgeExitInfo &EEI) {
8857 BasicBlock *ExitBB = EEI.first;
8858 const ExitLimit &EL = EEI.second;
8859 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken,
8860 EL.ConstantMaxNotTaken, EL.SymbolicMaxNotTaken,
8861 EL.Predicates);
8862 });
8863 assert((isa<SCEVCouldNotCompute>(ConstantMax) ||
8864 isa<SCEVConstant>(ConstantMax)) &&
8865 "No point in having a non-constant max backedge taken count!");
8866}
8867
8868/// Compute the number of times the backedge of the specified loop will execute.
8869ScalarEvolution::BackedgeTakenInfo
8870ScalarEvolution::computeBackedgeTakenCount(const Loop *L,
8871 bool AllowPredicates) {
8872 SmallVector<BasicBlock *, 8> ExitingBlocks;
8873 L->getExitingBlocks(ExitingBlocks);
8874
8875 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8876
8878 bool CouldComputeBECount = true;
8879 BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
8880 const SCEV *MustExitMaxBECount = nullptr;
8881 const SCEV *MayExitMaxBECount = nullptr;
8882 bool MustExitMaxOrZero = false;
8883 bool IsOnlyExit = ExitingBlocks.size() == 1;
8884
8885 // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts
8886 // and compute maxBECount.
8887 // Do a union of all the predicates here.
8888 for (BasicBlock *ExitBB : ExitingBlocks) {
8889 // We canonicalize untaken exits to br (constant), ignore them so that
8890 // proving an exit untaken doesn't negatively impact our ability to reason
8891 // about the loop as whole.
8892 if (auto *BI = dyn_cast<BranchInst>(ExitBB->getTerminator()))
8893 if (auto *CI = dyn_cast<ConstantInt>(BI->getCondition())) {
8894 bool ExitIfTrue = !L->contains(BI->getSuccessor(0));
8895 if (ExitIfTrue == CI->isZero())
8896 continue;
8897 }
8898
8899 ExitLimit EL = computeExitLimit(L, ExitBB, IsOnlyExit, AllowPredicates);
8900
8901 assert((AllowPredicates || EL.Predicates.empty()) &&
8902 "Predicated exit limit when predicates are not allowed!");
8903
8904 // 1. For each exit that can be computed, add an entry to ExitCounts.
8905 // CouldComputeBECount is true only if all exits can be computed.
8906 if (EL.ExactNotTaken != getCouldNotCompute())
8907 ++NumExitCountsComputed;
8908 else
8909 // We couldn't compute an exact value for this exit, so
8910 // we won't be able to compute an exact value for the loop.
8911 CouldComputeBECount = false;
8912 // Remember exit count if either exact or symbolic is known. Because
8913 // Exact always implies symbolic, only check symbolic.
8914 if (EL.SymbolicMaxNotTaken != getCouldNotCompute())
8915 ExitCounts.emplace_back(ExitBB, EL);
8916 else {
8917 assert(EL.ExactNotTaken == getCouldNotCompute() &&
8918 "Exact is known but symbolic isn't?");
8919 ++NumExitCountsNotComputed;
8920 }
8921
8922 // 2. Derive the loop's MaxBECount from each exit's max number of
8923 // non-exiting iterations. Partition the loop exits into two kinds:
8924 // LoopMustExits and LoopMayExits.
8925 //
8926 // If the exit dominates the loop latch, it is a LoopMustExit otherwise it
8927 // is a LoopMayExit. If any computable LoopMustExit is found, then
8928 // MaxBECount is the minimum EL.ConstantMaxNotTaken of computable
8929 // LoopMustExits. Otherwise, MaxBECount is conservatively the maximum
8930 // EL.ConstantMaxNotTaken, where CouldNotCompute is considered greater than
8931 // any
8932 // computable EL.ConstantMaxNotTaken.
8933 if (EL.ConstantMaxNotTaken != getCouldNotCompute() && Latch &&
8934 DT.dominates(ExitBB, Latch)) {
8935 if (!MustExitMaxBECount) {
8936 MustExitMaxBECount = EL.ConstantMaxNotTaken;
8937 MustExitMaxOrZero = EL.MaxOrZero;
8938 } else {
8939 MustExitMaxBECount = getUMinFromMismatchedTypes(MustExitMaxBECount,
8940 EL.ConstantMaxNotTaken);
8941 }
8942 } else if (MayExitMaxBECount != getCouldNotCompute()) {
8943 if (!MayExitMaxBECount || EL.ConstantMaxNotTaken == getCouldNotCompute())
8944 MayExitMaxBECount = EL.ConstantMaxNotTaken;
8945 else {
8946 MayExitMaxBECount = getUMaxFromMismatchedTypes(MayExitMaxBECount,
8947 EL.ConstantMaxNotTaken);
8948 }
8949 }
8950 }
8951 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
8952 (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute());
8953 // The loop backedge will be taken the maximum or zero times if there's
8954 // a single exit that must be taken the maximum or zero times.
8955 bool MaxOrZero = (MustExitMaxOrZero && ExitingBlocks.size() == 1);
8956
8957 // Remember which SCEVs are used in exit limits for invalidation purposes.
8958 // We only care about non-constant SCEVs here, so we can ignore
8959 // EL.ConstantMaxNotTaken
8960 // and MaxBECount, which must be SCEVConstant.
8961 for (const auto &Pair : ExitCounts) {
8962 if (!isa<SCEVConstant>(Pair.second.ExactNotTaken))
8963 BECountUsers[Pair.second.ExactNotTaken].insert({L, AllowPredicates});
8964 if (!isa<SCEVConstant>(Pair.second.SymbolicMaxNotTaken))
8965 BECountUsers[Pair.second.SymbolicMaxNotTaken].insert(
8966 {L, AllowPredicates});
8967 }
8968 return BackedgeTakenInfo(std::move(ExitCounts), CouldComputeBECount,
8969 MaxBECount, MaxOrZero);
8970}
8971
8972ScalarEvolution::ExitLimit
8973ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock,
8974 bool IsOnlyExit, bool AllowPredicates) {
8975 assert(L->contains(ExitingBlock) && "Exit count for non-loop block?");
8976 // If our exiting block does not dominate the latch, then its connection with
8977 // loop's exit limit may be far from trivial.
8978 const BasicBlock *Latch = L->getLoopLatch();
8979 if (!Latch || !DT.dominates(ExitingBlock, Latch))
8980 return getCouldNotCompute();
8981
8982 Instruction *Term = ExitingBlock->getTerminator();
8983 if (BranchInst *BI = dyn_cast<BranchInst>(Term)) {
8984 assert(BI->isConditional() && "If unconditional, it can't be in loop!");
8985 bool ExitIfTrue = !L->contains(BI->getSuccessor(0));
8986 assert(ExitIfTrue == L->contains(BI->getSuccessor(1)) &&
8987 "It should have one successor in loop and one exit block!");
8988 // Proceed to the next level to examine the exit condition expression.
8989 return computeExitLimitFromCond(L, BI->getCondition(), ExitIfTrue,
8990 /*ControlsOnlyExit=*/IsOnlyExit,
8991 AllowPredicates);
8992 }
8993
8994 if (SwitchInst *SI = dyn_cast<SwitchInst>(Term)) {
8995 // For switch, make sure that there is a single exit from the loop.
8996 BasicBlock *Exit = nullptr;
8997 for (auto *SBB : successors(ExitingBlock))
8998 if (!L->contains(SBB)) {
8999 if (Exit) // Multiple exit successors.
9000 return getCouldNotCompute();
9001 Exit = SBB;
9002 }
9003 assert(Exit && "Exiting block must have at least one exit");
9004 return computeExitLimitFromSingleExitSwitch(
9005 L, SI, Exit, /*ControlsOnlyExit=*/IsOnlyExit);
9006 }
9007
9008 return getCouldNotCompute();
9009}
9010
9012 const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9013 bool AllowPredicates) {
9014 ScalarEvolution::ExitLimitCacheTy Cache(L, ExitIfTrue, AllowPredicates);
9015 return computeExitLimitFromCondCached(Cache, L, ExitCond, ExitIfTrue,
9016 ControlsOnlyExit, AllowPredicates);
9017}
9018
9019std::optional<ScalarEvolution::ExitLimit>
9020ScalarEvolution::ExitLimitCache::find(const Loop *L, Value *ExitCond,
9021 bool ExitIfTrue, bool ControlsOnlyExit,
9022 bool AllowPredicates) {
9023 (void)this->L;
9024 (void)this->ExitIfTrue;
9025 (void)this->AllowPredicates;
9026
9027 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9028 this->AllowPredicates == AllowPredicates &&
9029 "Variance in assumed invariant key components!");
9030 auto Itr = TripCountMap.find({ExitCond, ControlsOnlyExit});
9031 if (Itr == TripCountMap.end())
9032 return std::nullopt;
9033 return Itr->second;
9034}
9035
9036void ScalarEvolution::ExitLimitCache::insert(const Loop *L, Value *ExitCond,
9037 bool ExitIfTrue,
9038 bool ControlsOnlyExit,
9039 bool AllowPredicates,
9040 const ExitLimit &EL) {
9041 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9042 this->AllowPredicates == AllowPredicates &&
9043 "Variance in assumed invariant key components!");
9044
9045 auto InsertResult = TripCountMap.insert({{ExitCond, ControlsOnlyExit}, EL});
9046 assert(InsertResult.second && "Expected successful insertion!");
9047 (void)InsertResult;
9048 (void)ExitIfTrue;
9049}
9050
9051ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondCached(
9052 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9053 bool ControlsOnlyExit, bool AllowPredicates) {
9054
9055 if (auto MaybeEL = Cache.find(L, ExitCond, ExitIfTrue, ControlsOnlyExit,
9056 AllowPredicates))
9057 return *MaybeEL;
9058
9059 ExitLimit EL = computeExitLimitFromCondImpl(
9060 Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates);
9061 Cache.insert(L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates, EL);
9062 return EL;
9063}
9064
9065ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl(
9066 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9067 bool ControlsOnlyExit, bool AllowPredicates) {
9068 // Handle BinOp conditions (And, Or).
9069 if (auto LimitFromBinOp = computeExitLimitFromCondFromBinOp(
9070 Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates))
9071 return *LimitFromBinOp;
9072
9073 // With an icmp, it may be feasible to compute an exact backedge-taken count.
9074 // Proceed to the next level to examine the icmp.
9075 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond)) {
9076 ExitLimit EL =
9077 computeExitLimitFromICmp(L, ExitCondICmp, ExitIfTrue, ControlsOnlyExit);
9078 if (EL.hasFullInfo() || !AllowPredicates)
9079 return EL;
9080
9081 // Try again, but use SCEV predicates this time.
9082 return computeExitLimitFromICmp(L, ExitCondICmp, ExitIfTrue,
9083 ControlsOnlyExit,
9084 /*AllowPredicates=*/true);
9085 }
9086
9087 // Check for a constant condition. These are normally stripped out by
9088 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
9089 // preserve the CFG and is temporarily leaving constant conditions
9090 // in place.
9091 if (ConstantInt *CI = dyn_cast<ConstantInt>(ExitCond)) {
9092 if (ExitIfTrue == !CI->getZExtValue())
9093 // The backedge is always taken.
9094 return getCouldNotCompute();
9095 // The backedge is never taken.
9096 return getZero(CI->getType());
9097 }
9098
9099 // If we're exiting based on the overflow flag of an x.with.overflow intrinsic
9100 // with a constant step, we can form an equivalent icmp predicate and figure
9101 // out how many iterations will be taken before we exit.
9102 const WithOverflowInst *WO;
9103 const APInt *C;
9104 if (match(ExitCond, m_ExtractValue<1>(m_WithOverflowInst(WO))) &&
9105 match(WO->getRHS(), m_APInt(C))) {
9106 ConstantRange NWR =
9108 WO->getNoWrapKind());
9109 CmpInst::Predicate Pred;
9110 APInt NewRHSC, Offset;
9111 NWR.getEquivalentICmp(Pred, NewRHSC, Offset);
9112 if (!ExitIfTrue)
9113 Pred = ICmpInst::getInversePredicate(Pred);
9114 auto *LHS = getSCEV(WO->getLHS());
9115 if (Offset != 0)
9117 auto EL = computeExitLimitFromICmp(L, Pred, LHS, getConstant(NewRHSC),
9118 ControlsOnlyExit, AllowPredicates);
9119 if (EL.hasAnyInfo())
9120 return EL;
9121 }
9122
9123 // If it's not an integer or pointer comparison then compute it the hard way.
9124 return computeExitCountExhaustively(L, ExitCond, ExitIfTrue);
9125}
9126
9127std::optional<ScalarEvolution::ExitLimit>
9128ScalarEvolution::computeExitLimitFromCondFromBinOp(
9129 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9130 bool ControlsOnlyExit, bool AllowPredicates) {
9131 // Check if the controlling expression for this loop is an And or Or.
9132 Value *Op0, *Op1;
9133 bool IsAnd = false;
9134 if (match(ExitCond, m_LogicalAnd(m_Value(Op0), m_Value(Op1))))
9135 IsAnd = true;
9136 else if (match(ExitCond, m_LogicalOr(m_Value(Op0), m_Value(Op1))))
9137 IsAnd = false;
9138 else
9139 return std::nullopt;
9140
9141 // EitherMayExit is true in these two cases:
9142 // br (and Op0 Op1), loop, exit
9143 // br (or Op0 Op1), exit, loop
9144 bool EitherMayExit = IsAnd ^ ExitIfTrue;
9145 ExitLimit EL0 = computeExitLimitFromCondCached(
9146 Cache, L, Op0, ExitIfTrue, ControlsOnlyExit && !EitherMayExit,
9147 AllowPredicates);
9148 ExitLimit EL1 = computeExitLimitFromCondCached(
9149 Cache, L, Op1, ExitIfTrue, ControlsOnlyExit && !EitherMayExit,
9150 AllowPredicates);
9151
9152 // Be robust against unsimplified IR for the form "op i1 X, NeutralElement"
9153 const Constant *NeutralElement = ConstantInt::get(ExitCond->getType(), IsAnd);
9154 if (isa<ConstantInt>(Op1))
9155 return Op1 == NeutralElement ? EL0 : EL1;
9156 if (isa<ConstantInt>(Op0))
9157 return Op0 == NeutralElement ? EL1 : EL0;
9158
9159 const SCEV *BECount = getCouldNotCompute();
9160 const SCEV *ConstantMaxBECount = getCouldNotCompute();
9161 const SCEV *SymbolicMaxBECount = getCouldNotCompute();
9162 if (EitherMayExit) {
9163 bool UseSequentialUMin = !isa<BinaryOperator>(ExitCond);
9164 // Both conditions must be same for the loop to continue executing.
9165 // Choose the less conservative count.
9166 if (EL0.ExactNotTaken != getCouldNotCompute() &&
9167 EL1.ExactNotTaken != getCouldNotCompute()) {
9168 BECount = getUMinFromMismatchedTypes(EL0.ExactNotTaken, EL1.ExactNotTaken,
9169 UseSequentialUMin);
9170 }
9171 if (EL0.ConstantMaxNotTaken == getCouldNotCompute())
9172 ConstantMaxBECount = EL1.ConstantMaxNotTaken;
9173 else if (EL1.ConstantMaxNotTaken == getCouldNotCompute())
9174 ConstantMaxBECount = EL0.ConstantMaxNotTaken;
9175 else
9176 ConstantMaxBECount = getUMinFromMismatchedTypes(EL0.ConstantMaxNotTaken,
9177 EL1.ConstantMaxNotTaken);
9178 if (EL0.SymbolicMaxNotTaken == getCouldNotCompute())
9179 SymbolicMaxBECount = EL1.SymbolicMaxNotTaken;
9180 else if (EL1.SymbolicMaxNotTaken == getCouldNotCompute())
9181 SymbolicMaxBECount = EL0.SymbolicMaxNotTaken;
9182 else
9183 SymbolicMaxBECount = getUMinFromMismatchedTypes(
9184 EL0.SymbolicMaxNotTaken, EL1.SymbolicMaxNotTaken, UseSequentialUMin);
9185 } else {
9186 // Both conditions must be same at the same time for the loop to exit.
9187 // For now, be conservative.
9188 if (EL0.ExactNotTaken == EL1.ExactNotTaken)
9189 BECount = EL0.ExactNotTaken;
9190 }
9191
9192 // There are cases (e.g. PR26207) where computeExitLimitFromCond is able
9193 // to be more aggressive when computing BECount than when computing
9194 // ConstantMaxBECount. In these cases it is possible for EL0.ExactNotTaken
9195 // and
9196 // EL1.ExactNotTaken to match, but for EL0.ConstantMaxNotTaken and
9197 // EL1.ConstantMaxNotTaken to not.
9198 if (isa<SCEVCouldNotCompute>(ConstantMaxBECount) &&
9199 !isa<SCEVCouldNotCompute>(BECount))
9200 ConstantMaxBECount = getConstant(getUnsignedRangeMax(BECount));
9201 if (isa<SCEVCouldNotCompute>(SymbolicMaxBECount))
9202 SymbolicMaxBECount =
9203 isa<SCEVCouldNotCompute>(BECount) ? ConstantMaxBECount : BECount;
9204 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
9205 {ArrayRef(EL0.Predicates), ArrayRef(EL1.Predicates)});
9206}
9207
9208ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9209 const Loop *L, ICmpInst *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9210 bool AllowPredicates) {
9211 // If the condition was exit on true, convert the condition to exit on false
9212 CmpPredicate Pred;
9213 if (!ExitIfTrue)
9214 Pred = ExitCond->getCmpPredicate();
9215 else
9216 Pred = ExitCond->getInverseCmpPredicate();
9217 const ICmpInst::Predicate OriginalPred = Pred;
9218
9219 const SCEV *LHS = getSCEV(ExitCond->getOperand(0));
9220 const SCEV *RHS = getSCEV(ExitCond->getOperand(1));
9221
9222 ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, ControlsOnlyExit,
9223 AllowPredicates);
9224 if (EL.hasAnyInfo())
9225 return EL;
9226
9227 auto *ExhaustiveCount =
9228 computeExitCountExhaustively(L, ExitCond, ExitIfTrue);
9229
9230 if (!isa<SCEVCouldNotCompute>(ExhaustiveCount))
9231 return ExhaustiveCount;
9232
9233 return computeShiftCompareExitLimit(ExitCond->getOperand(0),
9234 ExitCond->getOperand(1), L, OriginalPred);
9235}
9236ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9237 const Loop *L, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS,
9238 bool ControlsOnlyExit, bool AllowPredicates) {
9239
9240 // Try to evaluate any dependencies out of the loop.
9241 LHS = getSCEVAtScope(LHS, L);
9242 RHS = getSCEVAtScope(RHS, L);
9243
9244 // At this point, we would like to compute how many iterations of the
9245 // loop the predicate will return true for these inputs.
9246 if (isLoopInvariant(LHS, L) && !isLoopInvariant(RHS, L)) {
9247 // If there is a loop-invariant, force it into the RHS.
9248 std::swap(LHS, RHS);
9250 }
9251
9252 bool ControllingFiniteLoop = ControlsOnlyExit && loopHasNoAbnormalExits(L) &&
9254 // Simplify the operands before analyzing them.
9255 (void)SimplifyICmpOperands(Pred, LHS, RHS, /*Depth=*/0);
9256
9257 // If we have a comparison of a chrec against a constant, try to use value
9258 // ranges to answer this query.
9259 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS))
9260 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS))
9261 if (AddRec->getLoop() == L) {
9262 // Form the constant range.
9263 ConstantRange CompRange =
9264 ConstantRange::makeExactICmpRegion(Pred, RHSC->getAPInt());
9265
9266 const SCEV *Ret = AddRec->getNumIterationsInRange(CompRange, *this);
9267 if (!isa<SCEVCouldNotCompute>(Ret)) return Ret;
9268 }
9269
9270 // If this loop must exit based on this condition (or execute undefined
9271 // behaviour), see if we can improve wrap flags. This is essentially
9272 // a must execute style proof.
9273 if (ControllingFiniteLoop && isLoopInvariant(RHS, L)) {
9274 // If we can prove the test sequence produced must repeat the same values
9275 // on self-wrap of the IV, then we can infer that IV doesn't self wrap
9276 // because if it did, we'd have an infinite (undefined) loop.
9277 // TODO: We can peel off any functions which are invertible *in L*. Loop
9278 // invariant terms are effectively constants for our purposes here.
9279 auto *InnerLHS = LHS;
9280 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(LHS))
9281 InnerLHS = ZExt->getOperand();
9282 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(InnerLHS);
9283 AR && !AR->hasNoSelfWrap() && AR->getLoop() == L && AR->isAffine() &&
9284 isKnownToBeAPowerOfTwo(AR->getStepRecurrence(*this), /*OrZero=*/true,
9285 /*OrNegative=*/true)) {
9286 auto Flags = AR->getNoWrapFlags();
9287 Flags = setFlags(Flags, SCEV::FlagNW);
9288 SmallVector<const SCEV *> Operands{AR->operands()};
9289 Flags = StrengthenNoWrapFlags(this, scAddRecExpr, Operands, Flags);
9290 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), Flags);
9291 }
9292
9293 // For a slt/ult condition with a positive step, can we prove nsw/nuw?
9294 // From no-self-wrap, this follows trivially from the fact that every
9295 // (un)signed-wrapped, but not self-wrapped value must be LT than the
9296 // last value before (un)signed wrap. Since we know that last value
9297 // didn't exit, nor will any smaller one.
9298 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT) {
9299 auto WrapType = Pred == ICmpInst::ICMP_SLT ? SCEV::FlagNSW : SCEV::FlagNUW;
9300 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS);
9301 AR && AR->getLoop() == L && AR->isAffine() &&
9302 !AR->getNoWrapFlags(WrapType) && AR->hasNoSelfWrap() &&
9303 isKnownPositive(AR->getStepRecurrence(*this))) {
9304 auto Flags = AR->getNoWrapFlags();
9305 Flags = setFlags(Flags, WrapType);
9306 SmallVector<const SCEV*> Operands{AR->operands()};
9307 Flags = StrengthenNoWrapFlags(this, scAddRecExpr, Operands, Flags);
9308 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), Flags);
9309 }
9310 }
9311 }
9312
9313 switch (Pred) {
9314 case ICmpInst::ICMP_NE: { // while (X != Y)
9315 // Convert to: while (X-Y != 0)
9316 if (LHS->getType()->isPointerTy()) {
9319 return LHS;
9320 }
9321 if (RHS->getType()->isPointerTy()) {
9324 return RHS;
9325 }
9326 ExitLimit EL = howFarToZero(getMinusSCEV(LHS, RHS), L, ControlsOnlyExit,
9327 AllowPredicates);
9328 if (EL.hasAnyInfo())
9329 return EL;
9330 break;
9331 }
9332 case ICmpInst::ICMP_EQ: { // while (X == Y)
9333 // Convert to: while (X-Y == 0)
9334 if (LHS->getType()->isPointerTy()) {
9337 return LHS;
9338 }
9339 if (RHS->getType()->isPointerTy()) {
9342 return RHS;
9343 }
9344 ExitLimit EL = howFarToNonZero(getMinusSCEV(LHS, RHS), L);
9345 if (EL.hasAnyInfo()) return EL;
9346 break;
9347 }
9348 case ICmpInst::ICMP_SLE:
9349 case ICmpInst::ICMP_ULE:
9350 // Since the loop is finite, an invariant RHS cannot include the boundary
9351 // value, otherwise it would loop forever.
9352 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9353 !isLoopInvariant(RHS, L)) {
9354 // Otherwise, perform the addition in a wider type, to avoid overflow.
9355 // If the LHS is an addrec with the appropriate nowrap flag, the
9356 // extension will be sunk into it and the exit count can be analyzed.
9357 auto *OldType = dyn_cast<IntegerType>(LHS->getType());
9358 if (!OldType)
9359 break;
9360 // Prefer doubling the bitwidth over adding a single bit to make it more
9361 // likely that we use a legal type.
9362 auto *NewType =
9363 Type::getIntNTy(OldType->getContext(), OldType->getBitWidth() * 2);
9364 if (ICmpInst::isSigned(Pred)) {
9365 LHS = getSignExtendExpr(LHS, NewType);
9366 RHS = getSignExtendExpr(RHS, NewType);
9367 } else {
9368 LHS = getZeroExtendExpr(LHS, NewType);
9369 RHS = getZeroExtendExpr(RHS, NewType);
9370 }
9371 }
9373 [[fallthrough]];
9374 case ICmpInst::ICMP_SLT:
9375 case ICmpInst::ICMP_ULT: { // while (X < Y)
9376 bool IsSigned = ICmpInst::isSigned(Pred);
9377 ExitLimit EL = howManyLessThans(LHS, RHS, L, IsSigned, ControlsOnlyExit,
9378 AllowPredicates);
9379 if (EL.hasAnyInfo())
9380 return EL;
9381 break;
9382 }
9383 case ICmpInst::ICMP_SGE:
9384 case ICmpInst::ICMP_UGE:
9385 // Since the loop is finite, an invariant RHS cannot include the boundary
9386 // value, otherwise it would loop forever.
9387 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9388 !isLoopInvariant(RHS, L))
9389 break;
9391 [[fallthrough]];
9392 case ICmpInst::ICMP_SGT:
9393 case ICmpInst::ICMP_UGT: { // while (X > Y)
9394 bool IsSigned = ICmpInst::isSigned(Pred);
9395 ExitLimit EL = howManyGreaterThans(LHS, RHS, L, IsSigned, ControlsOnlyExit,
9396 AllowPredicates);
9397 if (EL.hasAnyInfo())
9398 return EL;
9399 break;
9400 }
9401 default:
9402 break;
9403 }
9404
9405 return getCouldNotCompute();
9406}
9407
9408ScalarEvolution::ExitLimit
9409ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L,
9410 SwitchInst *Switch,
9411 BasicBlock *ExitingBlock,
9412 bool ControlsOnlyExit) {
9413 assert(!L->contains(ExitingBlock) && "Not an exiting block!");
9414
9415 // Give up if the exit is the default dest of a switch.
9416 if (Switch->getDefaultDest() == ExitingBlock)
9417 return getCouldNotCompute();
9418
9419 assert(L->contains(Switch->getDefaultDest()) &&
9420 "Default case must not exit the loop!");
9421 const SCEV *LHS = getSCEVAtScope(Switch->getCondition(), L);
9422 const SCEV *RHS = getConstant(Switch->findCaseDest(ExitingBlock));
9423
9424 // while (X != Y) --> while (X-Y != 0)
9425 ExitLimit EL = howFarToZero(getMinusSCEV(LHS, RHS), L, ControlsOnlyExit);
9426 if (EL.hasAnyInfo())
9427 return EL;
9428
9429 return getCouldNotCompute();
9430}
9431
9432static ConstantInt *
9434 ScalarEvolution &SE) {
9435 const SCEV *InVal = SE.getConstant(C);
9436 const SCEV *Val = AddRec->evaluateAtIteration(InVal, SE);
9438 "Evaluation of SCEV at constant didn't fold correctly?");
9439 return cast<SCEVConstant>(Val)->getValue();
9440}
9441
9442ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit(
9443 Value *LHS, Value *RHSV, const Loop *L, ICmpInst::Predicate Pred) {
9444 ConstantInt *RHS = dyn_cast<ConstantInt>(RHSV);
9445 if (!RHS)
9446 return getCouldNotCompute();
9447
9448 const BasicBlock *Latch = L->getLoopLatch();
9449 if (!Latch)
9450 return getCouldNotCompute();
9451
9452 const BasicBlock *Predecessor = L->getLoopPredecessor();
9453 if (!Predecessor)
9454 return getCouldNotCompute();
9455
9456 // Return true if V is of the form "LHS `shift_op` <positive constant>".
9457 // Return LHS in OutLHS and shift_opt in OutOpCode.
9458 auto MatchPositiveShift =
9459 [](Value *V, Value *&OutLHS, Instruction::BinaryOps &OutOpCode) {
9460
9461 using namespace PatternMatch;
9462
9463 ConstantInt *ShiftAmt;
9464 if (match(V, m_LShr(m_Value(OutLHS), m_ConstantInt(ShiftAmt))))
9465 OutOpCode = Instruction::LShr;
9466 else if (match(V, m_AShr(m_Value(OutLHS), m_ConstantInt(ShiftAmt))))
9467 OutOpCode = Instruction::AShr;
9468 else if (match(V, m_Shl(m_Value(OutLHS), m_ConstantInt(ShiftAmt))))
9469 OutOpCode = Instruction::Shl;
9470 else
9471 return false;
9472
9473 return ShiftAmt->getValue().isStrictlyPositive();
9474 };
9475
9476 // Recognize a "shift recurrence" either of the form %iv or of %iv.shifted in
9477 //
9478 // loop:
9479 // %iv = phi i32 [ %iv.shifted, %loop ], [ %val, %preheader ]
9480 // %iv.shifted = lshr i32 %iv, <positive constant>
9481 //
9482 // Return true on a successful match. Return the corresponding PHI node (%iv
9483 // above) in PNOut and the opcode of the shift operation in OpCodeOut.
9484 auto MatchShiftRecurrence =
9485 [&](Value *V, PHINode *&PNOut, Instruction::BinaryOps &OpCodeOut) {
9486 std::optional<Instruction::BinaryOps> PostShiftOpCode;
9487
9488 {
9490 Value *V;
9491
9492 // If we encounter a shift instruction, "peel off" the shift operation,
9493 // and remember that we did so. Later when we inspect %iv's backedge
9494 // value, we will make sure that the backedge value uses the same
9495 // operation.
9496 //
9497 // Note: the peeled shift operation does not have to be the same
9498 // instruction as the one feeding into the PHI's backedge value. We only
9499 // really care about it being the same *kind* of shift instruction --
9500 // that's all that is required for our later inferences to hold.
9501 if (MatchPositiveShift(LHS, V, OpC)) {
9502 PostShiftOpCode = OpC;
9503 LHS = V;
9504 }
9505 }
9506
9507 PNOut = dyn_cast<PHINode>(LHS);
9508 if (!PNOut || PNOut->getParent() != L->getHeader())
9509 return false;
9510
9511 Value *BEValue = PNOut->getIncomingValueForBlock(Latch);
9512 Value *OpLHS;
9513
9514 return
9515 // The backedge value for the PHI node must be a shift by a positive
9516 // amount
9517 MatchPositiveShift(BEValue, OpLHS, OpCodeOut) &&
9518
9519 // of the PHI node itself
9520 OpLHS == PNOut &&
9521
9522 // and the kind of shift should be match the kind of shift we peeled
9523 // off, if any.
9524 (!PostShiftOpCode || *PostShiftOpCode == OpCodeOut);
9525 };
9526
9527 PHINode *PN;
9529 if (!MatchShiftRecurrence(LHS, PN, OpCode))
9530 return getCouldNotCompute();
9531
9532 const DataLayout &DL = getDataLayout();
9533
9534 // The key rationale for this optimization is that for some kinds of shift
9535 // recurrences, the value of the recurrence "stabilizes" to either 0 or -1
9536 // within a finite number of iterations. If the condition guarding the
9537 // backedge (in the sense that the backedge is taken if the condition is true)
9538 // is false for the value the shift recurrence stabilizes to, then we know
9539 // that the backedge is taken only a finite number of times.
9540
9541 ConstantInt *StableValue = nullptr;
9542 switch (OpCode) {
9543 default:
9544 llvm_unreachable("Impossible case!");
9545
9546 case Instruction::AShr: {
9547 // {K,ashr,<positive-constant>} stabilizes to signum(K) in at most
9548 // bitwidth(K) iterations.
9549 Value *FirstValue = PN->getIncomingValueForBlock(Predecessor);
9550 KnownBits Known = computeKnownBits(FirstValue, DL, &AC,
9551 Predecessor->getTerminator(), &DT);
9552 auto *Ty = cast<IntegerType>(RHS->getType());
9553 if (Known.isNonNegative())
9554 StableValue = ConstantInt::get(Ty, 0);
9555 else if (Known.isNegative())
9556 StableValue = ConstantInt::get(Ty, -1, true);
9557 else
9558 return getCouldNotCompute();
9559
9560 break;
9561 }
9562 case Instruction::LShr:
9563 case Instruction::Shl:
9564 // Both {K,lshr,<positive-constant>} and {K,shl,<positive-constant>}
9565 // stabilize to 0 in at most bitwidth(K) iterations.
9566 StableValue = ConstantInt::get(cast<IntegerType>(RHS->getType()), 0);
9567 break;
9568 }
9569
9570 auto *Result =
9571 ConstantFoldCompareInstOperands(Pred, StableValue, RHS, DL, &TLI);
9572 assert(Result->getType()->isIntegerTy(1) &&
9573 "Otherwise cannot be an operand to a branch instruction");
9574
9575 if (Result->isZeroValue()) {
9576 unsigned BitWidth = getTypeSizeInBits(RHS->getType());
9577 const SCEV *UpperBound =
9579 return ExitLimit(getCouldNotCompute(), UpperBound, UpperBound, false);
9580 }
9581
9582 return getCouldNotCompute();
9583}
9584
9585/// Return true if we can constant fold an instruction of the specified type,
9586/// assuming that all operands were constants.
9587static bool CanConstantFold(const Instruction *I) {
9591 return true;
9592
9593 if (const CallInst *CI = dyn_cast<CallInst>(I))
9594 if (const Function *F = CI->getCalledFunction())
9595 return canConstantFoldCallTo(CI, F);
9596 return false;
9597}
9598
9599/// Determine whether this instruction can constant evolve within this loop
9600/// assuming its operands can all constant evolve.
9601static bool canConstantEvolve(Instruction *I, const Loop *L) {
9602 // An instruction outside of the loop can't be derived from a loop PHI.
9603 if (!L->contains(I)) return false;
9604
9605 if (isa<PHINode>(I)) {
9606 // We don't currently keep track of the control flow needed to evaluate
9607 // PHIs, so we cannot handle PHIs inside of loops.
9608 return L->getHeader() == I->getParent();
9609 }
9610
9611 // If we won't be able to constant fold this expression even if the operands
9612 // are constants, bail early.
9613 return CanConstantFold(I);
9614}
9615
9616/// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
9617/// recursing through each instruction operand until reaching a loop header phi.
9618static PHINode *
9621 unsigned Depth) {
9623 return nullptr;
9624
9625 // Otherwise, we can evaluate this instruction if all of its operands are
9626 // constant or derived from a PHI node themselves.
9627 PHINode *PHI = nullptr;
9628 for (Value *Op : UseInst->operands()) {
9629 if (isa<Constant>(Op)) continue;
9630
9632 if (!OpInst || !canConstantEvolve(OpInst, L)) return nullptr;
9633
9634 PHINode *P = dyn_cast<PHINode>(OpInst);
9635 if (!P)
9636 // If this operand is already visited, reuse the prior result.
9637 // We may have P != PHI if this is the deepest point at which the
9638 // inconsistent paths meet.
9639 P = PHIMap.lookup(OpInst);
9640 if (!P) {
9641 // Recurse and memoize the results, whether a phi is found or not.
9642 // This recursive call invalidates pointers into PHIMap.
9643 P = getConstantEvolvingPHIOperands(OpInst, L, PHIMap, Depth + 1);
9644 PHIMap[OpInst] = P;
9645 }
9646 if (!P)
9647 return nullptr; // Not evolving from PHI
9648 if (PHI && PHI != P)
9649 return nullptr; // Evolving from multiple different PHIs.
9650 PHI = P;
9651 }
9652 // This is a expression evolving from a constant PHI!
9653 return PHI;
9654}
9655
9656/// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
9657/// in the loop that V is derived from. We allow arbitrary operations along the
9658/// way, but the operands of an operation must either be constants or a value
9659/// derived from a constant PHI. If this expression does not fit with these
9660/// constraints, return null.
9663 if (!I || !canConstantEvolve(I, L)) return nullptr;
9664
9665 if (PHINode *PN = dyn_cast<PHINode>(I))
9666 return PN;
9667
9668 // Record non-constant instructions contained by the loop.
9670 return getConstantEvolvingPHIOperands(I, L, PHIMap, 0);
9671}
9672
9673/// EvaluateExpression - Given an expression that passes the
9674/// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
9675/// in the loop has the value PHIVal. If we can't fold this expression for some
9676/// reason, return null.
9679 const DataLayout &DL,
9680 const TargetLibraryInfo *TLI) {
9681 // Convenient constant check, but redundant for recursive calls.
9682 if (Constant *C = dyn_cast<Constant>(V)) return C;
9684 if (!I) return nullptr;
9685
9686 if (Constant *C = Vals.lookup(I)) return C;
9687
9688 // An instruction inside the loop depends on a value outside the loop that we
9689 // weren't given a mapping for, or a value such as a call inside the loop.
9690 if (!canConstantEvolve(I, L)) return nullptr;
9691
9692 // An unmapped PHI can be due to a branch or another loop inside this loop,
9693 // or due to this not being the initial iteration through a loop where we
9694 // couldn't compute the evolution of this particular PHI last time.
9695 if (isa<PHINode>(I)) return nullptr;
9696
9697 std::vector<Constant*> Operands(I->getNumOperands());
9698
9699 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
9700 Instruction *Operand = dyn_cast<Instruction>(I->getOperand(i));
9701 if (!Operand) {
9702 Operands[i] = dyn_cast<Constant>(I->getOperand(i));
9703 if (!Operands[i]) return nullptr;
9704 continue;
9705 }
9706 Constant *C = EvaluateExpression(Operand, L, Vals, DL, TLI);
9707 Vals[Operand] = C;
9708 if (!C) return nullptr;
9709 Operands[i] = C;
9710 }
9711
9712 return ConstantFoldInstOperands(I, Operands, DL, TLI,
9713 /*AllowNonDeterministic=*/false);
9714}
9715
9716
9717// If every incoming value to PN except the one for BB is a specific Constant,
9718// return that, else return nullptr.
9720 Constant *IncomingVal = nullptr;
9721
9722 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
9723 if (PN->getIncomingBlock(i) == BB)
9724 continue;
9725
9726 auto *CurrentVal = dyn_cast<Constant>(PN->getIncomingValue(i));
9727 if (!CurrentVal)
9728 return nullptr;
9729
9730 if (IncomingVal != CurrentVal) {
9731 if (IncomingVal)
9732 return nullptr;
9733 IncomingVal = CurrentVal;
9734 }
9735 }
9736
9737 return IncomingVal;
9738}
9739
9740/// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
9741/// in the header of its containing loop, we know the loop executes a
9742/// constant number of times, and the PHI node is just a recurrence
9743/// involving constants, fold it.
9744Constant *
9745ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
9746 const APInt &BEs,
9747 const Loop *L) {
9748 auto [I, Inserted] = ConstantEvolutionLoopExitValue.try_emplace(PN);
9749 if (!Inserted)
9750 return I->second;
9751
9753 return nullptr; // Not going to evaluate it.
9754
9755 Constant *&RetVal = I->second;
9756
9757 DenseMap<Instruction *, Constant *> CurrentIterVals;
9758 BasicBlock *Header = L->getHeader();
9759 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
9760
9761 BasicBlock *Latch = L->getLoopLatch();
9762 if (!Latch)
9763 return nullptr;
9764
9765 for (PHINode &PHI : Header->phis()) {
9766 if (auto *StartCST = getOtherIncomingValue(&PHI, Latch))
9767 CurrentIterVals[&PHI] = StartCST;
9768 }
9769 if (!CurrentIterVals.count(PN))
9770 return RetVal = nullptr;
9771
9772 Value *BEValue = PN->getIncomingValueForBlock(Latch);
9773
9774 // Execute the loop symbolically to determine the exit value.
9775 assert(BEs.getActiveBits() < CHAR_BIT * sizeof(unsigned) &&
9776 "BEs is <= MaxBruteForceIterations which is an 'unsigned'!");
9777
9778 unsigned NumIterations = BEs.getZExtValue(); // must be in range
9779 unsigned IterationNum = 0;
9780 const DataLayout &DL = getDataLayout();
9781 for (; ; ++IterationNum) {
9782 if (IterationNum == NumIterations)
9783 return RetVal = CurrentIterVals[PN]; // Got exit value!
9784
9785 // Compute the value of the PHIs for the next iteration.
9786 // EvaluateExpression adds non-phi values to the CurrentIterVals map.
9787 DenseMap<Instruction *, Constant *> NextIterVals;
9788 Constant *NextPHI =
9789 EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI);
9790 if (!NextPHI)
9791 return nullptr; // Couldn't evaluate!
9792 NextIterVals[PN] = NextPHI;
9793
9794 bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
9795
9796 // Also evaluate the other PHI nodes. However, we don't get to stop if we
9797 // cease to be able to evaluate one of them or if they stop evolving,
9798 // because that doesn't necessarily prevent us from computing PN.
9800 for (const auto &I : CurrentIterVals) {
9801 PHINode *PHI = dyn_cast<PHINode>(I.first);
9802 if (!PHI || PHI == PN || PHI->getParent() != Header) continue;
9803 PHIsToCompute.emplace_back(PHI, I.second);
9804 }
9805 // We use two distinct loops because EvaluateExpression may invalidate any
9806 // iterators into CurrentIterVals.
9807 for (const auto &I : PHIsToCompute) {
9808 PHINode *PHI = I.first;
9809 Constant *&NextPHI = NextIterVals[PHI];
9810 if (!NextPHI) { // Not already computed.
9811 Value *BEValue = PHI->getIncomingValueForBlock(Latch);
9812 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI);
9813 }
9814 if (NextPHI != I.second)
9815 StoppedEvolving = false;
9816 }
9817
9818 // If all entries in CurrentIterVals == NextIterVals then we can stop
9819 // iterating, the loop can't continue to change.
9820 if (StoppedEvolving)
9821 return RetVal = CurrentIterVals[PN];
9822
9823 CurrentIterVals.swap(NextIterVals);
9824 }
9825}
9826
9827const SCEV *ScalarEvolution::computeExitCountExhaustively(const Loop *L,
9828 Value *Cond,
9829 bool ExitWhen) {
9830 PHINode *PN = getConstantEvolvingPHI(Cond, L);
9831 if (!PN) return getCouldNotCompute();
9832
9833 // If the loop is canonicalized, the PHI will have exactly two entries.
9834 // That's the only form we support here.
9835 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
9836
9837 DenseMap<Instruction *, Constant *> CurrentIterVals;
9838 BasicBlock *Header = L->getHeader();
9839 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
9840
9841 BasicBlock *Latch = L->getLoopLatch();
9842 assert(Latch && "Should follow from NumIncomingValues == 2!");
9843
9844 for (PHINode &PHI : Header->phis()) {
9845 if (auto *StartCST = getOtherIncomingValue(&PHI, Latch))
9846 CurrentIterVals[&PHI] = StartCST;
9847 }
9848 if (!CurrentIterVals.count(PN))
9849 return getCouldNotCompute();
9850
9851 // Okay, we find a PHI node that defines the trip count of this loop. Execute
9852 // the loop symbolically to determine when the condition gets a value of
9853 // "ExitWhen".
9854 unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis.
9855 const DataLayout &DL = getDataLayout();
9856 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
9857 auto *CondVal = dyn_cast_or_null<ConstantInt>(
9858 EvaluateExpression(Cond, L, CurrentIterVals, DL, &TLI));
9859
9860 // Couldn't symbolically evaluate.
9861 if (!CondVal) return getCouldNotCompute();
9862
9863 if (CondVal->getValue() == uint64_t(ExitWhen)) {
9864 ++NumBruteForceTripCountsComputed;
9865 return getConstant(Type::getInt32Ty(getContext()), IterationNum);
9866 }
9867
9868 // Update all the PHI nodes for the next iteration.
9869 DenseMap<Instruction *, Constant *> NextIterVals;
9870
9871 // Create a list of which PHIs we need to compute. We want to do this before
9872 // calling EvaluateExpression on them because that may invalidate iterators
9873 // into CurrentIterVals.
9874 SmallVector<PHINode *, 8> PHIsToCompute;
9875 for (const auto &I : CurrentIterVals) {
9876 PHINode *PHI = dyn_cast<PHINode>(I.first);
9877 if (!PHI || PHI->getParent() != Header) continue;
9878 PHIsToCompute.push_back(PHI);
9879 }
9880 for (PHINode *PHI : PHIsToCompute) {
9881 Constant *&NextPHI = NextIterVals[PHI];
9882 if (NextPHI) continue; // Already computed!
9883
9884 Value *BEValue = PHI->getIncomingValueForBlock(Latch);
9885 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI);
9886 }
9887 CurrentIterVals.swap(NextIterVals);
9888 }
9889
9890 // Too many iterations were needed to evaluate.
9891 return getCouldNotCompute();
9892}
9893
9894const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
9896 ValuesAtScopes[V];
9897 // Check to see if we've folded this expression at this loop before.
9898 for (auto &LS : Values)
9899 if (LS.first == L)
9900 return LS.second ? LS.second : V;
9901
9902 Values.emplace_back(L, nullptr);
9903
9904 // Otherwise compute it.
9905 const SCEV *C = computeSCEVAtScope(V, L);
9906 for (auto &LS : reverse(ValuesAtScopes[V]))
9907 if (LS.first == L) {
9908 LS.second = C;
9909 if (!isa<SCEVConstant>(C))
9910 ValuesAtScopesUsers[C].push_back({L, V});
9911 break;
9912 }
9913 return C;
9914}
9915
9916/// This builds up a Constant using the ConstantExpr interface. That way, we
9917/// will return Constants for objects which aren't represented by a
9918/// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
9919/// Returns NULL if the SCEV isn't representable as a Constant.
9921 switch (V->getSCEVType()) {
9922 case scCouldNotCompute:
9923 case scAddRecExpr:
9924 case scVScale:
9925 return nullptr;
9926 case scConstant:
9927 return cast<SCEVConstant>(V)->getValue();
9928 case scUnknown:
9929 return dyn_cast<Constant>(cast<SCEVUnknown>(V)->getValue());
9930 case scPtrToInt: {
9932 if (Constant *CastOp = BuildConstantFromSCEV(P2I->getOperand()))
9933 return ConstantExpr::getPtrToInt(CastOp, P2I->getType());
9934
9935 return nullptr;
9936 }
9937 case scTruncate: {
9939 if (Constant *CastOp = BuildConstantFromSCEV(ST->getOperand()))
9940 return ConstantExpr::getTrunc(CastOp, ST->getType());
9941 return nullptr;
9942 }
9943 case scAddExpr: {
9944 const SCEVAddExpr *SA = cast<SCEVAddExpr>(V);
9945 Constant *C = nullptr;
9946 for (const SCEV *Op : SA->operands()) {
9948 if (!OpC)
9949 return nullptr;
9950 if (!C) {
9951 C = OpC;
9952 continue;
9953 }
9954 assert(!C->getType()->isPointerTy() &&
9955 "Can only have one pointer, and it must be last");
9956 if (OpC->getType()->isPointerTy()) {
9957 // The offsets have been converted to bytes. We can add bytes using
9958 // an i8 GEP.
9960 OpC, C);
9961 } else {
9962 C = ConstantExpr::getAdd(C, OpC);
9963 }
9964 }
9965 return C;
9966 }
9967 case scMulExpr:
9968 case scSignExtend:
9969 case scZeroExtend:
9970 case scUDivExpr:
9971 case scSMaxExpr:
9972 case scUMaxExpr:
9973 case scSMinExpr:
9974 case scUMinExpr:
9976 return nullptr;
9977 }
9978 llvm_unreachable("Unknown SCEV kind!");
9979}
9980
9981const SCEV *
9982ScalarEvolution::getWithOperands(const SCEV *S,
9983 SmallVectorImpl<const SCEV *> &NewOps) {
9984 switch (S->getSCEVType()) {
9985 case scTruncate:
9986 case scZeroExtend:
9987 case scSignExtend:
9988 case scPtrToInt:
9989 return getCastExpr(S->getSCEVType(), NewOps[0], S->getType());
9990 case scAddRecExpr: {
9991 auto *AddRec = cast<SCEVAddRecExpr>(S);
9992 return getAddRecExpr(NewOps, AddRec->getLoop(), AddRec->getNoWrapFlags());
9993 }
9994 case scAddExpr:
9995 return getAddExpr(NewOps, cast<SCEVAddExpr>(S)->getNoWrapFlags());
9996 case scMulExpr:
9997 return getMulExpr(NewOps, cast<SCEVMulExpr>(S)->getNoWrapFlags());
9998 case scUDivExpr:
9999 return getUDivExpr(NewOps[0], NewOps[1]);
10000 case scUMaxExpr:
10001 case scSMaxExpr:
10002 case scUMinExpr:
10003 case scSMinExpr:
10004 return getMinMaxExpr(S->getSCEVType(), NewOps);
10006 return getSequentialMinMaxExpr(S->getSCEVType(), NewOps);
10007 case scConstant:
10008 case scVScale:
10009 case scUnknown:
10010 return S;
10011 case scCouldNotCompute:
10012 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10013 }
10014 llvm_unreachable("Unknown SCEV kind!");
10015}
10016
10017const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
10018 switch (V->getSCEVType()) {
10019 case scConstant:
10020 case scVScale:
10021 return V;
10022 case scAddRecExpr: {
10023 // If this is a loop recurrence for a loop that does not contain L, then we
10024 // are dealing with the final value computed by the loop.
10025 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(V);
10026 // First, attempt to evaluate each operand.
10027 // Avoid performing the look-up in the common case where the specified
10028 // expression has no loop-variant portions.
10029 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
10030 const SCEV *OpAtScope = getSCEVAtScope(AddRec->getOperand(i), L);
10031 if (OpAtScope == AddRec->getOperand(i))
10032 continue;
10033
10034 // Okay, at least one of these operands is loop variant but might be
10035 // foldable. Build a new instance of the folded commutative expression.
10037 NewOps.reserve(AddRec->getNumOperands());
10038 append_range(NewOps, AddRec->operands().take_front(i));
10039 NewOps.push_back(OpAtScope);
10040 for (++i; i != e; ++i)
10041 NewOps.push_back(getSCEVAtScope(AddRec->getOperand(i), L));
10042
10043 const SCEV *FoldedRec = getAddRecExpr(
10044 NewOps, AddRec->getLoop(), AddRec->getNoWrapFlags(SCEV::FlagNW));
10045 AddRec = dyn_cast<SCEVAddRecExpr>(FoldedRec);
10046 // The addrec may be folded to a nonrecurrence, for example, if the
10047 // induction variable is multiplied by zero after constant folding. Go
10048 // ahead and return the folded value.
10049 if (!AddRec)
10050 return FoldedRec;
10051 break;
10052 }
10053
10054 // If the scope is outside the addrec's loop, evaluate it by using the
10055 // loop exit value of the addrec.
10056 if (!AddRec->getLoop()->contains(L)) {
10057 // To evaluate this recurrence, we need to know how many times the AddRec
10058 // loop iterates. Compute this now.
10059 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(AddRec->getLoop());
10060 if (BackedgeTakenCount == getCouldNotCompute())
10061 return AddRec;
10062
10063 // Then, evaluate the AddRec.
10064 return AddRec->evaluateAtIteration(BackedgeTakenCount, *this);
10065 }
10066
10067 return AddRec;
10068 }
10069 case scTruncate:
10070 case scZeroExtend:
10071 case scSignExtend:
10072 case scPtrToInt:
10073 case scAddExpr:
10074 case scMulExpr:
10075 case scUDivExpr:
10076 case scUMaxExpr:
10077 case scSMaxExpr:
10078 case scUMinExpr:
10079 case scSMinExpr:
10080 case scSequentialUMinExpr: {
10081 ArrayRef<const SCEV *> Ops = V->operands();
10082 // Avoid performing the look-up in the common case where the specified
10083 // expression has no loop-variant portions.
10084 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
10085 const SCEV *OpAtScope = getSCEVAtScope(Ops[i], L);
10086 if (OpAtScope != Ops[i]) {
10087 // Okay, at least one of these operands is loop variant but might be
10088 // foldable. Build a new instance of the folded commutative expression.
10090 NewOps.reserve(Ops.size());
10091 append_range(NewOps, Ops.take_front(i));
10092 NewOps.push_back(OpAtScope);
10093
10094 for (++i; i != e; ++i) {
10095 OpAtScope = getSCEVAtScope(Ops[i], L);
10096 NewOps.push_back(OpAtScope);
10097 }
10098
10099 return getWithOperands(V, NewOps);
10100 }
10101 }
10102 // If we got here, all operands are loop invariant.
10103 return V;
10104 }
10105 case scUnknown: {
10106 // If this instruction is evolved from a constant-evolving PHI, compute the
10107 // exit value from the loop without using SCEVs.
10108 const SCEVUnknown *SU = cast<SCEVUnknown>(V);
10110 if (!I)
10111 return V; // This is some other type of SCEVUnknown, just return it.
10112
10113 if (PHINode *PN = dyn_cast<PHINode>(I)) {
10114 const Loop *CurrLoop = this->LI[I->getParent()];
10115 // Looking for loop exit value.
10116 if (CurrLoop && CurrLoop->getParentLoop() == L &&
10117 PN->getParent() == CurrLoop->getHeader()) {
10118 // Okay, there is no closed form solution for the PHI node. Check
10119 // to see if the loop that contains it has a known backedge-taken
10120 // count. If so, we may be able to force computation of the exit
10121 // value.
10122 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(CurrLoop);
10123 // This trivial case can show up in some degenerate cases where
10124 // the incoming IR has not yet been fully simplified.
10125 if (BackedgeTakenCount->isZero()) {
10126 Value *InitValue = nullptr;
10127 bool MultipleInitValues = false;
10128 for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) {
10129 if (!CurrLoop->contains(PN->getIncomingBlock(i))) {
10130 if (!InitValue)
10131 InitValue = PN->getIncomingValue(i);
10132 else if (InitValue != PN->getIncomingValue(i)) {
10133 MultipleInitValues = true;
10134 break;
10135 }
10136 }
10137 }
10138 if (!MultipleInitValues && InitValue)
10139 return getSCEV(InitValue);
10140 }
10141 // Do we have a loop invariant value flowing around the backedge
10142 // for a loop which must execute the backedge?
10143 if (!isa<SCEVCouldNotCompute>(BackedgeTakenCount) &&
10144 isKnownNonZero(BackedgeTakenCount) &&
10145 PN->getNumIncomingValues() == 2) {
10146
10147 unsigned InLoopPred =
10148 CurrLoop->contains(PN->getIncomingBlock(0)) ? 0 : 1;
10149 Value *BackedgeVal = PN->getIncomingValue(InLoopPred);
10150 if (CurrLoop->isLoopInvariant(BackedgeVal))
10151 return getSCEV(BackedgeVal);
10152 }
10153 if (auto *BTCC = dyn_cast<SCEVConstant>(BackedgeTakenCount)) {
10154 // Okay, we know how many times the containing loop executes. If
10155 // this is a constant evolving PHI node, get the final value at
10156 // the specified iteration number.
10157 Constant *RV =
10158 getConstantEvolutionLoopExitValue(PN, BTCC->getAPInt(), CurrLoop);
10159 if (RV)
10160 return getSCEV(RV);
10161 }
10162 }
10163 }
10164
10165 // Okay, this is an expression that we cannot symbolically evaluate
10166 // into a SCEV. Check to see if it's possible to symbolically evaluate
10167 // the arguments into constants, and if so, try to constant propagate the
10168 // result. This is particularly useful for computing loop exit values.
10169 if (!CanConstantFold(I))
10170 return V; // This is some other type of SCEVUnknown, just return it.
10171
10173 Operands.reserve(I->getNumOperands());
10174 bool MadeImprovement = false;
10175 for (Value *Op : I->operands()) {
10176 if (Constant *C = dyn_cast<Constant>(Op)) {
10177 Operands.push_back(C);
10178 continue;
10179 }
10180
10181 // If any of the operands is non-constant and if they are
10182 // non-integer and non-pointer, don't even try to analyze them
10183 // with scev techniques.
10184 if (!isSCEVable(Op->getType()))
10185 return V;
10186
10187 const SCEV *OrigV = getSCEV(Op);
10188 const SCEV *OpV = getSCEVAtScope(OrigV, L);
10189 MadeImprovement |= OrigV != OpV;
10190
10192 if (!C)
10193 return V;
10194 assert(C->getType() == Op->getType() && "Type mismatch");
10195 Operands.push_back(C);
10196 }
10197
10198 // Check to see if getSCEVAtScope actually made an improvement.
10199 if (!MadeImprovement)
10200 return V; // This is some other type of SCEVUnknown, just return it.
10201
10202 Constant *C = nullptr;
10203 const DataLayout &DL = getDataLayout();
10204 C = ConstantFoldInstOperands(I, Operands, DL, &TLI,
10205 /*AllowNonDeterministic=*/false);
10206 if (!C)
10207 return V;
10208 return getSCEV(C);
10209 }
10210 case scCouldNotCompute:
10211 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10212 }
10213 llvm_unreachable("Unknown SCEV type!");
10214}
10215
10217 return getSCEVAtScope(getSCEV(V), L);
10218}
10219
10220const SCEV *ScalarEvolution::stripInjectiveFunctions(const SCEV *S) const {
10222 return stripInjectiveFunctions(ZExt->getOperand());
10224 return stripInjectiveFunctions(SExt->getOperand());
10225 return S;
10226}
10227
10228/// Finds the minimum unsigned root of the following equation:
10229///
10230/// A * X = B (mod N)
10231///
10232/// where N = 2^BW and BW is the common bit width of A and B. The signedness of
10233/// A and B isn't important.
10234///
10235/// If the equation does not have a solution, SCEVCouldNotCompute is returned.
10236static const SCEV *
10239 ScalarEvolution &SE, const Loop *L) {
10240 uint32_t BW = A.getBitWidth();
10241 assert(BW == SE.getTypeSizeInBits(B->getType()));
10242 assert(A != 0 && "A must be non-zero.");
10243
10244 // 1. D = gcd(A, N)
10245 //
10246 // The gcd of A and N may have only one prime factor: 2. The number of
10247 // trailing zeros in A is its multiplicity
10248 uint32_t Mult2 = A.countr_zero();
10249 // D = 2^Mult2
10250
10251 // 2. Check if B is divisible by D.
10252 //
10253 // B is divisible by D if and only if the multiplicity of prime factor 2 for B
10254 // is not less than multiplicity of this prime factor for D.
10255 unsigned MinTZ = SE.getMinTrailingZeros(B);
10256 // Try again with the terminator of the loop predecessor for context-specific
10257 // result, if MinTZ s too small.
10258 if (MinTZ < Mult2 && L->getLoopPredecessor())
10259 MinTZ = SE.getMinTrailingZeros(B, L->getLoopPredecessor()->getTerminator());
10260 if (MinTZ < Mult2) {
10261 // Check if we can prove there's no remainder using URem.
10262 const SCEV *URem =
10263 SE.getURemExpr(B, SE.getConstant(APInt::getOneBitSet(BW, Mult2)));
10264 const SCEV *Zero = SE.getZero(B->getType());
10265 if (!SE.isKnownPredicate(CmpInst::ICMP_EQ, URem, Zero)) {
10266 // Try to add a predicate ensuring B is a multiple of 1 << Mult2.
10267 if (!Predicates)
10268 return SE.getCouldNotCompute();
10269
10270 // Avoid adding a predicate that is known to be false.
10271 if (SE.isKnownPredicate(CmpInst::ICMP_NE, URem, Zero))
10272 return SE.getCouldNotCompute();
10273 Predicates->push_back(SE.getEqualPredicate(URem, Zero));
10274 }
10275 }
10276
10277 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
10278 // modulo (N / D).
10279 //
10280 // If D == 1, (N / D) == N == 2^BW, so we need one extra bit to represent
10281 // (N / D) in general. The inverse itself always fits into BW bits, though,
10282 // so we immediately truncate it.
10283 APInt AD = A.lshr(Mult2).trunc(BW - Mult2); // AD = A / D
10284 APInt I = AD.multiplicativeInverse().zext(BW);
10285
10286 // 4. Compute the minimum unsigned root of the equation:
10287 // I * (B / D) mod (N / D)
10288 // To simplify the computation, we factor out the divide by D:
10289 // (I * B mod N) / D
10290 const SCEV *D = SE.getConstant(APInt::getOneBitSet(BW, Mult2));
10291 return SE.getUDivExactExpr(SE.getMulExpr(B, SE.getConstant(I)), D);
10292}
10293
10294/// For a given quadratic addrec, generate coefficients of the corresponding
10295/// quadratic equation, multiplied by a common value to ensure that they are
10296/// integers.
10297/// The returned value is a tuple { A, B, C, M, BitWidth }, where
10298/// Ax^2 + Bx + C is the quadratic function, M is the value that A, B and C
10299/// were multiplied by, and BitWidth is the bit width of the original addrec
10300/// coefficients.
10301/// This function returns std::nullopt if the addrec coefficients are not
10302/// compile- time constants.
10303static std::optional<std::tuple<APInt, APInt, APInt, APInt, unsigned>>
10305 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!");
10306 const SCEVConstant *LC = dyn_cast<SCEVConstant>(AddRec->getOperand(0));
10307 const SCEVConstant *MC = dyn_cast<SCEVConstant>(AddRec->getOperand(1));
10308 const SCEVConstant *NC = dyn_cast<SCEVConstant>(AddRec->getOperand(2));
10309 LLVM_DEBUG(dbgs() << __func__ << ": analyzing quadratic addrec: "
10310 << *AddRec << '\n');
10311
10312 // We currently can only solve this if the coefficients are constants.
10313 if (!LC || !MC || !NC) {
10314 LLVM_DEBUG(dbgs() << __func__ << ": coefficients are not constant\n");
10315 return std::nullopt;
10316 }
10317
10318 APInt L = LC->getAPInt();
10319 APInt M = MC->getAPInt();
10320 APInt N = NC->getAPInt();
10321 assert(!N.isZero() && "This is not a quadratic addrec");
10322
10323 unsigned BitWidth = LC->getAPInt().getBitWidth();
10324 unsigned NewWidth = BitWidth + 1;
10325 LLVM_DEBUG(dbgs() << __func__ << ": addrec coeff bw: "
10326 << BitWidth << '\n');
10327 // The sign-extension (as opposed to a zero-extension) here matches the
10328 // extension used in SolveQuadraticEquationWrap (with the same motivation).
10329 N = N.sext(NewWidth);
10330 M = M.sext(NewWidth);
10331 L = L.sext(NewWidth);
10332
10333 // The increments are M, M+N, M+2N, ..., so the accumulated values are
10334 // L+M, (L+M)+(M+N), (L+M)+(M+N)+(M+2N), ..., that is,
10335 // L+M, L+2M+N, L+3M+3N, ...
10336 // After n iterations the accumulated value Acc is L + nM + n(n-1)/2 N.
10337 //
10338 // The equation Acc = 0 is then
10339 // L + nM + n(n-1)/2 N = 0, or 2L + 2M n + n(n-1) N = 0.
10340 // In a quadratic form it becomes:
10341 // N n^2 + (2M-N) n + 2L = 0.
10342
10343 APInt A = N;
10344 APInt B = 2 * M - A;
10345 APInt C = 2 * L;
10346 APInt T = APInt(NewWidth, 2);
10347 LLVM_DEBUG(dbgs() << __func__ << ": equation " << A << "x^2 + " << B
10348 << "x + " << C << ", coeff bw: " << NewWidth
10349 << ", multiplied by " << T << '\n');
10350 return std::make_tuple(A, B, C, T, BitWidth);
10351}
10352
10353/// Helper function to compare optional APInts:
10354/// (a) if X and Y both exist, return min(X, Y),
10355/// (b) if neither X nor Y exist, return std::nullopt,
10356/// (c) if exactly one of X and Y exists, return that value.
10357static std::optional<APInt> MinOptional(std::optional<APInt> X,
10358 std::optional<APInt> Y) {
10359 if (X && Y) {
10360 unsigned W = std::max(X->getBitWidth(), Y->getBitWidth());
10361 APInt XW = X->sext(W);
10362 APInt YW = Y->sext(W);
10363 return XW.slt(YW) ? *X : *Y;
10364 }
10365 if (!X && !Y)
10366 return std::nullopt;
10367 return X ? *X : *Y;
10368}
10369
10370/// Helper function to truncate an optional APInt to a given BitWidth.
10371/// When solving addrec-related equations, it is preferable to return a value
10372/// that has the same bit width as the original addrec's coefficients. If the
10373/// solution fits in the original bit width, truncate it (except for i1).
10374/// Returning a value of a different bit width may inhibit some optimizations.
10375///
10376/// In general, a solution to a quadratic equation generated from an addrec
10377/// may require BW+1 bits, where BW is the bit width of the addrec's
10378/// coefficients. The reason is that the coefficients of the quadratic
10379/// equation are BW+1 bits wide (to avoid truncation when converting from
10380/// the addrec to the equation).
10381static std::optional<APInt> TruncIfPossible(std::optional<APInt> X,
10382 unsigned BitWidth) {
10383 if (!X)
10384 return std::nullopt;
10385 unsigned W = X->getBitWidth();
10387 return X->trunc(BitWidth);
10388 return X;
10389}
10390
10391/// Let c(n) be the value of the quadratic chrec {L,+,M,+,N} after n
10392/// iterations. The values L, M, N are assumed to be signed, and they
10393/// should all have the same bit widths.
10394/// Find the least n >= 0 such that c(n) = 0 in the arithmetic modulo 2^BW,
10395/// where BW is the bit width of the addrec's coefficients.
10396/// If the calculated value is a BW-bit integer (for BW > 1), it will be
10397/// returned as such, otherwise the bit width of the returned value may
10398/// be greater than BW.
10399///
10400/// This function returns std::nullopt if
10401/// (a) the addrec coefficients are not constant, or
10402/// (b) SolveQuadraticEquationWrap was unable to find a solution. For cases
10403/// like x^2 = 5, no integer solutions exist, in other cases an integer
10404/// solution may exist, but SolveQuadraticEquationWrap may fail to find it.
10405static std::optional<APInt>
10407 APInt A, B, C, M;
10408 unsigned BitWidth;
10409 auto T = GetQuadraticEquation(AddRec);
10410 if (!T)
10411 return std::nullopt;
10412
10413 std::tie(A, B, C, M, BitWidth) = *T;
10414 LLVM_DEBUG(dbgs() << __func__ << ": solving for unsigned overflow\n");
10415 std::optional<APInt> X =
10417 if (!X)
10418 return std::nullopt;
10419
10420 ConstantInt *CX = ConstantInt::get(SE.getContext(), *X);
10421 ConstantInt *V = EvaluateConstantChrecAtConstant(AddRec, CX, SE);
10422 if (!V->isZero())
10423 return std::nullopt;
10424
10425 return TruncIfPossible(X, BitWidth);
10426}
10427
10428/// Let c(n) be the value of the quadratic chrec {0,+,M,+,N} after n
10429/// iterations. The values M, N are assumed to be signed, and they
10430/// should all have the same bit widths.
10431/// Find the least n such that c(n) does not belong to the given range,
10432/// while c(n-1) does.
10433///
10434/// This function returns std::nullopt if
10435/// (a) the addrec coefficients are not constant, or
10436/// (b) SolveQuadraticEquationWrap was unable to find a solution for the
10437/// bounds of the range.
10438static std::optional<APInt>
10440 const ConstantRange &Range, ScalarEvolution &SE) {
10441 assert(AddRec->getOperand(0)->isZero() &&
10442 "Starting value of addrec should be 0");
10443 LLVM_DEBUG(dbgs() << __func__ << ": solving boundary crossing for range "
10444 << Range << ", addrec " << *AddRec << '\n');
10445 // This case is handled in getNumIterationsInRange. Here we can assume that
10446 // we start in the range.
10447 assert(Range.contains(APInt(SE.getTypeSizeInBits(AddRec->getType()), 0)) &&
10448 "Addrec's initial value should be in range");
10449
10450 APInt A, B, C, M;
10451 unsigned BitWidth;
10452 auto T = GetQuadraticEquation(AddRec);
10453 if (!T)
10454 return std::nullopt;
10455
10456 // Be careful about the return value: there can be two reasons for not
10457 // returning an actual number. First, if no solutions to the equations
10458 // were found, and second, if the solutions don't leave the given range.
10459 // The first case means that the actual solution is "unknown", the second
10460 // means that it's known, but not valid. If the solution is unknown, we
10461 // cannot make any conclusions.
10462 // Return a pair: the optional solution and a flag indicating if the
10463 // solution was found.
10464 auto SolveForBoundary =
10465 [&](APInt Bound) -> std::pair<std::optional<APInt>, bool> {
10466 // Solve for signed overflow and unsigned overflow, pick the lower
10467 // solution.
10468 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: checking boundary "
10469 << Bound << " (before multiplying by " << M << ")\n");
10470 Bound *= M; // The quadratic equation multiplier.
10471
10472 std::optional<APInt> SO;
10473 if (BitWidth > 1) {
10474 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10475 "signed overflow\n");
10477 }
10478 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10479 "unsigned overflow\n");
10480 std::optional<APInt> UO =
10482
10483 auto LeavesRange = [&] (const APInt &X) {
10484 ConstantInt *C0 = ConstantInt::get(SE.getContext(), X);
10485 ConstantInt *V0 = EvaluateConstantChrecAtConstant(AddRec, C0, SE);
10486 if (Range.contains(V0->getValue()))
10487 return false;
10488 // X should be at least 1, so X-1 is non-negative.
10489 ConstantInt *C1 = ConstantInt::get(SE.getContext(), X-1);
10490 ConstantInt *V1 = EvaluateConstantChrecAtConstant(AddRec, C1, SE);
10491 if (Range.contains(V1->getValue()))
10492 return true;
10493 return false;
10494 };
10495
10496 // If SolveQuadraticEquationWrap returns std::nullopt, it means that there
10497 // can be a solution, but the function failed to find it. We cannot treat it
10498 // as "no solution".
10499 if (!SO || !UO)
10500 return {std::nullopt, false};
10501
10502 // Check the smaller value first to see if it leaves the range.
10503 // At this point, both SO and UO must have values.
10504 std::optional<APInt> Min = MinOptional(SO, UO);
10505 if (LeavesRange(*Min))
10506 return { Min, true };
10507 std::optional<APInt> Max = Min == SO ? UO : SO;
10508 if (LeavesRange(*Max))
10509 return { Max, true };
10510
10511 // Solutions were found, but were eliminated, hence the "true".
10512 return {std::nullopt, true};
10513 };
10514
10515 std::tie(A, B, C, M, BitWidth) = *T;
10516 // Lower bound is inclusive, subtract 1 to represent the exiting value.
10517 APInt Lower = Range.getLower().sext(A.getBitWidth()) - 1;
10518 APInt Upper = Range.getUpper().sext(A.getBitWidth());
10519 auto SL = SolveForBoundary(Lower);
10520 auto SU = SolveForBoundary(Upper);
10521 // If any of the solutions was unknown, no meaninigful conclusions can
10522 // be made.
10523 if (!SL.second || !SU.second)
10524 return std::nullopt;
10525
10526 // Claim: The correct solution is not some value between Min and Max.
10527 //
10528 // Justification: Assuming that Min and Max are different values, one of
10529 // them is when the first signed overflow happens, the other is when the
10530 // first unsigned overflow happens. Crossing the range boundary is only
10531 // possible via an overflow (treating 0 as a special case of it, modeling
10532 // an overflow as crossing k*2^W for some k).
10533 //
10534 // The interesting case here is when Min was eliminated as an invalid
10535 // solution, but Max was not. The argument is that if there was another
10536 // overflow between Min and Max, it would also have been eliminated if
10537 // it was considered.
10538 //
10539 // For a given boundary, it is possible to have two overflows of the same
10540 // type (signed/unsigned) without having the other type in between: this
10541 // can happen when the vertex of the parabola is between the iterations
10542 // corresponding to the overflows. This is only possible when the two
10543 // overflows cross k*2^W for the same k. In such case, if the second one
10544 // left the range (and was the first one to do so), the first overflow
10545 // would have to enter the range, which would mean that either we had left
10546 // the range before or that we started outside of it. Both of these cases
10547 // are contradictions.
10548 //
10549 // Claim: In the case where SolveForBoundary returns std::nullopt, the correct
10550 // solution is not some value between the Max for this boundary and the
10551 // Min of the other boundary.
10552 //
10553 // Justification: Assume that we had such Max_A and Min_B corresponding
10554 // to range boundaries A and B and such that Max_A < Min_B. If there was
10555 // a solution between Max_A and Min_B, it would have to be caused by an
10556 // overflow corresponding to either A or B. It cannot correspond to B,
10557 // since Min_B is the first occurrence of such an overflow. If it
10558 // corresponded to A, it would have to be either a signed or an unsigned
10559 // overflow that is larger than both eliminated overflows for A. But
10560 // between the eliminated overflows and this overflow, the values would
10561 // cover the entire value space, thus crossing the other boundary, which
10562 // is a contradiction.
10563
10564 return TruncIfPossible(MinOptional(SL.first, SU.first), BitWidth);
10565}
10566
10567ScalarEvolution::ExitLimit ScalarEvolution::howFarToZero(const SCEV *V,
10568 const Loop *L,
10569 bool ControlsOnlyExit,
10570 bool AllowPredicates) {
10571
10572 // This is only used for loops with a "x != y" exit test. The exit condition
10573 // is now expressed as a single expression, V = x-y. So the exit test is
10574 // effectively V != 0. We know and take advantage of the fact that this
10575 // expression only being used in a comparison by zero context.
10576
10578 // If the value is a constant
10579 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
10580 // If the value is already zero, the branch will execute zero times.
10581 if (C->getValue()->isZero()) return C;
10582 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10583 }
10584
10585 const SCEVAddRecExpr *AddRec =
10586 dyn_cast<SCEVAddRecExpr>(stripInjectiveFunctions(V));
10587
10588 if (!AddRec && AllowPredicates)
10589 // Try to make this an AddRec using runtime tests, in the first X
10590 // iterations of this loop, where X is the SCEV expression found by the
10591 // algorithm below.
10592 AddRec = convertSCEVToAddRecWithPredicates(V, L, Predicates);
10593
10594 if (!AddRec || AddRec->getLoop() != L)
10595 return getCouldNotCompute();
10596
10597 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
10598 // the quadratic equation to solve it.
10599 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
10600 // We can only use this value if the chrec ends up with an exact zero
10601 // value at this index. When solving for "X*X != 5", for example, we
10602 // should not accept a root of 2.
10603 if (auto S = SolveQuadraticAddRecExact(AddRec, *this)) {
10604 const auto *R = cast<SCEVConstant>(getConstant(*S));
10605 return ExitLimit(R, R, R, false, Predicates);
10606 }
10607 return getCouldNotCompute();
10608 }
10609
10610 // Otherwise we can only handle this if it is affine.
10611 if (!AddRec->isAffine())
10612 return getCouldNotCompute();
10613
10614 // If this is an affine expression, the execution count of this branch is
10615 // the minimum unsigned root of the following equation:
10616 //
10617 // Start + Step*N = 0 (mod 2^BW)
10618 //
10619 // equivalent to:
10620 //
10621 // Step*N = -Start (mod 2^BW)
10622 //
10623 // where BW is the common bit width of Start and Step.
10624
10625 // Get the initial value for the loop.
10626 const SCEV *Start = getSCEVAtScope(AddRec->getStart(), L->getParentLoop());
10627 const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1), L->getParentLoop());
10628
10629 if (!isLoopInvariant(Step, L))
10630 return getCouldNotCompute();
10631
10632 LoopGuards Guards = LoopGuards::collect(L, *this);
10633 // Specialize step for this loop so we get context sensitive facts below.
10634 const SCEV *StepWLG = applyLoopGuards(Step, Guards);
10635
10636 // For positive steps (counting up until unsigned overflow):
10637 // N = -Start/Step (as unsigned)
10638 // For negative steps (counting down to zero):
10639 // N = Start/-Step
10640 // First compute the unsigned distance from zero in the direction of Step.
10641 bool CountDown = isKnownNegative(StepWLG);
10642 if (!CountDown && !isKnownNonNegative(StepWLG))
10643 return getCouldNotCompute();
10644
10645 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(Start);
10646 // Handle unitary steps, which cannot wraparound.
10647 // 1*N = -Start; -1*N = Start (mod 2^BW), so:
10648 // N = Distance (as unsigned)
10649
10650 if (match(Step, m_CombineOr(m_scev_One(), m_scev_AllOnes()))) {
10651 APInt MaxBECount = getUnsignedRangeMax(applyLoopGuards(Distance, Guards));
10652 MaxBECount = APIntOps::umin(MaxBECount, getUnsignedRangeMax(Distance));
10653
10654 // When a loop like "for (int i = 0; i != n; ++i) { /* body */ }" is rotated,
10655 // we end up with a loop whose backedge-taken count is n - 1. Detect this
10656 // case, and see if we can improve the bound.
10657 //
10658 // Explicitly handling this here is necessary because getUnsignedRange
10659 // isn't context-sensitive; it doesn't know that we only care about the
10660 // range inside the loop.
10661 const SCEV *Zero = getZero(Distance->getType());
10662 const SCEV *One = getOne(Distance->getType());
10663 const SCEV *DistancePlusOne = getAddExpr(Distance, One);
10664 if (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_NE, DistancePlusOne, Zero)) {
10665 // If Distance + 1 doesn't overflow, we can compute the maximum distance
10666 // as "unsigned_max(Distance + 1) - 1".
10667 ConstantRange CR = getUnsignedRange(DistancePlusOne);
10668 MaxBECount = APIntOps::umin(MaxBECount, CR.getUnsignedMax() - 1);
10669 }
10670 return ExitLimit(Distance, getConstant(MaxBECount), Distance, false,
10671 Predicates);
10672 }
10673
10674 // If the condition controls loop exit (the loop exits only if the expression
10675 // is true) and the addition is no-wrap we can use unsigned divide to
10676 // compute the backedge count. In this case, the step may not divide the
10677 // distance, but we don't care because if the condition is "missed" the loop
10678 // will have undefined behavior due to wrapping.
10679 if (ControlsOnlyExit && AddRec->hasNoSelfWrap() &&
10680 loopHasNoAbnormalExits(AddRec->getLoop())) {
10681
10682 // If the stride is zero and the start is non-zero, the loop must be
10683 // infinite. In C++, most loops are finite by assumption, in which case the
10684 // step being zero implies UB must execute if the loop is entered.
10685 if (!(loopIsFiniteByAssumption(L) && isKnownNonZero(Start)) &&
10686 !isKnownNonZero(StepWLG))
10687 return getCouldNotCompute();
10688
10689 const SCEV *Exact =
10690 getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step);
10691 const SCEV *ConstantMax = getCouldNotCompute();
10692 if (Exact != getCouldNotCompute()) {
10693 APInt MaxInt = getUnsignedRangeMax(applyLoopGuards(Exact, Guards));
10694 ConstantMax =
10696 }
10697 const SCEV *SymbolicMax =
10698 isa<SCEVCouldNotCompute>(Exact) ? ConstantMax : Exact;
10699 return ExitLimit(Exact, ConstantMax, SymbolicMax, false, Predicates);
10700 }
10701
10702 // Solve the general equation.
10703 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Step);
10704 if (!StepC || StepC->getValue()->isZero())
10705 return getCouldNotCompute();
10706 const SCEV *E = SolveLinEquationWithOverflow(
10707 StepC->getAPInt(), getNegativeSCEV(Start),
10708 AllowPredicates ? &Predicates : nullptr, *this, L);
10709
10710 const SCEV *M = E;
10711 if (E != getCouldNotCompute()) {
10712 APInt MaxWithGuards = getUnsignedRangeMax(applyLoopGuards(E, Guards));
10713 M = getConstant(APIntOps::umin(MaxWithGuards, getUnsignedRangeMax(E)));
10714 }
10715 auto *S = isa<SCEVCouldNotCompute>(E) ? M : E;
10716 return ExitLimit(E, M, S, false, Predicates);
10717}
10718
10719ScalarEvolution::ExitLimit
10720ScalarEvolution::howFarToNonZero(const SCEV *V, const Loop *L) {
10721 // Loops that look like: while (X == 0) are very strange indeed. We don't
10722 // handle them yet except for the trivial case. This could be expanded in the
10723 // future as needed.
10724
10725 // If the value is a constant, check to see if it is known to be non-zero
10726 // already. If so, the backedge will execute zero times.
10727 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
10728 if (!C->getValue()->isZero())
10729 return getZero(C->getType());
10730 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10731 }
10732
10733 // We could implement others, but I really doubt anyone writes loops like
10734 // this, and if they did, they would already be constant folded.
10735 return getCouldNotCompute();
10736}
10737
10738std::pair<const BasicBlock *, const BasicBlock *>
10739ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(const BasicBlock *BB)
10740 const {
10741 // If the block has a unique predecessor, then there is no path from the
10742 // predecessor to the block that does not go through the direct edge
10743 // from the predecessor to the block.
10744 if (const BasicBlock *Pred = BB->getSinglePredecessor())
10745 return {Pred, BB};
10746
10747 // A loop's header is defined to be a block that dominates the loop.
10748 // If the header has a unique predecessor outside the loop, it must be
10749 // a block that has exactly one successor that can reach the loop.
10750 if (const Loop *L = LI.getLoopFor(BB))
10751 return {L->getLoopPredecessor(), L->getHeader()};
10752
10753 return {nullptr, BB};
10754}
10755
10756/// SCEV structural equivalence is usually sufficient for testing whether two
10757/// expressions are equal, however for the purposes of looking for a condition
10758/// guarding a loop, it can be useful to be a little more general, since a
10759/// front-end may have replicated the controlling expression.
10760static bool HasSameValue(const SCEV *A, const SCEV *B) {
10761 // Quick check to see if they are the same SCEV.
10762 if (A == B) return true;
10763
10764 auto ComputesEqualValues = [](const Instruction *A, const Instruction *B) {
10765 // Not all instructions that are "identical" compute the same value. For
10766 // instance, two distinct alloca instructions allocating the same type are
10767 // identical and do not read memory; but compute distinct values.
10768 return A->isIdenticalTo(B) && (isa<BinaryOperator>(A) || isa<GetElementPtrInst>(A));
10769 };
10770
10771 // Otherwise, if they're both SCEVUnknown, it's possible that they hold
10772 // two different instructions with the same value. Check for this case.
10773 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(A))
10774 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(B))
10775 if (const Instruction *AI = dyn_cast<Instruction>(AU->getValue()))
10776 if (const Instruction *BI = dyn_cast<Instruction>(BU->getValue()))
10777 if (ComputesEqualValues(AI, BI))
10778 return true;
10779
10780 // Otherwise assume they may have a different value.
10781 return false;
10782}
10783
10784static bool MatchBinarySub(const SCEV *S, const SCEV *&LHS, const SCEV *&RHS) {
10785 const SCEV *Op0, *Op1;
10786 if (!match(S, m_scev_Add(m_SCEV(Op0), m_SCEV(Op1))))
10787 return false;
10788 if (match(Op0, m_scev_Mul(m_scev_AllOnes(), m_SCEV(RHS)))) {
10789 LHS = Op1;
10790 return true;
10791 }
10792 if (match(Op1, m_scev_Mul(m_scev_AllOnes(), m_SCEV(RHS)))) {
10793 LHS = Op0;
10794 return true;
10795 }
10796 return false;
10797}
10798
10800 const SCEV *&RHS, unsigned Depth) {
10801 bool Changed = false;
10802 // Simplifies ICMP to trivial true or false by turning it into '0 == 0' or
10803 // '0 != 0'.
10804 auto TrivialCase = [&](bool TriviallyTrue) {
10806 Pred = TriviallyTrue ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
10807 return true;
10808 };
10809 // If we hit the max recursion limit bail out.
10810 if (Depth >= 3)
10811 return false;
10812
10813 const SCEV *NewLHS, *NewRHS;
10814 if (match(LHS, m_scev_c_Mul(m_SCEV(NewLHS), m_SCEVVScale())) &&
10815 match(RHS, m_scev_c_Mul(m_SCEV(NewRHS), m_SCEVVScale()))) {
10816 const SCEVMulExpr *LMul = cast<SCEVMulExpr>(LHS);
10817 const SCEVMulExpr *RMul = cast<SCEVMulExpr>(RHS);
10818
10819 // (X * vscale) pred (Y * vscale) ==> X pred Y
10820 // when both multiples are NSW.
10821 // (X * vscale) uicmp/eq/ne (Y * vscale) ==> X uicmp/eq/ne Y
10822 // when both multiples are NUW.
10823 if ((LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap()) ||
10824 (LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap() &&
10825 !ICmpInst::isSigned(Pred))) {
10826 LHS = NewLHS;
10827 RHS = NewRHS;
10828 Changed = true;
10829 }
10830 }
10831
10832 // Canonicalize a constant to the right side.
10833 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
10834 // Check for both operands constant.
10835 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
10836 if (!ICmpInst::compare(LHSC->getAPInt(), RHSC->getAPInt(), Pred))
10837 return TrivialCase(false);
10838 return TrivialCase(true);
10839 }
10840 // Otherwise swap the operands to put the constant on the right.
10841 std::swap(LHS, RHS);
10843 Changed = true;
10844 }
10845
10846 // If we're comparing an addrec with a value which is loop-invariant in the
10847 // addrec's loop, put the addrec on the left. Also make a dominance check,
10848 // as both operands could be addrecs loop-invariant in each other's loop.
10849 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) {
10850 const Loop *L = AR->getLoop();
10851 if (isLoopInvariant(LHS, L) && properlyDominates(LHS, L->getHeader())) {
10852 std::swap(LHS, RHS);
10854 Changed = true;
10855 }
10856 }
10857
10858 // If there's a constant operand, canonicalize comparisons with boundary
10859 // cases, and canonicalize *-or-equal comparisons to regular comparisons.
10860 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS)) {
10861 const APInt &RA = RC->getAPInt();
10862
10863 bool SimplifiedByConstantRange = false;
10864
10865 if (!ICmpInst::isEquality(Pred)) {
10867 if (ExactCR.isFullSet())
10868 return TrivialCase(true);
10869 if (ExactCR.isEmptySet())
10870 return TrivialCase(false);
10871
10872 APInt NewRHS;
10873 CmpInst::Predicate NewPred;
10874 if (ExactCR.getEquivalentICmp(NewPred, NewRHS) &&
10875 ICmpInst::isEquality(NewPred)) {
10876 // We were able to convert an inequality to an equality.
10877 Pred = NewPred;
10878 RHS = getConstant(NewRHS);
10879 Changed = SimplifiedByConstantRange = true;
10880 }
10881 }
10882
10883 if (!SimplifiedByConstantRange) {
10884 switch (Pred) {
10885 default:
10886 break;
10887 case ICmpInst::ICMP_EQ:
10888 case ICmpInst::ICMP_NE:
10889 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b.
10890 if (RA.isZero() && MatchBinarySub(LHS, LHS, RHS))
10891 Changed = true;
10892 break;
10893
10894 // The "Should have been caught earlier!" messages refer to the fact
10895 // that the ExactCR.isFullSet() or ExactCR.isEmptySet() check above
10896 // should have fired on the corresponding cases, and canonicalized the
10897 // check to trivial case.
10898
10899 case ICmpInst::ICMP_UGE:
10900 assert(!RA.isMinValue() && "Should have been caught earlier!");
10901 Pred = ICmpInst::ICMP_UGT;
10902 RHS = getConstant(RA - 1);
10903 Changed = true;
10904 break;
10905 case ICmpInst::ICMP_ULE:
10906 assert(!RA.isMaxValue() && "Should have been caught earlier!");
10907 Pred = ICmpInst::ICMP_ULT;
10908 RHS = getConstant(RA + 1);
10909 Changed = true;
10910 break;
10911 case ICmpInst::ICMP_SGE:
10912 assert(!RA.isMinSignedValue() && "Should have been caught earlier!");
10913 Pred = ICmpInst::ICMP_SGT;
10914 RHS = getConstant(RA - 1);
10915 Changed = true;
10916 break;
10917 case ICmpInst::ICMP_SLE:
10918 assert(!RA.isMaxSignedValue() && "Should have been caught earlier!");
10919 Pred = ICmpInst::ICMP_SLT;
10920 RHS = getConstant(RA + 1);
10921 Changed = true;
10922 break;
10923 }
10924 }
10925 }
10926
10927 // Check for obvious equality.
10928 if (HasSameValue(LHS, RHS)) {
10929 if (ICmpInst::isTrueWhenEqual(Pred))
10930 return TrivialCase(true);
10932 return TrivialCase(false);
10933 }
10934
10935 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
10936 // adding or subtracting 1 from one of the operands.
10937 switch (Pred) {
10938 case ICmpInst::ICMP_SLE:
10939 if (!getSignedRangeMax(RHS).isMaxSignedValue()) {
10940 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
10942 Pred = ICmpInst::ICMP_SLT;
10943 Changed = true;
10944 } else if (!getSignedRangeMin(LHS).isMinSignedValue()) {
10945 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS,
10947 Pred = ICmpInst::ICMP_SLT;
10948 Changed = true;
10949 }
10950 break;
10951 case ICmpInst::ICMP_SGE:
10952 if (!getSignedRangeMin(RHS).isMinSignedValue()) {
10953 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS,
10955 Pred = ICmpInst::ICMP_SGT;
10956 Changed = true;
10957 } else if (!getSignedRangeMax(LHS).isMaxSignedValue()) {
10958 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
10960 Pred = ICmpInst::ICMP_SGT;
10961 Changed = true;
10962 }
10963 break;
10964 case ICmpInst::ICMP_ULE:
10965 if (!getUnsignedRangeMax(RHS).isMaxValue()) {
10966 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
10968 Pred = ICmpInst::ICMP_ULT;
10969 Changed = true;
10970 } else if (!getUnsignedRangeMin(LHS).isMinValue()) {
10971 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS);
10972 Pred = ICmpInst::ICMP_ULT;
10973 Changed = true;
10974 }
10975 break;
10976 case ICmpInst::ICMP_UGE:
10977 // If RHS is an op we can fold the -1, try that first.
10978 // Otherwise prefer LHS to preserve the nuw flag.
10979 if ((isa<SCEVConstant>(RHS) ||
10981 isa<SCEVConstant>(cast<SCEVNAryExpr>(RHS)->getOperand(0)))) &&
10982 !getUnsignedRangeMin(RHS).isMinValue()) {
10983 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS);
10984 Pred = ICmpInst::ICMP_UGT;
10985 Changed = true;
10986 } else if (!getUnsignedRangeMax(LHS).isMaxValue()) {
10987 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
10989 Pred = ICmpInst::ICMP_UGT;
10990 Changed = true;
10991 } else if (!getUnsignedRangeMin(RHS).isMinValue()) {
10992 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS);
10993 Pred = ICmpInst::ICMP_UGT;
10994 Changed = true;
10995 }
10996 break;
10997 default:
10998 break;
10999 }
11000
11001 // TODO: More simplifications are possible here.
11002
11003 // Recursively simplify until we either hit a recursion limit or nothing
11004 // changes.
11005 if (Changed)
11006 (void)SimplifyICmpOperands(Pred, LHS, RHS, Depth + 1);
11007
11008 return Changed;
11009}
11010
11012 return getSignedRangeMax(S).isNegative();
11013}
11014
11018
11020 return !getSignedRangeMin(S).isNegative();
11021}
11022
11026
11028 // Query push down for cases where the unsigned range is
11029 // less than sufficient.
11030 if (const auto *SExt = dyn_cast<SCEVSignExtendExpr>(S))
11031 return isKnownNonZero(SExt->getOperand(0));
11032 return getUnsignedRangeMin(S) != 0;
11033}
11034
11036 bool OrNegative) {
11037 auto NonRecursive = [this, OrNegative](const SCEV *S) {
11038 if (auto *C = dyn_cast<SCEVConstant>(S))
11039 return C->getAPInt().isPowerOf2() ||
11040 (OrNegative && C->getAPInt().isNegatedPowerOf2());
11041
11042 // The vscale_range indicates vscale is a power-of-two.
11043 return isa<SCEVVScale>(S) && F.hasFnAttribute(Attribute::VScaleRange);
11044 };
11045
11046 if (NonRecursive(S))
11047 return true;
11048
11049 auto *Mul = dyn_cast<SCEVMulExpr>(S);
11050 if (!Mul)
11051 return false;
11052 return all_of(Mul->operands(), NonRecursive) && (OrZero || isKnownNonZero(S));
11053}
11054
11056 const SCEV *S, uint64_t M,
11058 if (M == 0)
11059 return false;
11060 if (M == 1)
11061 return true;
11062
11063 // Recursively check AddRec operands. An AddRecExpr S is a multiple of M if S
11064 // starts with a multiple of M and at every iteration step S only adds
11065 // multiples of M.
11066 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(S))
11067 return isKnownMultipleOf(AddRec->getStart(), M, Assumptions) &&
11068 isKnownMultipleOf(AddRec->getStepRecurrence(*this), M, Assumptions);
11069
11070 // For a constant, check that "S % M == 0".
11071 if (auto *Cst = dyn_cast<SCEVConstant>(S)) {
11072 APInt C = Cst->getAPInt();
11073 return C.urem(M) == 0;
11074 }
11075
11076 // TODO: Also check other SCEV expressions, i.e., SCEVAddRecExpr, etc.
11077
11078 // Basic tests have failed.
11079 // Check "S % M == 0" at compile time and record runtime Assumptions.
11080 auto *STy = dyn_cast<IntegerType>(S->getType());
11081 const SCEV *SmodM =
11082 getURemExpr(S, getConstant(ConstantInt::get(STy, M, false)));
11083 const SCEV *Zero = getZero(STy);
11084
11085 // Check whether "S % M == 0" is known at compile time.
11086 if (isKnownPredicate(ICmpInst::ICMP_EQ, SmodM, Zero))
11087 return true;
11088
11089 // Check whether "S % M != 0" is known at compile time.
11090 if (isKnownPredicate(ICmpInst::ICMP_NE, SmodM, Zero))
11091 return false;
11092
11094
11095 // Detect redundant predicates.
11096 for (auto *A : Assumptions)
11097 if (A->implies(P, *this))
11098 return true;
11099
11100 // Only record non-redundant predicates.
11101 Assumptions.push_back(P);
11102 return true;
11103}
11104
11105std::pair<const SCEV *, const SCEV *>
11107 // Compute SCEV on entry of loop L.
11108 const SCEV *Start = SCEVInitRewriter::rewrite(S, L, *this);
11109 if (Start == getCouldNotCompute())
11110 return { Start, Start };
11111 // Compute post increment SCEV for loop L.
11112 const SCEV *PostInc = SCEVPostIncRewriter::rewrite(S, L, *this);
11113 assert(PostInc != getCouldNotCompute() && "Unexpected could not compute");
11114 return { Start, PostInc };
11115}
11116
11118 const SCEV *RHS) {
11119 // First collect all loops.
11121 getUsedLoops(LHS, LoopsUsed);
11122 getUsedLoops(RHS, LoopsUsed);
11123
11124 if (LoopsUsed.empty())
11125 return false;
11126
11127 // Domination relationship must be a linear order on collected loops.
11128#ifndef NDEBUG
11129 for (const auto *L1 : LoopsUsed)
11130 for (const auto *L2 : LoopsUsed)
11131 assert((DT.dominates(L1->getHeader(), L2->getHeader()) ||
11132 DT.dominates(L2->getHeader(), L1->getHeader())) &&
11133 "Domination relationship is not a linear order");
11134#endif
11135
11136 const Loop *MDL =
11137 *llvm::max_element(LoopsUsed, [&](const Loop *L1, const Loop *L2) {
11138 return DT.properlyDominates(L1->getHeader(), L2->getHeader());
11139 });
11140
11141 // Get init and post increment value for LHS.
11142 auto SplitLHS = SplitIntoInitAndPostInc(MDL, LHS);
11143 // if LHS contains unknown non-invariant SCEV then bail out.
11144 if (SplitLHS.first == getCouldNotCompute())
11145 return false;
11146 assert (SplitLHS.second != getCouldNotCompute() && "Unexpected CNC");
11147 // Get init and post increment value for RHS.
11148 auto SplitRHS = SplitIntoInitAndPostInc(MDL, RHS);
11149 // if RHS contains unknown non-invariant SCEV then bail out.
11150 if (SplitRHS.first == getCouldNotCompute())
11151 return false;
11152 assert (SplitRHS.second != getCouldNotCompute() && "Unexpected CNC");
11153 // It is possible that init SCEV contains an invariant load but it does
11154 // not dominate MDL and is not available at MDL loop entry, so we should
11155 // check it here.
11156 if (!isAvailableAtLoopEntry(SplitLHS.first, MDL) ||
11157 !isAvailableAtLoopEntry(SplitRHS.first, MDL))
11158 return false;
11159
11160 // It seems backedge guard check is faster than entry one so in some cases
11161 // it can speed up whole estimation by short circuit
11162 return isLoopBackedgeGuardedByCond(MDL, Pred, SplitLHS.second,
11163 SplitRHS.second) &&
11164 isLoopEntryGuardedByCond(MDL, Pred, SplitLHS.first, SplitRHS.first);
11165}
11166
11168 const SCEV *RHS) {
11169 // Canonicalize the inputs first.
11170 (void)SimplifyICmpOperands(Pred, LHS, RHS);
11171
11172 if (isKnownViaInduction(Pred, LHS, RHS))
11173 return true;
11174
11175 if (isKnownPredicateViaSplitting(Pred, LHS, RHS))
11176 return true;
11177
11178 // Otherwise see what can be done with some simple reasoning.
11179 return isKnownViaNonRecursiveReasoning(Pred, LHS, RHS);
11180}
11181
11183 const SCEV *LHS,
11184 const SCEV *RHS) {
11185 if (isKnownPredicate(Pred, LHS, RHS))
11186 return true;
11188 return false;
11189 return std::nullopt;
11190}
11191
11193 const SCEV *RHS,
11194 const Instruction *CtxI) {
11195 // TODO: Analyze guards and assumes from Context's block.
11196 return isKnownPredicate(Pred, LHS, RHS) ||
11197 isBasicBlockEntryGuardedByCond(CtxI->getParent(), Pred, LHS, RHS);
11198}
11199
11200std::optional<bool>
11202 const SCEV *RHS, const Instruction *CtxI) {
11203 std::optional<bool> KnownWithoutContext = evaluatePredicate(Pred, LHS, RHS);
11204 if (KnownWithoutContext)
11205 return KnownWithoutContext;
11206
11207 if (isBasicBlockEntryGuardedByCond(CtxI->getParent(), Pred, LHS, RHS))
11208 return true;
11210 CtxI->getParent(), ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11211 return false;
11212 return std::nullopt;
11213}
11214
11216 const SCEVAddRecExpr *LHS,
11217 const SCEV *RHS) {
11218 const Loop *L = LHS->getLoop();
11219 return isLoopEntryGuardedByCond(L, Pred, LHS->getStart(), RHS) &&
11220 isLoopBackedgeGuardedByCond(L, Pred, LHS->getPostIncExpr(*this), RHS);
11221}
11222
11223std::optional<ScalarEvolution::MonotonicPredicateType>
11225 ICmpInst::Predicate Pred) {
11226 auto Result = getMonotonicPredicateTypeImpl(LHS, Pred);
11227
11228#ifndef NDEBUG
11229 // Verify an invariant: inverting the predicate should turn a monotonically
11230 // increasing change to a monotonically decreasing one, and vice versa.
11231 if (Result) {
11232 auto ResultSwapped =
11233 getMonotonicPredicateTypeImpl(LHS, ICmpInst::getSwappedPredicate(Pred));
11234
11235 assert(*ResultSwapped != *Result &&
11236 "monotonicity should flip as we flip the predicate");
11237 }
11238#endif
11239
11240 return Result;
11241}
11242
11243std::optional<ScalarEvolution::MonotonicPredicateType>
11244ScalarEvolution::getMonotonicPredicateTypeImpl(const SCEVAddRecExpr *LHS,
11245 ICmpInst::Predicate Pred) {
11246 // A zero step value for LHS means the induction variable is essentially a
11247 // loop invariant value. We don't really depend on the predicate actually
11248 // flipping from false to true (for increasing predicates, and the other way
11249 // around for decreasing predicates), all we care about is that *if* the
11250 // predicate changes then it only changes from false to true.
11251 //
11252 // A zero step value in itself is not very useful, but there may be places
11253 // where SCEV can prove X >= 0 but not prove X > 0, so it is helpful to be
11254 // as general as possible.
11255
11256 // Only handle LE/LT/GE/GT predicates.
11257 if (!ICmpInst::isRelational(Pred))
11258 return std::nullopt;
11259
11260 bool IsGreater = ICmpInst::isGE(Pred) || ICmpInst::isGT(Pred);
11261 assert((IsGreater || ICmpInst::isLE(Pred) || ICmpInst::isLT(Pred)) &&
11262 "Should be greater or less!");
11263
11264 // Check that AR does not wrap.
11265 if (ICmpInst::isUnsigned(Pred)) {
11266 if (!LHS->hasNoUnsignedWrap())
11267 return std::nullopt;
11269 }
11270 assert(ICmpInst::isSigned(Pred) &&
11271 "Relational predicate is either signed or unsigned!");
11272 if (!LHS->hasNoSignedWrap())
11273 return std::nullopt;
11274
11275 const SCEV *Step = LHS->getStepRecurrence(*this);
11276
11277 if (isKnownNonNegative(Step))
11279
11280 if (isKnownNonPositive(Step))
11282
11283 return std::nullopt;
11284}
11285
11286std::optional<ScalarEvolution::LoopInvariantPredicate>
11288 const SCEV *RHS, const Loop *L,
11289 const Instruction *CtxI) {
11290 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11291 if (!isLoopInvariant(RHS, L)) {
11292 if (!isLoopInvariant(LHS, L))
11293 return std::nullopt;
11294
11295 std::swap(LHS, RHS);
11297 }
11298
11299 const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(LHS);
11300 if (!ArLHS || ArLHS->getLoop() != L)
11301 return std::nullopt;
11302
11303 auto MonotonicType = getMonotonicPredicateType(ArLHS, Pred);
11304 if (!MonotonicType)
11305 return std::nullopt;
11306 // If the predicate "ArLHS `Pred` RHS" monotonically increases from false to
11307 // true as the loop iterates, and the backedge is control dependent on
11308 // "ArLHS `Pred` RHS" == true then we can reason as follows:
11309 //
11310 // * if the predicate was false in the first iteration then the predicate
11311 // is never evaluated again, since the loop exits without taking the
11312 // backedge.
11313 // * if the predicate was true in the first iteration then it will
11314 // continue to be true for all future iterations since it is
11315 // monotonically increasing.
11316 //
11317 // For both the above possibilities, we can replace the loop varying
11318 // predicate with its value on the first iteration of the loop (which is
11319 // loop invariant).
11320 //
11321 // A similar reasoning applies for a monotonically decreasing predicate, by
11322 // replacing true with false and false with true in the above two bullets.
11324 auto P = Increasing ? Pred : ICmpInst::getInverseCmpPredicate(Pred);
11325
11326 if (isLoopBackedgeGuardedByCond(L, P, LHS, RHS))
11328 RHS);
11329
11330 if (!CtxI)
11331 return std::nullopt;
11332 // Try to prove via context.
11333 // TODO: Support other cases.
11334 switch (Pred) {
11335 default:
11336 break;
11337 case ICmpInst::ICMP_ULE:
11338 case ICmpInst::ICMP_ULT: {
11339 assert(ArLHS->hasNoUnsignedWrap() && "Is a requirement of monotonicity!");
11340 // Given preconditions
11341 // (1) ArLHS does not cross the border of positive and negative parts of
11342 // range because of:
11343 // - Positive step; (TODO: lift this limitation)
11344 // - nuw - does not cross zero boundary;
11345 // - nsw - does not cross SINT_MAX boundary;
11346 // (2) ArLHS <s RHS
11347 // (3) RHS >=s 0
11348 // we can replace the loop variant ArLHS <u RHS condition with loop
11349 // invariant Start(ArLHS) <u RHS.
11350 //
11351 // Because of (1) there are two options:
11352 // - ArLHS is always negative. It means that ArLHS <u RHS is always false;
11353 // - ArLHS is always non-negative. Because of (3) RHS is also non-negative.
11354 // It means that ArLHS <s RHS <=> ArLHS <u RHS.
11355 // Because of (2) ArLHS <u RHS is trivially true.
11356 // All together it means that ArLHS <u RHS <=> Start(ArLHS) >=s 0.
11357 // We can strengthen this to Start(ArLHS) <u RHS.
11358 auto SignFlippedPred = ICmpInst::getFlippedSignednessPredicate(Pred);
11359 if (ArLHS->hasNoSignedWrap() && ArLHS->isAffine() &&
11360 isKnownPositive(ArLHS->getStepRecurrence(*this)) &&
11361 isKnownNonNegative(RHS) &&
11362 isKnownPredicateAt(SignFlippedPred, ArLHS, RHS, CtxI))
11364 RHS);
11365 }
11366 }
11367
11368 return std::nullopt;
11369}
11370
11371std::optional<ScalarEvolution::LoopInvariantPredicate>
11373 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11374 const Instruction *CtxI, const SCEV *MaxIter) {
11376 Pred, LHS, RHS, L, CtxI, MaxIter))
11377 return LIP;
11378 if (auto *UMin = dyn_cast<SCEVUMinExpr>(MaxIter))
11379 // Number of iterations expressed as UMIN isn't always great for expressing
11380 // the value on the last iteration. If the straightforward approach didn't
11381 // work, try the following trick: if the a predicate is invariant for X, it
11382 // is also invariant for umin(X, ...). So try to find something that works
11383 // among subexpressions of MaxIter expressed as umin.
11384 for (auto *Op : UMin->operands())
11386 Pred, LHS, RHS, L, CtxI, Op))
11387 return LIP;
11388 return std::nullopt;
11389}
11390
11391std::optional<ScalarEvolution::LoopInvariantPredicate>
11393 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11394 const Instruction *CtxI, const SCEV *MaxIter) {
11395 // Try to prove the following set of facts:
11396 // - The predicate is monotonic in the iteration space.
11397 // - If the check does not fail on the 1st iteration:
11398 // - No overflow will happen during first MaxIter iterations;
11399 // - It will not fail on the MaxIter'th iteration.
11400 // If the check does fail on the 1st iteration, we leave the loop and no
11401 // other checks matter.
11402
11403 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11404 if (!isLoopInvariant(RHS, L)) {
11405 if (!isLoopInvariant(LHS, L))
11406 return std::nullopt;
11407
11408 std::swap(LHS, RHS);
11410 }
11411
11412 auto *AR = dyn_cast<SCEVAddRecExpr>(LHS);
11413 if (!AR || AR->getLoop() != L)
11414 return std::nullopt;
11415
11416 // The predicate must be relational (i.e. <, <=, >=, >).
11417 if (!ICmpInst::isRelational(Pred))
11418 return std::nullopt;
11419
11420 // TODO: Support steps other than +/- 1.
11421 const SCEV *Step = AR->getStepRecurrence(*this);
11422 auto *One = getOne(Step->getType());
11423 auto *MinusOne = getNegativeSCEV(One);
11424 if (Step != One && Step != MinusOne)
11425 return std::nullopt;
11426
11427 // Type mismatch here means that MaxIter is potentially larger than max
11428 // unsigned value in start type, which mean we cannot prove no wrap for the
11429 // indvar.
11430 if (AR->getType() != MaxIter->getType())
11431 return std::nullopt;
11432
11433 // Value of IV on suggested last iteration.
11434 const SCEV *Last = AR->evaluateAtIteration(MaxIter, *this);
11435 // Does it still meet the requirement?
11436 if (!isLoopBackedgeGuardedByCond(L, Pred, Last, RHS))
11437 return std::nullopt;
11438 // Because step is +/- 1 and MaxIter has same type as Start (i.e. it does
11439 // not exceed max unsigned value of this type), this effectively proves
11440 // that there is no wrap during the iteration. To prove that there is no
11441 // signed/unsigned wrap, we need to check that
11442 // Start <= Last for step = 1 or Start >= Last for step = -1.
11443 ICmpInst::Predicate NoOverflowPred =
11445 if (Step == MinusOne)
11446 NoOverflowPred = ICmpInst::getSwappedCmpPredicate(NoOverflowPred);
11447 const SCEV *Start = AR->getStart();
11448 if (!isKnownPredicateAt(NoOverflowPred, Start, Last, CtxI))
11449 return std::nullopt;
11450
11451 // Everything is fine.
11452 return ScalarEvolution::LoopInvariantPredicate(Pred, Start, RHS);
11453}
11454
11455bool ScalarEvolution::isKnownPredicateViaConstantRanges(CmpPredicate Pred,
11456 const SCEV *LHS,
11457 const SCEV *RHS) {
11458 if (HasSameValue(LHS, RHS))
11459 return ICmpInst::isTrueWhenEqual(Pred);
11460
11461 auto CheckRange = [&](bool IsSigned) {
11462 auto RangeLHS = IsSigned ? getSignedRange(LHS) : getUnsignedRange(LHS);
11463 auto RangeRHS = IsSigned ? getSignedRange(RHS) : getUnsignedRange(RHS);
11464 return RangeLHS.icmp(Pred, RangeRHS);
11465 };
11466
11467 // The check at the top of the function catches the case where the values are
11468 // known to be equal.
11469 if (Pred == CmpInst::ICMP_EQ)
11470 return false;
11471
11472 if (Pred == CmpInst::ICMP_NE) {
11473 if (CheckRange(true) || CheckRange(false))
11474 return true;
11475 auto *Diff = getMinusSCEV(LHS, RHS);
11476 return !isa<SCEVCouldNotCompute>(Diff) && isKnownNonZero(Diff);
11477 }
11478
11479 return CheckRange(CmpInst::isSigned(Pred));
11480}
11481
11482bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred,
11483 const SCEV *LHS,
11484 const SCEV *RHS) {
11485 // Match X to (A + C1)<ExpectedFlags> and Y to (A + C2)<ExpectedFlags>, where
11486 // C1 and C2 are constant integers. If either X or Y are not add expressions,
11487 // consider them as X + 0 and Y + 0 respectively. C1 and C2 are returned via
11488 // OutC1 and OutC2.
11489 auto MatchBinaryAddToConst = [this](const SCEV *X, const SCEV *Y,
11490 APInt &OutC1, APInt &OutC2,
11491 SCEV::NoWrapFlags ExpectedFlags) {
11492 const SCEV *XNonConstOp, *XConstOp;
11493 const SCEV *YNonConstOp, *YConstOp;
11494 SCEV::NoWrapFlags XFlagsPresent;
11495 SCEV::NoWrapFlags YFlagsPresent;
11496
11497 if (!splitBinaryAdd(X, XConstOp, XNonConstOp, XFlagsPresent)) {
11498 XConstOp = getZero(X->getType());
11499 XNonConstOp = X;
11500 XFlagsPresent = ExpectedFlags;
11501 }
11502 if (!isa<SCEVConstant>(XConstOp))
11503 return false;
11504
11505 if (!splitBinaryAdd(Y, YConstOp, YNonConstOp, YFlagsPresent)) {
11506 YConstOp = getZero(Y->getType());
11507 YNonConstOp = Y;
11508 YFlagsPresent = ExpectedFlags;
11509 }
11510
11511 if (YNonConstOp != XNonConstOp)
11512 return false;
11513
11514 if (!isa<SCEVConstant>(YConstOp))
11515 return false;
11516
11517 // When matching ADDs with NUW flags (and unsigned predicates), only the
11518 // second ADD (with the larger constant) requires NUW.
11519 if ((YFlagsPresent & ExpectedFlags) != ExpectedFlags)
11520 return false;
11521 if (ExpectedFlags != SCEV::FlagNUW &&
11522 (XFlagsPresent & ExpectedFlags) != ExpectedFlags) {
11523 return false;
11524 }
11525
11526 OutC1 = cast<SCEVConstant>(XConstOp)->getAPInt();
11527 OutC2 = cast<SCEVConstant>(YConstOp)->getAPInt();
11528
11529 return true;
11530 };
11531
11532 APInt C1;
11533 APInt C2;
11534
11535 switch (Pred) {
11536 default:
11537 break;
11538
11539 case ICmpInst::ICMP_SGE:
11540 std::swap(LHS, RHS);
11541 [[fallthrough]];
11542 case ICmpInst::ICMP_SLE:
11543 // (X + C1)<nsw> s<= (X + C2)<nsw> if C1 s<= C2.
11544 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.sle(C2))
11545 return true;
11546
11547 break;
11548
11549 case ICmpInst::ICMP_SGT:
11550 std::swap(LHS, RHS);
11551 [[fallthrough]];
11552 case ICmpInst::ICMP_SLT:
11553 // (X + C1)<nsw> s< (X + C2)<nsw> if C1 s< C2.
11554 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.slt(C2))
11555 return true;
11556
11557 break;
11558
11559 case ICmpInst::ICMP_UGE:
11560 std::swap(LHS, RHS);
11561 [[fallthrough]];
11562 case ICmpInst::ICMP_ULE:
11563 // (X + C1) u<= (X + C2)<nuw> for C1 u<= C2.
11564 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ule(C2))
11565 return true;
11566
11567 break;
11568
11569 case ICmpInst::ICMP_UGT:
11570 std::swap(LHS, RHS);
11571 [[fallthrough]];
11572 case ICmpInst::ICMP_ULT:
11573 // (X + C1) u< (X + C2)<nuw> if C1 u< C2.
11574 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ult(C2))
11575 return true;
11576 break;
11577 }
11578
11579 return false;
11580}
11581
11582bool ScalarEvolution::isKnownPredicateViaSplitting(CmpPredicate Pred,
11583 const SCEV *LHS,
11584 const SCEV *RHS) {
11585 if (Pred != ICmpInst::ICMP_ULT || ProvingSplitPredicate)
11586 return false;
11587
11588 // Allowing arbitrary number of activations of isKnownPredicateViaSplitting on
11589 // the stack can result in exponential time complexity.
11590 SaveAndRestore Restore(ProvingSplitPredicate, true);
11591
11592 // If L >= 0 then I `ult` L <=> I >= 0 && I `slt` L
11593 //
11594 // To prove L >= 0 we use isKnownNonNegative whereas to prove I >= 0 we use
11595 // isKnownPredicate. isKnownPredicate is more powerful, but also more
11596 // expensive; and using isKnownNonNegative(RHS) is sufficient for most of the
11597 // interesting cases seen in practice. We can consider "upgrading" L >= 0 to
11598 // use isKnownPredicate later if needed.
11599 return isKnownNonNegative(RHS) &&
11602}
11603
11604bool ScalarEvolution::isImpliedViaGuard(const BasicBlock *BB, CmpPredicate Pred,
11605 const SCEV *LHS, const SCEV *RHS) {
11606 // No need to even try if we know the module has no guards.
11607 if (!HasGuards)
11608 return false;
11609
11610 return any_of(*BB, [&](const Instruction &I) {
11611 using namespace llvm::PatternMatch;
11612
11613 Value *Condition;
11615 m_Value(Condition))) &&
11616 isImpliedCond(Pred, LHS, RHS, Condition, false);
11617 });
11618}
11619
11620/// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
11621/// protected by a conditional between LHS and RHS. This is used to
11622/// to eliminate casts.
11624 CmpPredicate Pred,
11625 const SCEV *LHS,
11626 const SCEV *RHS) {
11627 // Interpret a null as meaning no loop, where there is obviously no guard
11628 // (interprocedural conditions notwithstanding). Do not bother about
11629 // unreachable loops.
11630 if (!L || !DT.isReachableFromEntry(L->getHeader()))
11631 return true;
11632
11633 if (VerifyIR)
11634 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()) &&
11635 "This cannot be done on broken IR!");
11636
11637
11638 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11639 return true;
11640
11641 BasicBlock *Latch = L->getLoopLatch();
11642 if (!Latch)
11643 return false;
11644
11645 BranchInst *LoopContinuePredicate =
11647 if (LoopContinuePredicate && LoopContinuePredicate->isConditional() &&
11648 isImpliedCond(Pred, LHS, RHS,
11649 LoopContinuePredicate->getCondition(),
11650 LoopContinuePredicate->getSuccessor(0) != L->getHeader()))
11651 return true;
11652
11653 // We don't want more than one activation of the following loops on the stack
11654 // -- that can lead to O(n!) time complexity.
11655 if (WalkingBEDominatingConds)
11656 return false;
11657
11658 SaveAndRestore ClearOnExit(WalkingBEDominatingConds, true);
11659
11660 // See if we can exploit a trip count to prove the predicate.
11661 const auto &BETakenInfo = getBackedgeTakenInfo(L);
11662 const SCEV *LatchBECount = BETakenInfo.getExact(Latch, this);
11663 if (LatchBECount != getCouldNotCompute()) {
11664 // We know that Latch branches back to the loop header exactly
11665 // LatchBECount times. This means the backdege condition at Latch is
11666 // equivalent to "{0,+,1} u< LatchBECount".
11667 Type *Ty = LatchBECount->getType();
11668 auto NoWrapFlags = SCEV::NoWrapFlags(SCEV::FlagNUW | SCEV::FlagNW);
11669 const SCEV *LoopCounter =
11670 getAddRecExpr(getZero(Ty), getOne(Ty), L, NoWrapFlags);
11671 if (isImpliedCond(Pred, LHS, RHS, ICmpInst::ICMP_ULT, LoopCounter,
11672 LatchBECount))
11673 return true;
11674 }
11675
11676 // Check conditions due to any @llvm.assume intrinsics.
11677 for (auto &AssumeVH : AC.assumptions()) {
11678 if (!AssumeVH)
11679 continue;
11680 auto *CI = cast<CallInst>(AssumeVH);
11681 if (!DT.dominates(CI, Latch->getTerminator()))
11682 continue;
11683
11684 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
11685 return true;
11686 }
11687
11688 if (isImpliedViaGuard(Latch, Pred, LHS, RHS))
11689 return true;
11690
11691 for (DomTreeNode *DTN = DT[Latch], *HeaderDTN = DT[L->getHeader()];
11692 DTN != HeaderDTN; DTN = DTN->getIDom()) {
11693 assert(DTN && "should reach the loop header before reaching the root!");
11694
11695 BasicBlock *BB = DTN->getBlock();
11696 if (isImpliedViaGuard(BB, Pred, LHS, RHS))
11697 return true;
11698
11699 BasicBlock *PBB = BB->getSinglePredecessor();
11700 if (!PBB)
11701 continue;
11702
11703 BranchInst *ContinuePredicate = dyn_cast<BranchInst>(PBB->getTerminator());
11704 if (!ContinuePredicate || !ContinuePredicate->isConditional())
11705 continue;
11706
11707 Value *Condition = ContinuePredicate->getCondition();
11708
11709 // If we have an edge `E` within the loop body that dominates the only
11710 // latch, the condition guarding `E` also guards the backedge. This
11711 // reasoning works only for loops with a single latch.
11712
11713 BasicBlockEdge DominatingEdge(PBB, BB);
11714 if (DominatingEdge.isSingleEdge()) {
11715 // We're constructively (and conservatively) enumerating edges within the
11716 // loop body that dominate the latch. The dominator tree better agree
11717 // with us on this:
11718 assert(DT.dominates(DominatingEdge, Latch) && "should be!");
11719
11720 if (isImpliedCond(Pred, LHS, RHS, Condition,
11721 BB != ContinuePredicate->getSuccessor(0)))
11722 return true;
11723 }
11724 }
11725
11726 return false;
11727}
11728
11730 CmpPredicate Pred,
11731 const SCEV *LHS,
11732 const SCEV *RHS) {
11733 // Do not bother proving facts for unreachable code.
11734 if (!DT.isReachableFromEntry(BB))
11735 return true;
11736 if (VerifyIR)
11737 assert(!verifyFunction(*BB->getParent(), &dbgs()) &&
11738 "This cannot be done on broken IR!");
11739
11740 // If we cannot prove strict comparison (e.g. a > b), maybe we can prove
11741 // the facts (a >= b && a != b) separately. A typical situation is when the
11742 // non-strict comparison is known from ranges and non-equality is known from
11743 // dominating predicates. If we are proving strict comparison, we always try
11744 // to prove non-equality and non-strict comparison separately.
11745 CmpPredicate NonStrictPredicate = ICmpInst::getNonStrictCmpPredicate(Pred);
11746 const bool ProvingStrictComparison =
11747 Pred != NonStrictPredicate.dropSameSign();
11748 bool ProvedNonStrictComparison = false;
11749 bool ProvedNonEquality = false;
11750
11751 auto SplitAndProve = [&](std::function<bool(CmpPredicate)> Fn) -> bool {
11752 if (!ProvedNonStrictComparison)
11753 ProvedNonStrictComparison = Fn(NonStrictPredicate);
11754 if (!ProvedNonEquality)
11755 ProvedNonEquality = Fn(ICmpInst::ICMP_NE);
11756 if (ProvedNonStrictComparison && ProvedNonEquality)
11757 return true;
11758 return false;
11759 };
11760
11761 if (ProvingStrictComparison) {
11762 auto ProofFn = [&](CmpPredicate P) {
11763 return isKnownViaNonRecursiveReasoning(P, LHS, RHS);
11764 };
11765 if (SplitAndProve(ProofFn))
11766 return true;
11767 }
11768
11769 // Try to prove (Pred, LHS, RHS) using isImpliedCond.
11770 auto ProveViaCond = [&](const Value *Condition, bool Inverse) {
11771 const Instruction *CtxI = &BB->front();
11772 if (isImpliedCond(Pred, LHS, RHS, Condition, Inverse, CtxI))
11773 return true;
11774 if (ProvingStrictComparison) {
11775 auto ProofFn = [&](CmpPredicate P) {
11776 return isImpliedCond(P, LHS, RHS, Condition, Inverse, CtxI);
11777 };
11778 if (SplitAndProve(ProofFn))
11779 return true;
11780 }
11781 return false;
11782 };
11783
11784 // Starting at the block's predecessor, climb up the predecessor chain, as long
11785 // as there are predecessors that can be found that have unique successors
11786 // leading to the original block.
11787 const Loop *ContainingLoop = LI.getLoopFor(BB);
11788 const BasicBlock *PredBB;
11789 if (ContainingLoop && ContainingLoop->getHeader() == BB)
11790 PredBB = ContainingLoop->getLoopPredecessor();
11791 else
11792 PredBB = BB->getSinglePredecessor();
11793 for (std::pair<const BasicBlock *, const BasicBlock *> Pair(PredBB, BB);
11794 Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
11795 const BranchInst *BlockEntryPredicate =
11796 dyn_cast<BranchInst>(Pair.first->getTerminator());
11797 if (!BlockEntryPredicate || BlockEntryPredicate->isUnconditional())
11798 continue;
11799
11800 if (ProveViaCond(BlockEntryPredicate->getCondition(),
11801 BlockEntryPredicate->getSuccessor(0) != Pair.second))
11802 return true;
11803 }
11804
11805 // Check conditions due to any @llvm.assume intrinsics.
11806 for (auto &AssumeVH : AC.assumptions()) {
11807 if (!AssumeVH)
11808 continue;
11809 auto *CI = cast<CallInst>(AssumeVH);
11810 if (!DT.dominates(CI, BB))
11811 continue;
11812
11813 if (ProveViaCond(CI->getArgOperand(0), false))
11814 return true;
11815 }
11816
11817 // Check conditions due to any @llvm.experimental.guard intrinsics.
11818 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
11819 F.getParent(), Intrinsic::experimental_guard);
11820 if (GuardDecl)
11821 for (const auto *GU : GuardDecl->users())
11822 if (const auto *Guard = dyn_cast<IntrinsicInst>(GU))
11823 if (Guard->getFunction() == BB->getParent() && DT.dominates(Guard, BB))
11824 if (ProveViaCond(Guard->getArgOperand(0), false))
11825 return true;
11826 return false;
11827}
11828
11830 const SCEV *LHS,
11831 const SCEV *RHS) {
11832 // Interpret a null as meaning no loop, where there is obviously no guard
11833 // (interprocedural conditions notwithstanding).
11834 if (!L)
11835 return false;
11836
11837 // Both LHS and RHS must be available at loop entry.
11839 "LHS is not available at Loop Entry");
11841 "RHS is not available at Loop Entry");
11842
11843 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11844 return true;
11845
11846 return isBasicBlockEntryGuardedByCond(L->getHeader(), Pred, LHS, RHS);
11847}
11848
11849bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
11850 const SCEV *RHS,
11851 const Value *FoundCondValue, bool Inverse,
11852 const Instruction *CtxI) {
11853 // False conditions implies anything. Do not bother analyzing it further.
11854 if (FoundCondValue ==
11855 ConstantInt::getBool(FoundCondValue->getContext(), Inverse))
11856 return true;
11857
11858 if (!PendingLoopPredicates.insert(FoundCondValue).second)
11859 return false;
11860
11861 auto ClearOnExit =
11862 make_scope_exit([&]() { PendingLoopPredicates.erase(FoundCondValue); });
11863
11864 // Recursively handle And and Or conditions.
11865 const Value *Op0, *Op1;
11866 if (match(FoundCondValue, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) {
11867 if (!Inverse)
11868 return isImpliedCond(Pred, LHS, RHS, Op0, Inverse, CtxI) ||
11869 isImpliedCond(Pred, LHS, RHS, Op1, Inverse, CtxI);
11870 } else if (match(FoundCondValue, m_LogicalOr(m_Value(Op0), m_Value(Op1)))) {
11871 if (Inverse)
11872 return isImpliedCond(Pred, LHS, RHS, Op0, Inverse, CtxI) ||
11873 isImpliedCond(Pred, LHS, RHS, Op1, Inverse, CtxI);
11874 }
11875
11876 const ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue);
11877 if (!ICI) return false;
11878
11879 // Now that we found a conditional branch that dominates the loop or controls
11880 // the loop latch. Check to see if it is the comparison we are looking for.
11881 CmpPredicate FoundPred;
11882 if (Inverse)
11883 FoundPred = ICI->getInverseCmpPredicate();
11884 else
11885 FoundPred = ICI->getCmpPredicate();
11886
11887 const SCEV *FoundLHS = getSCEV(ICI->getOperand(0));
11888 const SCEV *FoundRHS = getSCEV(ICI->getOperand(1));
11889
11890 return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS, CtxI);
11891}
11892
11893bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
11894 const SCEV *RHS, CmpPredicate FoundPred,
11895 const SCEV *FoundLHS, const SCEV *FoundRHS,
11896 const Instruction *CtxI) {
11897 // Balance the types.
11898 if (getTypeSizeInBits(LHS->getType()) <
11899 getTypeSizeInBits(FoundLHS->getType())) {
11900 // For unsigned and equality predicates, try to prove that both found
11901 // operands fit into narrow unsigned range. If so, try to prove facts in
11902 // narrow types.
11903 if (!CmpInst::isSigned(FoundPred) && !FoundLHS->getType()->isPointerTy() &&
11904 !FoundRHS->getType()->isPointerTy()) {
11905 auto *NarrowType = LHS->getType();
11906 auto *WideType = FoundLHS->getType();
11907 auto BitWidth = getTypeSizeInBits(NarrowType);
11908 const SCEV *MaxValue = getZeroExtendExpr(
11910 if (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_ULE, FoundLHS,
11911 MaxValue) &&
11912 isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_ULE, FoundRHS,
11913 MaxValue)) {
11914 const SCEV *TruncFoundLHS = getTruncateExpr(FoundLHS, NarrowType);
11915 const SCEV *TruncFoundRHS = getTruncateExpr(FoundRHS, NarrowType);
11916 // We cannot preserve samesign after truncation.
11917 if (isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred.dropSameSign(),
11918 TruncFoundLHS, TruncFoundRHS, CtxI))
11919 return true;
11920 }
11921 }
11922
11923 if (LHS->getType()->isPointerTy() || RHS->getType()->isPointerTy())
11924 return false;
11925 if (CmpInst::isSigned(Pred)) {
11926 LHS = getSignExtendExpr(LHS, FoundLHS->getType());
11927 RHS = getSignExtendExpr(RHS, FoundLHS->getType());
11928 } else {
11929 LHS = getZeroExtendExpr(LHS, FoundLHS->getType());
11930 RHS = getZeroExtendExpr(RHS, FoundLHS->getType());
11931 }
11932 } else if (getTypeSizeInBits(LHS->getType()) >
11933 getTypeSizeInBits(FoundLHS->getType())) {
11934 if (FoundLHS->getType()->isPointerTy() || FoundRHS->getType()->isPointerTy())
11935 return false;
11936 if (CmpInst::isSigned(FoundPred)) {
11937 FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType());
11938 FoundRHS = getSignExtendExpr(FoundRHS, LHS->getType());
11939 } else {
11940 FoundLHS = getZeroExtendExpr(FoundLHS, LHS->getType());
11941 FoundRHS = getZeroExtendExpr(FoundRHS, LHS->getType());
11942 }
11943 }
11944 return isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred, FoundLHS,
11945 FoundRHS, CtxI);
11946}
11947
11948bool ScalarEvolution::isImpliedCondBalancedTypes(
11949 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, CmpPredicate FoundPred,
11950 const SCEV *FoundLHS, const SCEV *FoundRHS, const Instruction *CtxI) {
11952 getTypeSizeInBits(FoundLHS->getType()) &&
11953 "Types should be balanced!");
11954 // Canonicalize the query to match the way instcombine will have
11955 // canonicalized the comparison.
11956 if (SimplifyICmpOperands(Pred, LHS, RHS))
11957 if (LHS == RHS)
11958 return CmpInst::isTrueWhenEqual(Pred);
11959 if (SimplifyICmpOperands(FoundPred, FoundLHS, FoundRHS))
11960 if (FoundLHS == FoundRHS)
11961 return CmpInst::isFalseWhenEqual(FoundPred);
11962
11963 // Check to see if we can make the LHS or RHS match.
11964 if (LHS == FoundRHS || RHS == FoundLHS) {
11965 if (isa<SCEVConstant>(RHS)) {
11966 std::swap(FoundLHS, FoundRHS);
11967 FoundPred = ICmpInst::getSwappedCmpPredicate(FoundPred);
11968 } else {
11969 std::swap(LHS, RHS);
11971 }
11972 }
11973
11974 // Check whether the found predicate is the same as the desired predicate.
11975 if (auto P = CmpPredicate::getMatching(FoundPred, Pred))
11976 return isImpliedCondOperands(*P, LHS, RHS, FoundLHS, FoundRHS, CtxI);
11977
11978 // Check whether swapping the found predicate makes it the same as the
11979 // desired predicate.
11980 if (auto P = CmpPredicate::getMatching(
11981 ICmpInst::getSwappedCmpPredicate(FoundPred), Pred)) {
11982 // We can write the implication
11983 // 0. LHS Pred RHS <- FoundLHS SwapPred FoundRHS
11984 // using one of the following ways:
11985 // 1. LHS Pred RHS <- FoundRHS Pred FoundLHS
11986 // 2. RHS SwapPred LHS <- FoundLHS SwapPred FoundRHS
11987 // 3. LHS Pred RHS <- ~FoundLHS Pred ~FoundRHS
11988 // 4. ~LHS SwapPred ~RHS <- FoundLHS SwapPred FoundRHS
11989 // Forms 1. and 2. require swapping the operands of one condition. Don't
11990 // do this if it would break canonical constant/addrec ordering.
11992 return isImpliedCondOperands(ICmpInst::getSwappedCmpPredicate(*P), RHS,
11993 LHS, FoundLHS, FoundRHS, CtxI);
11994 if (!isa<SCEVConstant>(FoundRHS) && !isa<SCEVAddRecExpr>(FoundLHS))
11995 return isImpliedCondOperands(*P, LHS, RHS, FoundRHS, FoundLHS, CtxI);
11996
11997 // There's no clear preference between forms 3. and 4., try both. Avoid
11998 // forming getNotSCEV of pointer values as the resulting subtract is
11999 // not legal.
12000 if (!LHS->getType()->isPointerTy() && !RHS->getType()->isPointerTy() &&
12001 isImpliedCondOperands(ICmpInst::getSwappedCmpPredicate(*P),
12002 getNotSCEV(LHS), getNotSCEV(RHS), FoundLHS,
12003 FoundRHS, CtxI))
12004 return true;
12005
12006 if (!FoundLHS->getType()->isPointerTy() &&
12007 !FoundRHS->getType()->isPointerTy() &&
12008 isImpliedCondOperands(*P, LHS, RHS, getNotSCEV(FoundLHS),
12009 getNotSCEV(FoundRHS), CtxI))
12010 return true;
12011
12012 return false;
12013 }
12014
12015 auto IsSignFlippedPredicate = [](CmpInst::Predicate P1,
12016 CmpInst::Predicate P2) {
12017 assert(P1 != P2 && "Handled earlier!");
12018 return CmpInst::isRelational(P2) &&
12020 };
12021 if (IsSignFlippedPredicate(Pred, FoundPred)) {
12022 // Unsigned comparison is the same as signed comparison when both the
12023 // operands are non-negative or negative.
12024 if ((isKnownNonNegative(FoundLHS) && isKnownNonNegative(FoundRHS)) ||
12025 (isKnownNegative(FoundLHS) && isKnownNegative(FoundRHS)))
12026 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, CtxI);
12027 // Create local copies that we can freely swap and canonicalize our
12028 // conditions to "le/lt".
12029 CmpPredicate CanonicalPred = Pred, CanonicalFoundPred = FoundPred;
12030 const SCEV *CanonicalLHS = LHS, *CanonicalRHS = RHS,
12031 *CanonicalFoundLHS = FoundLHS, *CanonicalFoundRHS = FoundRHS;
12032 if (ICmpInst::isGT(CanonicalPred) || ICmpInst::isGE(CanonicalPred)) {
12033 CanonicalPred = ICmpInst::getSwappedCmpPredicate(CanonicalPred);
12034 CanonicalFoundPred = ICmpInst::getSwappedCmpPredicate(CanonicalFoundPred);
12035 std::swap(CanonicalLHS, CanonicalRHS);
12036 std::swap(CanonicalFoundLHS, CanonicalFoundRHS);
12037 }
12038 assert((ICmpInst::isLT(CanonicalPred) || ICmpInst::isLE(CanonicalPred)) &&
12039 "Must be!");
12040 assert((ICmpInst::isLT(CanonicalFoundPred) ||
12041 ICmpInst::isLE(CanonicalFoundPred)) &&
12042 "Must be!");
12043 if (ICmpInst::isSigned(CanonicalPred) && isKnownNonNegative(CanonicalRHS))
12044 // Use implication:
12045 // x <u y && y >=s 0 --> x <s y.
12046 // If we can prove the left part, the right part is also proven.
12047 return isImpliedCondOperands(CanonicalFoundPred, CanonicalLHS,
12048 CanonicalRHS, CanonicalFoundLHS,
12049 CanonicalFoundRHS);
12050 if (ICmpInst::isUnsigned(CanonicalPred) && isKnownNegative(CanonicalRHS))
12051 // Use implication:
12052 // x <s y && y <s 0 --> x <u y.
12053 // If we can prove the left part, the right part is also proven.
12054 return isImpliedCondOperands(CanonicalFoundPred, CanonicalLHS,
12055 CanonicalRHS, CanonicalFoundLHS,
12056 CanonicalFoundRHS);
12057 }
12058
12059 // Check if we can make progress by sharpening ranges.
12060 if (FoundPred == ICmpInst::ICMP_NE &&
12061 (isa<SCEVConstant>(FoundLHS) || isa<SCEVConstant>(FoundRHS))) {
12062
12063 const SCEVConstant *C = nullptr;
12064 const SCEV *V = nullptr;
12065
12066 if (isa<SCEVConstant>(FoundLHS)) {
12067 C = cast<SCEVConstant>(FoundLHS);
12068 V = FoundRHS;
12069 } else {
12070 C = cast<SCEVConstant>(FoundRHS);
12071 V = FoundLHS;
12072 }
12073
12074 // The guarding predicate tells us that C != V. If the known range
12075 // of V is [C, t), we can sharpen the range to [C + 1, t). The
12076 // range we consider has to correspond to same signedness as the
12077 // predicate we're interested in folding.
12078
12079 APInt Min = ICmpInst::isSigned(Pred) ?
12081
12082 if (Min == C->getAPInt()) {
12083 // Given (V >= Min && V != Min) we conclude V >= (Min + 1).
12084 // This is true even if (Min + 1) wraps around -- in case of
12085 // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)).
12086
12087 APInt SharperMin = Min + 1;
12088
12089 switch (Pred) {
12090 case ICmpInst::ICMP_SGE:
12091 case ICmpInst::ICMP_UGE:
12092 // We know V `Pred` SharperMin. If this implies LHS `Pred`
12093 // RHS, we're done.
12094 if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(SharperMin),
12095 CtxI))
12096 return true;
12097 [[fallthrough]];
12098
12099 case ICmpInst::ICMP_SGT:
12100 case ICmpInst::ICMP_UGT:
12101 // We know from the range information that (V `Pred` Min ||
12102 // V == Min). We know from the guarding condition that !(V
12103 // == Min). This gives us
12104 //
12105 // V `Pred` Min || V == Min && !(V == Min)
12106 // => V `Pred` Min
12107 //
12108 // If V `Pred` Min implies LHS `Pred` RHS, we're done.
12109
12110 if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(Min), CtxI))
12111 return true;
12112 break;
12113
12114 // `LHS < RHS` and `LHS <= RHS` are handled in the same way as `RHS > LHS` and `RHS >= LHS` respectively.
12115 case ICmpInst::ICMP_SLE:
12116 case ICmpInst::ICMP_ULE:
12117 if (isImpliedCondOperands(ICmpInst::getSwappedCmpPredicate(Pred), RHS,
12118 LHS, V, getConstant(SharperMin), CtxI))
12119 return true;
12120 [[fallthrough]];
12121
12122 case ICmpInst::ICMP_SLT:
12123 case ICmpInst::ICMP_ULT:
12124 if (isImpliedCondOperands(ICmpInst::getSwappedCmpPredicate(Pred), RHS,
12125 LHS, V, getConstant(Min), CtxI))
12126 return true;
12127 break;
12128
12129 default:
12130 // No change
12131 break;
12132 }
12133 }
12134 }
12135
12136 // Check whether the actual condition is beyond sufficient.
12137 if (FoundPred == ICmpInst::ICMP_EQ)
12138 if (ICmpInst::isTrueWhenEqual(Pred))
12139 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, CtxI))
12140 return true;
12141 if (Pred == ICmpInst::ICMP_NE)
12142 if (!ICmpInst::isTrueWhenEqual(FoundPred))
12143 if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS, CtxI))
12144 return true;
12145
12146 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS))
12147 return true;
12148
12149 // Otherwise assume the worst.
12150 return false;
12151}
12152
12153bool ScalarEvolution::splitBinaryAdd(const SCEV *Expr,
12154 const SCEV *&L, const SCEV *&R,
12155 SCEV::NoWrapFlags &Flags) {
12156 if (!match(Expr, m_scev_Add(m_SCEV(L), m_SCEV(R))))
12157 return false;
12158
12159 Flags = cast<SCEVAddExpr>(Expr)->getNoWrapFlags();
12160 return true;
12161}
12162
12163std::optional<APInt>
12165 // We avoid subtracting expressions here because this function is usually
12166 // fairly deep in the call stack (i.e. is called many times).
12167
12168 unsigned BW = getTypeSizeInBits(More->getType());
12169 APInt Diff(BW, 0);
12170 APInt DiffMul(BW, 1);
12171 // Try various simplifications to reduce the difference to a constant. Limit
12172 // the number of allowed simplifications to keep compile-time low.
12173 for (unsigned I = 0; I < 8; ++I) {
12174 if (More == Less)
12175 return Diff;
12176
12177 // Reduce addrecs with identical steps to their start value.
12179 const auto *LAR = cast<SCEVAddRecExpr>(Less);
12180 const auto *MAR = cast<SCEVAddRecExpr>(More);
12181
12182 if (LAR->getLoop() != MAR->getLoop())
12183 return std::nullopt;
12184
12185 // We look at affine expressions only; not for correctness but to keep
12186 // getStepRecurrence cheap.
12187 if (!LAR->isAffine() || !MAR->isAffine())
12188 return std::nullopt;
12189
12190 if (LAR->getStepRecurrence(*this) != MAR->getStepRecurrence(*this))
12191 return std::nullopt;
12192
12193 Less = LAR->getStart();
12194 More = MAR->getStart();
12195 continue;
12196 }
12197
12198 // Try to match a common constant multiply.
12199 auto MatchConstMul =
12200 [](const SCEV *S) -> std::optional<std::pair<const SCEV *, APInt>> {
12201 const APInt *C;
12202 const SCEV *Op;
12203 if (match(S, m_scev_Mul(m_scev_APInt(C), m_SCEV(Op))))
12204 return {{Op, *C}};
12205 return std::nullopt;
12206 };
12207 if (auto MatchedMore = MatchConstMul(More)) {
12208 if (auto MatchedLess = MatchConstMul(Less)) {
12209 if (MatchedMore->second == MatchedLess->second) {
12210 More = MatchedMore->first;
12211 Less = MatchedLess->first;
12212 DiffMul *= MatchedMore->second;
12213 continue;
12214 }
12215 }
12216 }
12217
12218 // Try to cancel out common factors in two add expressions.
12220 auto Add = [&](const SCEV *S, int Mul) {
12221 if (auto *C = dyn_cast<SCEVConstant>(S)) {
12222 if (Mul == 1) {
12223 Diff += C->getAPInt() * DiffMul;
12224 } else {
12225 assert(Mul == -1);
12226 Diff -= C->getAPInt() * DiffMul;
12227 }
12228 } else
12229 Multiplicity[S] += Mul;
12230 };
12231 auto Decompose = [&](const SCEV *S, int Mul) {
12232 if (isa<SCEVAddExpr>(S)) {
12233 for (const SCEV *Op : S->operands())
12234 Add(Op, Mul);
12235 } else
12236 Add(S, Mul);
12237 };
12238 Decompose(More, 1);
12239 Decompose(Less, -1);
12240
12241 // Check whether all the non-constants cancel out, or reduce to new
12242 // More/Less values.
12243 const SCEV *NewMore = nullptr, *NewLess = nullptr;
12244 for (const auto &[S, Mul] : Multiplicity) {
12245 if (Mul == 0)
12246 continue;
12247 if (Mul == 1) {
12248 if (NewMore)
12249 return std::nullopt;
12250 NewMore = S;
12251 } else if (Mul == -1) {
12252 if (NewLess)
12253 return std::nullopt;
12254 NewLess = S;
12255 } else
12256 return std::nullopt;
12257 }
12258
12259 // Values stayed the same, no point in trying further.
12260 if (NewMore == More || NewLess == Less)
12261 return std::nullopt;
12262
12263 More = NewMore;
12264 Less = NewLess;
12265
12266 // Reduced to constant.
12267 if (!More && !Less)
12268 return Diff;
12269
12270 // Left with variable on only one side, bail out.
12271 if (!More || !Less)
12272 return std::nullopt;
12273 }
12274
12275 // Did not reduce to constant.
12276 return std::nullopt;
12277}
12278
12279bool ScalarEvolution::isImpliedCondOperandsViaAddRecStart(
12280 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12281 const SCEV *FoundRHS, const Instruction *CtxI) {
12282 // Try to recognize the following pattern:
12283 //
12284 // FoundRHS = ...
12285 // ...
12286 // loop:
12287 // FoundLHS = {Start,+,W}
12288 // context_bb: // Basic block from the same loop
12289 // known(Pred, FoundLHS, FoundRHS)
12290 //
12291 // If some predicate is known in the context of a loop, it is also known on
12292 // each iteration of this loop, including the first iteration. Therefore, in
12293 // this case, `FoundLHS Pred FoundRHS` implies `Start Pred FoundRHS`. Try to
12294 // prove the original pred using this fact.
12295 if (!CtxI)
12296 return false;
12297 const BasicBlock *ContextBB = CtxI->getParent();
12298 // Make sure AR varies in the context block.
12299 if (auto *AR = dyn_cast<SCEVAddRecExpr>(FoundLHS)) {
12300 const Loop *L = AR->getLoop();
12301 // Make sure that context belongs to the loop and executes on 1st iteration
12302 // (if it ever executes at all).
12303 if (!L->contains(ContextBB) || !DT.dominates(ContextBB, L->getLoopLatch()))
12304 return false;
12305 if (!isAvailableAtLoopEntry(FoundRHS, AR->getLoop()))
12306 return false;
12307 return isImpliedCondOperands(Pred, LHS, RHS, AR->getStart(), FoundRHS);
12308 }
12309
12310 if (auto *AR = dyn_cast<SCEVAddRecExpr>(FoundRHS)) {
12311 const Loop *L = AR->getLoop();
12312 // Make sure that context belongs to the loop and executes on 1st iteration
12313 // (if it ever executes at all).
12314 if (!L->contains(ContextBB) || !DT.dominates(ContextBB, L->getLoopLatch()))
12315 return false;
12316 if (!isAvailableAtLoopEntry(FoundLHS, AR->getLoop()))
12317 return false;
12318 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, AR->getStart());
12319 }
12320
12321 return false;
12322}
12323
12324bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow(CmpPredicate Pred,
12325 const SCEV *LHS,
12326 const SCEV *RHS,
12327 const SCEV *FoundLHS,
12328 const SCEV *FoundRHS) {
12329 if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT)
12330 return false;
12331
12332 const auto *AddRecLHS = dyn_cast<SCEVAddRecExpr>(LHS);
12333 if (!AddRecLHS)
12334 return false;
12335
12336 const auto *AddRecFoundLHS = dyn_cast<SCEVAddRecExpr>(FoundLHS);
12337 if (!AddRecFoundLHS)
12338 return false;
12339
12340 // We'd like to let SCEV reason about control dependencies, so we constrain
12341 // both the inequalities to be about add recurrences on the same loop. This
12342 // way we can use isLoopEntryGuardedByCond later.
12343
12344 const Loop *L = AddRecFoundLHS->getLoop();
12345 if (L != AddRecLHS->getLoop())
12346 return false;
12347
12348 // FoundLHS u< FoundRHS u< -C => (FoundLHS + C) u< (FoundRHS + C) ... (1)
12349 //
12350 // FoundLHS s< FoundRHS s< INT_MIN - C => (FoundLHS + C) s< (FoundRHS + C)
12351 // ... (2)
12352 //
12353 // Informal proof for (2), assuming (1) [*]:
12354 //
12355 // We'll also assume (A s< B) <=> ((A + INT_MIN) u< (B + INT_MIN)) ... (3)[**]
12356 //
12357 // Then
12358 //
12359 // FoundLHS s< FoundRHS s< INT_MIN - C
12360 // <=> (FoundLHS + INT_MIN) u< (FoundRHS + INT_MIN) u< -C [ using (3) ]
12361 // <=> (FoundLHS + INT_MIN + C) u< (FoundRHS + INT_MIN + C) [ using (1) ]
12362 // <=> (FoundLHS + INT_MIN + C + INT_MIN) s<
12363 // (FoundRHS + INT_MIN + C + INT_MIN) [ using (3) ]
12364 // <=> FoundLHS + C s< FoundRHS + C
12365 //
12366 // [*]: (1) can be proved by ruling out overflow.
12367 //
12368 // [**]: This can be proved by analyzing all the four possibilities:
12369 // (A s< 0, B s< 0), (A s< 0, B s>= 0), (A s>= 0, B s< 0) and
12370 // (A s>= 0, B s>= 0).
12371 //
12372 // Note:
12373 // Despite (2), "FoundRHS s< INT_MIN - C" does not mean that "FoundRHS + C"
12374 // will not sign underflow. For instance, say FoundLHS = (i8 -128), FoundRHS
12375 // = (i8 -127) and C = (i8 -100). Then INT_MIN - C = (i8 -28), and FoundRHS
12376 // s< (INT_MIN - C). Lack of sign overflow / underflow in "FoundRHS + C" is
12377 // neither necessary nor sufficient to prove "(FoundLHS + C) s< (FoundRHS +
12378 // C)".
12379
12380 std::optional<APInt> LDiff = computeConstantDifference(LHS, FoundLHS);
12381 if (!LDiff)
12382 return false;
12383 std::optional<APInt> RDiff = computeConstantDifference(RHS, FoundRHS);
12384 if (!RDiff || *LDiff != *RDiff)
12385 return false;
12386
12387 if (LDiff->isMinValue())
12388 return true;
12389
12390 APInt FoundRHSLimit;
12391
12392 if (Pred == CmpInst::ICMP_ULT) {
12393 FoundRHSLimit = -(*RDiff);
12394 } else {
12395 assert(Pred == CmpInst::ICMP_SLT && "Checked above!");
12396 FoundRHSLimit = APInt::getSignedMinValue(getTypeSizeInBits(RHS->getType())) - *RDiff;
12397 }
12398
12399 // Try to prove (1) or (2), as needed.
12400 return isAvailableAtLoopEntry(FoundRHS, L) &&
12401 isLoopEntryGuardedByCond(L, Pred, FoundRHS,
12402 getConstant(FoundRHSLimit));
12403}
12404
12405bool ScalarEvolution::isImpliedViaMerge(CmpPredicate Pred, const SCEV *LHS,
12406 const SCEV *RHS, const SCEV *FoundLHS,
12407 const SCEV *FoundRHS, unsigned Depth) {
12408 const PHINode *LPhi = nullptr, *RPhi = nullptr;
12409
12410 auto ClearOnExit = make_scope_exit([&]() {
12411 if (LPhi) {
12412 bool Erased = PendingMerges.erase(LPhi);
12413 assert(Erased && "Failed to erase LPhi!");
12414 (void)Erased;
12415 }
12416 if (RPhi) {
12417 bool Erased = PendingMerges.erase(RPhi);
12418 assert(Erased && "Failed to erase RPhi!");
12419 (void)Erased;
12420 }
12421 });
12422
12423 // Find respective Phis and check that they are not being pending.
12424 if (const SCEVUnknown *LU = dyn_cast<SCEVUnknown>(LHS))
12425 if (auto *Phi = dyn_cast<PHINode>(LU->getValue())) {
12426 if (!PendingMerges.insert(Phi).second)
12427 return false;
12428 LPhi = Phi;
12429 }
12430 if (const SCEVUnknown *RU = dyn_cast<SCEVUnknown>(RHS))
12431 if (auto *Phi = dyn_cast<PHINode>(RU->getValue())) {
12432 // If we detect a loop of Phi nodes being processed by this method, for
12433 // example:
12434 //
12435 // %a = phi i32 [ %some1, %preheader ], [ %b, %latch ]
12436 // %b = phi i32 [ %some2, %preheader ], [ %a, %latch ]
12437 //
12438 // we don't want to deal with a case that complex, so return conservative
12439 // answer false.
12440 if (!PendingMerges.insert(Phi).second)
12441 return false;
12442 RPhi = Phi;
12443 }
12444
12445 // If none of LHS, RHS is a Phi, nothing to do here.
12446 if (!LPhi && !RPhi)
12447 return false;
12448
12449 // If there is a SCEVUnknown Phi we are interested in, make it left.
12450 if (!LPhi) {
12451 std::swap(LHS, RHS);
12452 std::swap(FoundLHS, FoundRHS);
12453 std::swap(LPhi, RPhi);
12455 }
12456
12457 assert(LPhi && "LPhi should definitely be a SCEVUnknown Phi!");
12458 const BasicBlock *LBB = LPhi->getParent();
12459 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(RHS);
12460
12461 auto ProvedEasily = [&](const SCEV *S1, const SCEV *S2) {
12462 return isKnownViaNonRecursiveReasoning(Pred, S1, S2) ||
12463 isImpliedCondOperandsViaRanges(Pred, S1, S2, Pred, FoundLHS, FoundRHS) ||
12464 isImpliedViaOperations(Pred, S1, S2, FoundLHS, FoundRHS, Depth);
12465 };
12466
12467 if (RPhi && RPhi->getParent() == LBB) {
12468 // Case one: RHS is also a SCEVUnknown Phi from the same basic block.
12469 // If we compare two Phis from the same block, and for each entry block
12470 // the predicate is true for incoming values from this block, then the
12471 // predicate is also true for the Phis.
12472 for (const BasicBlock *IncBB : predecessors(LBB)) {
12473 const SCEV *L = getSCEV(LPhi->getIncomingValueForBlock(IncBB));
12474 const SCEV *R = getSCEV(RPhi->getIncomingValueForBlock(IncBB));
12475 if (!ProvedEasily(L, R))
12476 return false;
12477 }
12478 } else if (RAR && RAR->getLoop()->getHeader() == LBB) {
12479 // Case two: RHS is also a Phi from the same basic block, and it is an
12480 // AddRec. It means that there is a loop which has both AddRec and Unknown
12481 // PHIs, for it we can compare incoming values of AddRec from above the loop
12482 // and latch with their respective incoming values of LPhi.
12483 // TODO: Generalize to handle loops with many inputs in a header.
12484 if (LPhi->getNumIncomingValues() != 2) return false;
12485
12486 auto *RLoop = RAR->getLoop();
12487 auto *Predecessor = RLoop->getLoopPredecessor();
12488 assert(Predecessor && "Loop with AddRec with no predecessor?");
12489 const SCEV *L1 = getSCEV(LPhi->getIncomingValueForBlock(Predecessor));
12490 if (!ProvedEasily(L1, RAR->getStart()))
12491 return false;
12492 auto *Latch = RLoop->getLoopLatch();
12493 assert(Latch && "Loop with AddRec with no latch?");
12494 const SCEV *L2 = getSCEV(LPhi->getIncomingValueForBlock(Latch));
12495 if (!ProvedEasily(L2, RAR->getPostIncExpr(*this)))
12496 return false;
12497 } else {
12498 // In all other cases go over inputs of LHS and compare each of them to RHS,
12499 // the predicate is true for (LHS, RHS) if it is true for all such pairs.
12500 // At this point RHS is either a non-Phi, or it is a Phi from some block
12501 // different from LBB.
12502 for (const BasicBlock *IncBB : predecessors(LBB)) {
12503 // Check that RHS is available in this block.
12504 if (!dominates(RHS, IncBB))
12505 return false;
12506 const SCEV *L = getSCEV(LPhi->getIncomingValueForBlock(IncBB));
12507 // Make sure L does not refer to a value from a potentially previous
12508 // iteration of a loop.
12509 if (!properlyDominates(L, LBB))
12510 return false;
12511 // Addrecs are considered to properly dominate their loop, so are missed
12512 // by the previous check. Discard any values that have computable
12513 // evolution in this loop.
12514 if (auto *Loop = LI.getLoopFor(LBB))
12515 if (hasComputableLoopEvolution(L, Loop))
12516 return false;
12517 if (!ProvedEasily(L, RHS))
12518 return false;
12519 }
12520 }
12521 return true;
12522}
12523
12524bool ScalarEvolution::isImpliedCondOperandsViaShift(CmpPredicate Pred,
12525 const SCEV *LHS,
12526 const SCEV *RHS,
12527 const SCEV *FoundLHS,
12528 const SCEV *FoundRHS) {
12529 // We want to imply LHS < RHS from LHS < (RHS >> shiftvalue). First, make
12530 // sure that we are dealing with same LHS.
12531 if (RHS == FoundRHS) {
12532 std::swap(LHS, RHS);
12533 std::swap(FoundLHS, FoundRHS);
12535 }
12536 if (LHS != FoundLHS)
12537 return false;
12538
12539 auto *SUFoundRHS = dyn_cast<SCEVUnknown>(FoundRHS);
12540 if (!SUFoundRHS)
12541 return false;
12542
12543 Value *Shiftee, *ShiftValue;
12544
12545 using namespace PatternMatch;
12546 if (match(SUFoundRHS->getValue(),
12547 m_LShr(m_Value(Shiftee), m_Value(ShiftValue)))) {
12548 auto *ShifteeS = getSCEV(Shiftee);
12549 // Prove one of the following:
12550 // LHS <u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <u RHS
12551 // LHS <=u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <=u RHS
12552 // LHS <s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12553 // ---> LHS <s RHS
12554 // LHS <=s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12555 // ---> LHS <=s RHS
12556 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
12557 return isKnownPredicate(ICmpInst::ICMP_ULE, ShifteeS, RHS);
12558 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
12559 if (isKnownNonNegative(ShifteeS))
12560 return isKnownPredicate(ICmpInst::ICMP_SLE, ShifteeS, RHS);
12561 }
12562
12563 return false;
12564}
12565
12566bool ScalarEvolution::isImpliedCondOperands(CmpPredicate Pred, const SCEV *LHS,
12567 const SCEV *RHS,
12568 const SCEV *FoundLHS,
12569 const SCEV *FoundRHS,
12570 const Instruction *CtxI) {
12571 return isImpliedCondOperandsViaRanges(Pred, LHS, RHS, Pred, FoundLHS,
12572 FoundRHS) ||
12573 isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS,
12574 FoundRHS) ||
12575 isImpliedCondOperandsViaShift(Pred, LHS, RHS, FoundLHS, FoundRHS) ||
12576 isImpliedCondOperandsViaAddRecStart(Pred, LHS, RHS, FoundLHS, FoundRHS,
12577 CtxI) ||
12578 isImpliedCondOperandsHelper(Pred, LHS, RHS, FoundLHS, FoundRHS);
12579}
12580
12581/// Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
12582template <typename MinMaxExprType>
12583static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr,
12584 const SCEV *Candidate) {
12585 const MinMaxExprType *MinMaxExpr = dyn_cast<MinMaxExprType>(MaybeMinMaxExpr);
12586 if (!MinMaxExpr)
12587 return false;
12588
12589 return is_contained(MinMaxExpr->operands(), Candidate);
12590}
12591
12593 CmpPredicate Pred, const SCEV *LHS,
12594 const SCEV *RHS) {
12595 // If both sides are affine addrecs for the same loop, with equal
12596 // steps, and we know the recurrences don't wrap, then we only
12597 // need to check the predicate on the starting values.
12598
12599 if (!ICmpInst::isRelational(Pred))
12600 return false;
12601
12602 const SCEV *LStart, *RStart, *Step;
12603 const Loop *L;
12604 if (!match(LHS,
12605 m_scev_AffineAddRec(m_SCEV(LStart), m_SCEV(Step), m_Loop(L))) ||
12607 m_SpecificLoop(L))))
12608 return false;
12613 if (!LAR->getNoWrapFlags(NW) || !RAR->getNoWrapFlags(NW))
12614 return false;
12615
12616 return SE.isKnownPredicate(Pred, LStart, RStart);
12617}
12618
12619/// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max
12620/// expression?
12622 const SCEV *LHS, const SCEV *RHS) {
12623 switch (Pred) {
12624 default:
12625 return false;
12626
12627 case ICmpInst::ICMP_SGE:
12628 std::swap(LHS, RHS);
12629 [[fallthrough]];
12630 case ICmpInst::ICMP_SLE:
12631 return
12632 // min(A, ...) <= A
12634 // A <= max(A, ...)
12636
12637 case ICmpInst::ICMP_UGE:
12638 std::swap(LHS, RHS);
12639 [[fallthrough]];
12640 case ICmpInst::ICMP_ULE:
12641 return
12642 // min(A, ...) <= A
12643 // FIXME: what about umin_seq?
12645 // A <= max(A, ...)
12647 }
12648
12649 llvm_unreachable("covered switch fell through?!");
12650}
12651
12652bool ScalarEvolution::isImpliedViaOperations(CmpPredicate Pred, const SCEV *LHS,
12653 const SCEV *RHS,
12654 const SCEV *FoundLHS,
12655 const SCEV *FoundRHS,
12656 unsigned Depth) {
12659 "LHS and RHS have different sizes?");
12660 assert(getTypeSizeInBits(FoundLHS->getType()) ==
12661 getTypeSizeInBits(FoundRHS->getType()) &&
12662 "FoundLHS and FoundRHS have different sizes?");
12663 // We want to avoid hurting the compile time with analysis of too big trees.
12665 return false;
12666
12667 // We only want to work with GT comparison so far.
12668 if (ICmpInst::isLT(Pred)) {
12670 std::swap(LHS, RHS);
12671 std::swap(FoundLHS, FoundRHS);
12672 }
12673
12675
12676 // For unsigned, try to reduce it to corresponding signed comparison.
12677 if (P == ICmpInst::ICMP_UGT)
12678 // We can replace unsigned predicate with its signed counterpart if all
12679 // involved values are non-negative.
12680 // TODO: We could have better support for unsigned.
12681 if (isKnownNonNegative(FoundLHS) && isKnownNonNegative(FoundRHS)) {
12682 // Knowing that both FoundLHS and FoundRHS are non-negative, and knowing
12683 // FoundLHS >u FoundRHS, we also know that FoundLHS >s FoundRHS. Let us
12684 // use this fact to prove that LHS and RHS are non-negative.
12685 const SCEV *MinusOne = getMinusOne(LHS->getType());
12686 if (isImpliedCondOperands(ICmpInst::ICMP_SGT, LHS, MinusOne, FoundLHS,
12687 FoundRHS) &&
12688 isImpliedCondOperands(ICmpInst::ICMP_SGT, RHS, MinusOne, FoundLHS,
12689 FoundRHS))
12691 }
12692
12693 if (P != ICmpInst::ICMP_SGT)
12694 return false;
12695
12696 auto GetOpFromSExt = [&](const SCEV *S) {
12697 if (auto *Ext = dyn_cast<SCEVSignExtendExpr>(S))
12698 return Ext->getOperand();
12699 // TODO: If S is a SCEVConstant then you can cheaply "strip" the sext off
12700 // the constant in some cases.
12701 return S;
12702 };
12703
12704 // Acquire values from extensions.
12705 auto *OrigLHS = LHS;
12706 auto *OrigFoundLHS = FoundLHS;
12707 LHS = GetOpFromSExt(LHS);
12708 FoundLHS = GetOpFromSExt(FoundLHS);
12709
12710 // Is the SGT predicate can be proved trivially or using the found context.
12711 auto IsSGTViaContext = [&](const SCEV *S1, const SCEV *S2) {
12712 return isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_SGT, S1, S2) ||
12713 isImpliedViaOperations(ICmpInst::ICMP_SGT, S1, S2, OrigFoundLHS,
12714 FoundRHS, Depth + 1);
12715 };
12716
12717 if (auto *LHSAddExpr = dyn_cast<SCEVAddExpr>(LHS)) {
12718 // We want to avoid creation of any new non-constant SCEV. Since we are
12719 // going to compare the operands to RHS, we should be certain that we don't
12720 // need any size extensions for this. So let's decline all cases when the
12721 // sizes of types of LHS and RHS do not match.
12722 // TODO: Maybe try to get RHS from sext to catch more cases?
12724 return false;
12725
12726 // Should not overflow.
12727 if (!LHSAddExpr->hasNoSignedWrap())
12728 return false;
12729
12730 auto *LL = LHSAddExpr->getOperand(0);
12731 auto *LR = LHSAddExpr->getOperand(1);
12732 auto *MinusOne = getMinusOne(RHS->getType());
12733
12734 // Checks that S1 >= 0 && S2 > RHS, trivially or using the found context.
12735 auto IsSumGreaterThanRHS = [&](const SCEV *S1, const SCEV *S2) {
12736 return IsSGTViaContext(S1, MinusOne) && IsSGTViaContext(S2, RHS);
12737 };
12738 // Try to prove the following rule:
12739 // (LHS = LL + LR) && (LL >= 0) && (LR > RHS) => (LHS > RHS).
12740 // (LHS = LL + LR) && (LR >= 0) && (LL > RHS) => (LHS > RHS).
12741 if (IsSumGreaterThanRHS(LL, LR) || IsSumGreaterThanRHS(LR, LL))
12742 return true;
12743 } else if (auto *LHSUnknownExpr = dyn_cast<SCEVUnknown>(LHS)) {
12744 Value *LL, *LR;
12745 // FIXME: Once we have SDiv implemented, we can get rid of this matching.
12746
12747 using namespace llvm::PatternMatch;
12748
12749 if (match(LHSUnknownExpr->getValue(), m_SDiv(m_Value(LL), m_Value(LR)))) {
12750 // Rules for division.
12751 // We are going to perform some comparisons with Denominator and its
12752 // derivative expressions. In general case, creating a SCEV for it may
12753 // lead to a complex analysis of the entire graph, and in particular it
12754 // can request trip count recalculation for the same loop. This would
12755 // cache as SCEVCouldNotCompute to avoid the infinite recursion. To avoid
12756 // this, we only want to create SCEVs that are constants in this section.
12757 // So we bail if Denominator is not a constant.
12758 if (!isa<ConstantInt>(LR))
12759 return false;
12760
12761 auto *Denominator = cast<SCEVConstant>(getSCEV(LR));
12762
12763 // We want to make sure that LHS = FoundLHS / Denominator. If it is so,
12764 // then a SCEV for the numerator already exists and matches with FoundLHS.
12765 auto *Numerator = getExistingSCEV(LL);
12766 if (!Numerator || Numerator->getType() != FoundLHS->getType())
12767 return false;
12768
12769 // Make sure that the numerator matches with FoundLHS and the denominator
12770 // is positive.
12771 if (!HasSameValue(Numerator, FoundLHS) || !isKnownPositive(Denominator))
12772 return false;
12773
12774 auto *DTy = Denominator->getType();
12775 auto *FRHSTy = FoundRHS->getType();
12776 if (DTy->isPointerTy() != FRHSTy->isPointerTy())
12777 // One of types is a pointer and another one is not. We cannot extend
12778 // them properly to a wider type, so let us just reject this case.
12779 // TODO: Usage of getEffectiveSCEVType for DTy, FRHSTy etc should help
12780 // to avoid this check.
12781 return false;
12782
12783 // Given that:
12784 // FoundLHS > FoundRHS, LHS = FoundLHS / Denominator, Denominator > 0.
12785 auto *WTy = getWiderType(DTy, FRHSTy);
12786 auto *DenominatorExt = getNoopOrSignExtend(Denominator, WTy);
12787 auto *FoundRHSExt = getNoopOrSignExtend(FoundRHS, WTy);
12788
12789 // Try to prove the following rule:
12790 // (FoundRHS > Denominator - 2) && (RHS <= 0) => (LHS > RHS).
12791 // For example, given that FoundLHS > 2. It means that FoundLHS is at
12792 // least 3. If we divide it by Denominator < 4, we will have at least 1.
12793 auto *DenomMinusTwo = getMinusSCEV(DenominatorExt, getConstant(WTy, 2));
12794 if (isKnownNonPositive(RHS) &&
12795 IsSGTViaContext(FoundRHSExt, DenomMinusTwo))
12796 return true;
12797
12798 // Try to prove the following rule:
12799 // (FoundRHS > -1 - Denominator) && (RHS < 0) => (LHS > RHS).
12800 // For example, given that FoundLHS > -3. Then FoundLHS is at least -2.
12801 // If we divide it by Denominator > 2, then:
12802 // 1. If FoundLHS is negative, then the result is 0.
12803 // 2. If FoundLHS is non-negative, then the result is non-negative.
12804 // Anyways, the result is non-negative.
12805 auto *MinusOne = getMinusOne(WTy);
12806 auto *NegDenomMinusOne = getMinusSCEV(MinusOne, DenominatorExt);
12807 if (isKnownNegative(RHS) &&
12808 IsSGTViaContext(FoundRHSExt, NegDenomMinusOne))
12809 return true;
12810 }
12811 }
12812
12813 // If our expression contained SCEVUnknown Phis, and we split it down and now
12814 // need to prove something for them, try to prove the predicate for every
12815 // possible incoming values of those Phis.
12816 if (isImpliedViaMerge(Pred, OrigLHS, RHS, OrigFoundLHS, FoundRHS, Depth + 1))
12817 return true;
12818
12819 return false;
12820}
12821
12823 const SCEV *RHS) {
12824 // zext x u<= sext x, sext x s<= zext x
12825 const SCEV *Op;
12826 switch (Pred) {
12827 case ICmpInst::ICMP_SGE:
12828 std::swap(LHS, RHS);
12829 [[fallthrough]];
12830 case ICmpInst::ICMP_SLE: {
12831 // If operand >=s 0 then ZExt == SExt. If operand <s 0 then SExt <s ZExt.
12832 return match(LHS, m_scev_SExt(m_SCEV(Op))) &&
12834 }
12835 case ICmpInst::ICMP_UGE:
12836 std::swap(LHS, RHS);
12837 [[fallthrough]];
12838 case ICmpInst::ICMP_ULE: {
12839 // If operand >=u 0 then ZExt == SExt. If operand <u 0 then ZExt <u SExt.
12840 return match(LHS, m_scev_ZExt(m_SCEV(Op))) &&
12842 }
12843 default:
12844 return false;
12845 };
12846 llvm_unreachable("unhandled case");
12847}
12848
12849bool ScalarEvolution::isKnownViaNonRecursiveReasoning(CmpPredicate Pred,
12850 const SCEV *LHS,
12851 const SCEV *RHS) {
12852 return isKnownPredicateExtendIdiom(Pred, LHS, RHS) ||
12853 isKnownPredicateViaConstantRanges(Pred, LHS, RHS) ||
12854 IsKnownPredicateViaMinOrMax(*this, Pred, LHS, RHS) ||
12855 IsKnownPredicateViaAddRecStart(*this, Pred, LHS, RHS) ||
12856 isKnownPredicateViaNoOverflow(Pred, LHS, RHS);
12857}
12858
12859bool ScalarEvolution::isImpliedCondOperandsHelper(CmpPredicate Pred,
12860 const SCEV *LHS,
12861 const SCEV *RHS,
12862 const SCEV *FoundLHS,
12863 const SCEV *FoundRHS) {
12864 switch (Pred) {
12865 default:
12866 llvm_unreachable("Unexpected CmpPredicate value!");
12867 case ICmpInst::ICMP_EQ:
12868 case ICmpInst::ICMP_NE:
12869 if (HasSameValue(LHS, FoundLHS) && HasSameValue(RHS, FoundRHS))
12870 return true;
12871 break;
12872 case ICmpInst::ICMP_SLT:
12873 case ICmpInst::ICMP_SLE:
12874 if (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_SLE, LHS, FoundLHS) &&
12875 isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_SGE, RHS, FoundRHS))
12876 return true;
12877 break;
12878 case ICmpInst::ICMP_SGT:
12879 case ICmpInst::ICMP_SGE:
12880 if (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_SGE, LHS, FoundLHS) &&
12881 isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_SLE, RHS, FoundRHS))
12882 return true;
12883 break;
12884 case ICmpInst::ICMP_ULT:
12885 case ICmpInst::ICMP_ULE:
12886 if (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_ULE, LHS, FoundLHS) &&
12887 isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_UGE, RHS, FoundRHS))
12888 return true;
12889 break;
12890 case ICmpInst::ICMP_UGT:
12891 case ICmpInst::ICMP_UGE:
12892 if (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_UGE, LHS, FoundLHS) &&
12893 isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_ULE, RHS, FoundRHS))
12894 return true;
12895 break;
12896 }
12897
12898 // Maybe it can be proved via operations?
12899 if (isImpliedViaOperations(Pred, LHS, RHS, FoundLHS, FoundRHS))
12900 return true;
12901
12902 return false;
12903}
12904
12905bool ScalarEvolution::isImpliedCondOperandsViaRanges(
12906 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, CmpPredicate FoundPred,
12907 const SCEV *FoundLHS, const SCEV *FoundRHS) {
12908 if (!isa<SCEVConstant>(RHS) || !isa<SCEVConstant>(FoundRHS))
12909 // The restriction on `FoundRHS` be lifted easily -- it exists only to
12910 // reduce the compile time impact of this optimization.
12911 return false;
12912
12913 std::optional<APInt> Addend = computeConstantDifference(LHS, FoundLHS);
12914 if (!Addend)
12915 return false;
12916
12917 const APInt &ConstFoundRHS = cast<SCEVConstant>(FoundRHS)->getAPInt();
12918
12919 // `FoundLHSRange` is the range we know `FoundLHS` to be in by virtue of the
12920 // antecedent "`FoundLHS` `FoundPred` `FoundRHS`".
12921 ConstantRange FoundLHSRange =
12922 ConstantRange::makeExactICmpRegion(FoundPred, ConstFoundRHS);
12923
12924 // Since `LHS` is `FoundLHS` + `Addend`, we can compute a range for `LHS`:
12925 ConstantRange LHSRange = FoundLHSRange.add(ConstantRange(*Addend));
12926
12927 // We can also compute the range of values for `LHS` that satisfy the
12928 // consequent, "`LHS` `Pred` `RHS`":
12929 const APInt &ConstRHS = cast<SCEVConstant>(RHS)->getAPInt();
12930 // The antecedent implies the consequent if every value of `LHS` that
12931 // satisfies the antecedent also satisfies the consequent.
12932 return LHSRange.icmp(Pred, ConstRHS);
12933}
12934
12935bool ScalarEvolution::canIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride,
12936 bool IsSigned) {
12937 assert(isKnownPositive(Stride) && "Positive stride expected!");
12938
12939 unsigned BitWidth = getTypeSizeInBits(RHS->getType());
12940 const SCEV *One = getOne(Stride->getType());
12941
12942 if (IsSigned) {
12943 APInt MaxRHS = getSignedRangeMax(RHS);
12944 APInt MaxValue = APInt::getSignedMaxValue(BitWidth);
12945 APInt MaxStrideMinusOne = getSignedRangeMax(getMinusSCEV(Stride, One));
12946
12947 // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow!
12948 return (std::move(MaxValue) - MaxStrideMinusOne).slt(MaxRHS);
12949 }
12950
12951 APInt MaxRHS = getUnsignedRangeMax(RHS);
12952 APInt MaxValue = APInt::getMaxValue(BitWidth);
12953 APInt MaxStrideMinusOne = getUnsignedRangeMax(getMinusSCEV(Stride, One));
12954
12955 // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow!
12956 return (std::move(MaxValue) - MaxStrideMinusOne).ult(MaxRHS);
12957}
12958
12959bool ScalarEvolution::canIVOverflowOnGT(const SCEV *RHS, const SCEV *Stride,
12960 bool IsSigned) {
12961
12962 unsigned BitWidth = getTypeSizeInBits(RHS->getType());
12963 const SCEV *One = getOne(Stride->getType());
12964
12965 if (IsSigned) {
12966 APInt MinRHS = getSignedRangeMin(RHS);
12967 APInt MinValue = APInt::getSignedMinValue(BitWidth);
12968 APInt MaxStrideMinusOne = getSignedRangeMax(getMinusSCEV(Stride, One));
12969
12970 // SMinRHS - SMaxStrideMinusOne < SMinValue => overflow!
12971 return (std::move(MinValue) + MaxStrideMinusOne).sgt(MinRHS);
12972 }
12973
12974 APInt MinRHS = getUnsignedRangeMin(RHS);
12975 APInt MinValue = APInt::getMinValue(BitWidth);
12976 APInt MaxStrideMinusOne = getUnsignedRangeMax(getMinusSCEV(Stride, One));
12977
12978 // UMinRHS - UMaxStrideMinusOne < UMinValue => overflow!
12979 return (std::move(MinValue) + MaxStrideMinusOne).ugt(MinRHS);
12980}
12981
12983 // umin(N, 1) + floor((N - umin(N, 1)) / D)
12984 // This is equivalent to "1 + floor((N - 1) / D)" for N != 0. The umin
12985 // expression fixes the case of N=0.
12986 const SCEV *MinNOne = getUMinExpr(N, getOne(N->getType()));
12987 const SCEV *NMinusOne = getMinusSCEV(N, MinNOne);
12988 return getAddExpr(MinNOne, getUDivExpr(NMinusOne, D));
12989}
12990
12991const SCEV *ScalarEvolution::computeMaxBECountForLT(const SCEV *Start,
12992 const SCEV *Stride,
12993 const SCEV *End,
12994 unsigned BitWidth,
12995 bool IsSigned) {
12996 // The logic in this function assumes we can represent a positive stride.
12997 // If we can't, the backedge-taken count must be zero.
12998 if (IsSigned && BitWidth == 1)
12999 return getZero(Stride->getType());
13000
13001 // This code below only been closely audited for negative strides in the
13002 // unsigned comparison case, it may be correct for signed comparison, but
13003 // that needs to be established.
13004 if (IsSigned && isKnownNegative(Stride))
13005 return getCouldNotCompute();
13006
13007 // Calculate the maximum backedge count based on the range of values
13008 // permitted by Start, End, and Stride.
13009 APInt MinStart =
13010 IsSigned ? getSignedRangeMin(Start) : getUnsignedRangeMin(Start);
13011
13012 APInt MinStride =
13013 IsSigned ? getSignedRangeMin(Stride) : getUnsignedRangeMin(Stride);
13014
13015 // We assume either the stride is positive, or the backedge-taken count
13016 // is zero. So force StrideForMaxBECount to be at least one.
13017 APInt One(BitWidth, 1);
13018 APInt StrideForMaxBECount = IsSigned ? APIntOps::smax(One, MinStride)
13019 : APIntOps::umax(One, MinStride);
13020
13021 APInt MaxValue = IsSigned ? APInt::getSignedMaxValue(BitWidth)
13022 : APInt::getMaxValue(BitWidth);
13023 APInt Limit = MaxValue - (StrideForMaxBECount - 1);
13024
13025 // Although End can be a MAX expression we estimate MaxEnd considering only
13026 // the case End = RHS of the loop termination condition. This is safe because
13027 // in the other case (End - Start) is zero, leading to a zero maximum backedge
13028 // taken count.
13029 APInt MaxEnd = IsSigned ? APIntOps::smin(getSignedRangeMax(End), Limit)
13030 : APIntOps::umin(getUnsignedRangeMax(End), Limit);
13031
13032 // MaxBECount = ceil((max(MaxEnd, MinStart) - MinStart) / Stride)
13033 MaxEnd = IsSigned ? APIntOps::smax(MaxEnd, MinStart)
13034 : APIntOps::umax(MaxEnd, MinStart);
13035
13036 return getUDivCeilSCEV(getConstant(MaxEnd - MinStart) /* Delta */,
13037 getConstant(StrideForMaxBECount) /* Step */);
13038}
13039
13041ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
13042 const Loop *L, bool IsSigned,
13043 bool ControlsOnlyExit, bool AllowPredicates) {
13045
13047 bool PredicatedIV = false;
13048 if (!IV) {
13049 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(LHS)) {
13050 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(ZExt->getOperand());
13051 if (AR && AR->getLoop() == L && AR->isAffine()) {
13052 auto canProveNUW = [&]() {
13053 // We can use the comparison to infer no-wrap flags only if it fully
13054 // controls the loop exit.
13055 if (!ControlsOnlyExit)
13056 return false;
13057
13058 if (!isLoopInvariant(RHS, L))
13059 return false;
13060
13061 if (!isKnownNonZero(AR->getStepRecurrence(*this)))
13062 // We need the sequence defined by AR to strictly increase in the
13063 // unsigned integer domain for the logic below to hold.
13064 return false;
13065
13066 const unsigned InnerBitWidth = getTypeSizeInBits(AR->getType());
13067 const unsigned OuterBitWidth = getTypeSizeInBits(RHS->getType());
13068 // If RHS <=u Limit, then there must exist a value V in the sequence
13069 // defined by AR (e.g. {Start,+,Step}) such that V >u RHS, and
13070 // V <=u UINT_MAX. Thus, we must exit the loop before unsigned
13071 // overflow occurs. This limit also implies that a signed comparison
13072 // (in the wide bitwidth) is equivalent to an unsigned comparison as
13073 // the high bits on both sides must be zero.
13074 APInt StrideMax = getUnsignedRangeMax(AR->getStepRecurrence(*this));
13075 APInt Limit = APInt::getMaxValue(InnerBitWidth) - (StrideMax - 1);
13076 Limit = Limit.zext(OuterBitWidth);
13077 return getUnsignedRangeMax(applyLoopGuards(RHS, L)).ule(Limit);
13078 };
13079 auto Flags = AR->getNoWrapFlags();
13080 if (!hasFlags(Flags, SCEV::FlagNUW) && canProveNUW())
13081 Flags = setFlags(Flags, SCEV::FlagNUW);
13082
13083 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), Flags);
13084 if (AR->hasNoUnsignedWrap()) {
13085 // Emulate what getZeroExtendExpr would have done during construction
13086 // if we'd been able to infer the fact just above at that time.
13087 const SCEV *Step = AR->getStepRecurrence(*this);
13088 Type *Ty = ZExt->getType();
13089 auto *S = getAddRecExpr(
13091 getZeroExtendExpr(Step, Ty, 0), L, AR->getNoWrapFlags());
13093 }
13094 }
13095 }
13096 }
13097
13098
13099 if (!IV && AllowPredicates) {
13100 // Try to make this an AddRec using runtime tests, in the first X
13101 // iterations of this loop, where X is the SCEV expression found by the
13102 // algorithm below.
13103 IV = convertSCEVToAddRecWithPredicates(LHS, L, Predicates);
13104 PredicatedIV = true;
13105 }
13106
13107 // Avoid weird loops
13108 if (!IV || IV->getLoop() != L || !IV->isAffine())
13109 return getCouldNotCompute();
13110
13111 // A precondition of this method is that the condition being analyzed
13112 // reaches an exiting branch which dominates the latch. Given that, we can
13113 // assume that an increment which violates the nowrap specification and
13114 // produces poison must cause undefined behavior when the resulting poison
13115 // value is branched upon and thus we can conclude that the backedge is
13116 // taken no more often than would be required to produce that poison value.
13117 // Note that a well defined loop can exit on the iteration which violates
13118 // the nowrap specification if there is another exit (either explicit or
13119 // implicit/exceptional) which causes the loop to execute before the
13120 // exiting instruction we're analyzing would trigger UB.
13121 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13122 bool NoWrap = ControlsOnlyExit && IV->getNoWrapFlags(WrapType);
13124
13125 const SCEV *Stride = IV->getStepRecurrence(*this);
13126
13127 bool PositiveStride = isKnownPositive(Stride);
13128
13129 // Avoid negative or zero stride values.
13130 if (!PositiveStride) {
13131 // We can compute the correct backedge taken count for loops with unknown
13132 // strides if we can prove that the loop is not an infinite loop with side
13133 // effects. Here's the loop structure we are trying to handle -
13134 //
13135 // i = start
13136 // do {
13137 // A[i] = i;
13138 // i += s;
13139 // } while (i < end);
13140 //
13141 // The backedge taken count for such loops is evaluated as -
13142 // (max(end, start + stride) - start - 1) /u stride
13143 //
13144 // The additional preconditions that we need to check to prove correctness
13145 // of the above formula is as follows -
13146 //
13147 // a) IV is either nuw or nsw depending upon signedness (indicated by the
13148 // NoWrap flag).
13149 // b) the loop is guaranteed to be finite (e.g. is mustprogress and has
13150 // no side effects within the loop)
13151 // c) loop has a single static exit (with no abnormal exits)
13152 //
13153 // Precondition a) implies that if the stride is negative, this is a single
13154 // trip loop. The backedge taken count formula reduces to zero in this case.
13155 //
13156 // Precondition b) and c) combine to imply that if rhs is invariant in L,
13157 // then a zero stride means the backedge can't be taken without executing
13158 // undefined behavior.
13159 //
13160 // The positive stride case is the same as isKnownPositive(Stride) returning
13161 // true (original behavior of the function).
13162 //
13163 if (PredicatedIV || !NoWrap || !loopIsFiniteByAssumption(L) ||
13165 return getCouldNotCompute();
13166
13167 if (!isKnownNonZero(Stride)) {
13168 // If we have a step of zero, and RHS isn't invariant in L, we don't know
13169 // if it might eventually be greater than start and if so, on which
13170 // iteration. We can't even produce a useful upper bound.
13171 if (!isLoopInvariant(RHS, L))
13172 return getCouldNotCompute();
13173
13174 // We allow a potentially zero stride, but we need to divide by stride
13175 // below. Since the loop can't be infinite and this check must control
13176 // the sole exit, we can infer the exit must be taken on the first
13177 // iteration (e.g. backedge count = 0) if the stride is zero. Given that,
13178 // we know the numerator in the divides below must be zero, so we can
13179 // pick an arbitrary non-zero value for the denominator (e.g. stride)
13180 // and produce the right result.
13181 // FIXME: Handle the case where Stride is poison?
13182 auto wouldZeroStrideBeUB = [&]() {
13183 // Proof by contradiction. Suppose the stride were zero. If we can
13184 // prove that the backedge *is* taken on the first iteration, then since
13185 // we know this condition controls the sole exit, we must have an
13186 // infinite loop. We can't have a (well defined) infinite loop per
13187 // check just above.
13188 // Note: The (Start - Stride) term is used to get the start' term from
13189 // (start' + stride,+,stride). Remember that we only care about the
13190 // result of this expression when stride == 0 at runtime.
13191 auto *StartIfZero = getMinusSCEV(IV->getStart(), Stride);
13192 return isLoopEntryGuardedByCond(L, Cond, StartIfZero, RHS);
13193 };
13194 if (!wouldZeroStrideBeUB()) {
13195 Stride = getUMaxExpr(Stride, getOne(Stride->getType()));
13196 }
13197 }
13198 } else if (!NoWrap) {
13199 // Avoid proven overflow cases: this will ensure that the backedge taken
13200 // count will not generate any unsigned overflow.
13201 if (canIVOverflowOnLT(RHS, Stride, IsSigned))
13202 return getCouldNotCompute();
13203 }
13204
13205 // On all paths just preceeding, we established the following invariant:
13206 // IV can be assumed not to overflow up to and including the exiting
13207 // iteration. We proved this in one of two ways:
13208 // 1) We can show overflow doesn't occur before the exiting iteration
13209 // 1a) canIVOverflowOnLT, and b) step of one
13210 // 2) We can show that if overflow occurs, the loop must execute UB
13211 // before any possible exit.
13212 // Note that we have not yet proved RHS invariant (in general).
13213
13214 const SCEV *Start = IV->getStart();
13215
13216 // Preserve pointer-typed Start/RHS to pass to isLoopEntryGuardedByCond.
13217 // If we convert to integers, isLoopEntryGuardedByCond will miss some cases.
13218 // Use integer-typed versions for actual computation; we can't subtract
13219 // pointers in general.
13220 const SCEV *OrigStart = Start;
13221 const SCEV *OrigRHS = RHS;
13222 if (Start->getType()->isPointerTy()) {
13224 if (isa<SCEVCouldNotCompute>(Start))
13225 return Start;
13226 }
13227 if (RHS->getType()->isPointerTy()) {
13230 return RHS;
13231 }
13232
13233 const SCEV *End = nullptr, *BECount = nullptr,
13234 *BECountIfBackedgeTaken = nullptr;
13235 if (!isLoopInvariant(RHS, L)) {
13236 const auto *RHSAddRec = dyn_cast<SCEVAddRecExpr>(RHS);
13237 if (PositiveStride && RHSAddRec != nullptr && RHSAddRec->getLoop() == L &&
13238 RHSAddRec->getNoWrapFlags()) {
13239 // The structure of loop we are trying to calculate backedge count of:
13240 //
13241 // left = left_start
13242 // right = right_start
13243 //
13244 // while(left < right){
13245 // ... do something here ...
13246 // left += s1; // stride of left is s1 (s1 > 0)
13247 // right += s2; // stride of right is s2 (s2 < 0)
13248 // }
13249 //
13250
13251 const SCEV *RHSStart = RHSAddRec->getStart();
13252 const SCEV *RHSStride = RHSAddRec->getStepRecurrence(*this);
13253
13254 // If Stride - RHSStride is positive and does not overflow, we can write
13255 // backedge count as ->
13256 // ceil((End - Start) /u (Stride - RHSStride))
13257 // Where, End = max(RHSStart, Start)
13258
13259 // Check if RHSStride < 0 and Stride - RHSStride will not overflow.
13260 if (isKnownNegative(RHSStride) &&
13261 willNotOverflow(Instruction::Sub, /*Signed=*/true, Stride,
13262 RHSStride)) {
13263
13264 const SCEV *Denominator = getMinusSCEV(Stride, RHSStride);
13265 if (isKnownPositive(Denominator)) {
13266 End = IsSigned ? getSMaxExpr(RHSStart, Start)
13267 : getUMaxExpr(RHSStart, Start);
13268
13269 // We can do this because End >= Start, as End = max(RHSStart, Start)
13270 const SCEV *Delta = getMinusSCEV(End, Start);
13271
13272 BECount = getUDivCeilSCEV(Delta, Denominator);
13273 BECountIfBackedgeTaken =
13274 getUDivCeilSCEV(getMinusSCEV(RHSStart, Start), Denominator);
13275 }
13276 }
13277 }
13278 if (BECount == nullptr) {
13279 // If we cannot calculate ExactBECount, we can calculate the MaxBECount,
13280 // given the start, stride and max value for the end bound of the
13281 // loop (RHS), and the fact that IV does not overflow (which is
13282 // checked above).
13283 const SCEV *MaxBECount = computeMaxBECountForLT(
13284 Start, Stride, RHS, getTypeSizeInBits(LHS->getType()), IsSigned);
13285 return ExitLimit(getCouldNotCompute() /* ExactNotTaken */, MaxBECount,
13286 MaxBECount, false /*MaxOrZero*/, Predicates);
13287 }
13288 } else {
13289 // We use the expression (max(End,Start)-Start)/Stride to describe the
13290 // backedge count, as if the backedge is taken at least once
13291 // max(End,Start) is End and so the result is as above, and if not
13292 // max(End,Start) is Start so we get a backedge count of zero.
13293 auto *OrigStartMinusStride = getMinusSCEV(OrigStart, Stride);
13294 assert(isAvailableAtLoopEntry(OrigStartMinusStride, L) && "Must be!");
13295 assert(isAvailableAtLoopEntry(OrigStart, L) && "Must be!");
13296 assert(isAvailableAtLoopEntry(OrigRHS, L) && "Must be!");
13297 // Can we prove (max(RHS,Start) > Start - Stride?
13298 if (isLoopEntryGuardedByCond(L, Cond, OrigStartMinusStride, OrigStart) &&
13299 isLoopEntryGuardedByCond(L, Cond, OrigStartMinusStride, OrigRHS)) {
13300 // In this case, we can use a refined formula for computing backedge
13301 // taken count. The general formula remains:
13302 // "End-Start /uceiling Stride" where "End = max(RHS,Start)"
13303 // We want to use the alternate formula:
13304 // "((End - 1) - (Start - Stride)) /u Stride"
13305 // Let's do a quick case analysis to show these are equivalent under
13306 // our precondition that max(RHS,Start) > Start - Stride.
13307 // * For RHS <= Start, the backedge-taken count must be zero.
13308 // "((End - 1) - (Start - Stride)) /u Stride" reduces to
13309 // "((Start - 1) - (Start - Stride)) /u Stride" which simplies to
13310 // "Stride - 1 /u Stride" which is indeed zero for all non-zero values
13311 // of Stride. For 0 stride, we've use umin(1,Stride) above,
13312 // reducing this to the stride of 1 case.
13313 // * For RHS >= Start, the backedge count must be "RHS-Start /uceil
13314 // Stride".
13315 // "((End - 1) - (Start - Stride)) /u Stride" reduces to
13316 // "((RHS - 1) - (Start - Stride)) /u Stride" reassociates to
13317 // "((RHS - (Start - Stride) - 1) /u Stride".
13318 // Our preconditions trivially imply no overflow in that form.
13319 const SCEV *MinusOne = getMinusOne(Stride->getType());
13320 const SCEV *Numerator =
13321 getMinusSCEV(getAddExpr(RHS, MinusOne), getMinusSCEV(Start, Stride));
13322 BECount = getUDivExpr(Numerator, Stride);
13323 }
13324
13325 if (!BECount) {
13326 auto canProveRHSGreaterThanEqualStart = [&]() {
13327 auto CondGE = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13328 const SCEV *GuardedRHS = applyLoopGuards(OrigRHS, L);
13329 const SCEV *GuardedStart = applyLoopGuards(OrigStart, L);
13330
13331 if (isLoopEntryGuardedByCond(L, CondGE, OrigRHS, OrigStart) ||
13332 isKnownPredicate(CondGE, GuardedRHS, GuardedStart))
13333 return true;
13334
13335 // (RHS > Start - 1) implies RHS >= Start.
13336 // * "RHS >= Start" is trivially equivalent to "RHS > Start - 1" if
13337 // "Start - 1" doesn't overflow.
13338 // * For signed comparison, if Start - 1 does overflow, it's equal
13339 // to INT_MAX, and "RHS >s INT_MAX" is trivially false.
13340 // * For unsigned comparison, if Start - 1 does overflow, it's equal
13341 // to UINT_MAX, and "RHS >u UINT_MAX" is trivially false.
13342 //
13343 // FIXME: Should isLoopEntryGuardedByCond do this for us?
13344 auto CondGT = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13345 auto *StartMinusOne =
13346 getAddExpr(OrigStart, getMinusOne(OrigStart->getType()));
13347 return isLoopEntryGuardedByCond(L, CondGT, OrigRHS, StartMinusOne);
13348 };
13349
13350 // If we know that RHS >= Start in the context of loop, then we know
13351 // that max(RHS, Start) = RHS at this point.
13352 if (canProveRHSGreaterThanEqualStart()) {
13353 End = RHS;
13354 } else {
13355 // If RHS < Start, the backedge will be taken zero times. So in
13356 // general, we can write the backedge-taken count as:
13357 //
13358 // RHS >= Start ? ceil(RHS - Start) / Stride : 0
13359 //
13360 // We convert it to the following to make it more convenient for SCEV:
13361 //
13362 // ceil(max(RHS, Start) - Start) / Stride
13363 End = IsSigned ? getSMaxExpr(RHS, Start) : getUMaxExpr(RHS, Start);
13364
13365 // See what would happen if we assume the backedge is taken. This is
13366 // used to compute MaxBECount.
13367 BECountIfBackedgeTaken =
13368 getUDivCeilSCEV(getMinusSCEV(RHS, Start), Stride);
13369 }
13370
13371 // At this point, we know:
13372 //
13373 // 1. If IsSigned, Start <=s End; otherwise, Start <=u End
13374 // 2. The index variable doesn't overflow.
13375 //
13376 // Therefore, we know N exists such that
13377 // (Start + Stride * N) >= End, and computing "(Start + Stride * N)"
13378 // doesn't overflow.
13379 //
13380 // Using this information, try to prove whether the addition in
13381 // "(Start - End) + (Stride - 1)" has unsigned overflow.
13382 const SCEV *One = getOne(Stride->getType());
13383 bool MayAddOverflow = [&] {
13384 if (isKnownToBeAPowerOfTwo(Stride)) {
13385 // Suppose Stride is a power of two, and Start/End are unsigned
13386 // integers. Let UMAX be the largest representable unsigned
13387 // integer.
13388 //
13389 // By the preconditions of this function, we know
13390 // "(Start + Stride * N) >= End", and this doesn't overflow.
13391 // As a formula:
13392 //
13393 // End <= (Start + Stride * N) <= UMAX
13394 //
13395 // Subtracting Start from all the terms:
13396 //
13397 // End - Start <= Stride * N <= UMAX - Start
13398 //
13399 // Since Start is unsigned, UMAX - Start <= UMAX. Therefore:
13400 //
13401 // End - Start <= Stride * N <= UMAX
13402 //
13403 // Stride * N is a multiple of Stride. Therefore,
13404 //
13405 // End - Start <= Stride * N <= UMAX - (UMAX mod Stride)
13406 //
13407 // Since Stride is a power of two, UMAX + 1 is divisible by
13408 // Stride. Therefore, UMAX mod Stride == Stride - 1. So we can
13409 // write:
13410 //
13411 // End - Start <= Stride * N <= UMAX - Stride - 1
13412 //
13413 // Dropping the middle term:
13414 //
13415 // End - Start <= UMAX - Stride - 1
13416 //
13417 // Adding Stride - 1 to both sides:
13418 //
13419 // (End - Start) + (Stride - 1) <= UMAX
13420 //
13421 // In other words, the addition doesn't have unsigned overflow.
13422 //
13423 // A similar proof works if we treat Start/End as signed values.
13424 // Just rewrite steps before "End - Start <= Stride * N <= UMAX"
13425 // to use signed max instead of unsigned max. Note that we're
13426 // trying to prove a lack of unsigned overflow in either case.
13427 return false;
13428 }
13429 if (Start == Stride || Start == getMinusSCEV(Stride, One)) {
13430 // If Start is equal to Stride, (End - Start) + (Stride - 1) == End
13431 // - 1. If !IsSigned, 0 <u Stride == Start <=u End; so 0 <u End - 1
13432 // <u End. If IsSigned, 0 <s Stride == Start <=s End; so 0 <s End -
13433 // 1 <s End.
13434 //
13435 // If Start is equal to Stride - 1, (End - Start) + Stride - 1 ==
13436 // End.
13437 return false;
13438 }
13439 return true;
13440 }();
13441
13442 const SCEV *Delta = getMinusSCEV(End, Start);
13443 if (!MayAddOverflow) {
13444 // floor((D + (S - 1)) / S)
13445 // We prefer this formulation if it's legal because it's fewer
13446 // operations.
13447 BECount =
13448 getUDivExpr(getAddExpr(Delta, getMinusSCEV(Stride, One)), Stride);
13449 } else {
13450 BECount = getUDivCeilSCEV(Delta, Stride);
13451 }
13452 }
13453 }
13454
13455 const SCEV *ConstantMaxBECount;
13456 bool MaxOrZero = false;
13457 if (isa<SCEVConstant>(BECount)) {
13458 ConstantMaxBECount = BECount;
13459 } else if (BECountIfBackedgeTaken &&
13460 isa<SCEVConstant>(BECountIfBackedgeTaken)) {
13461 // If we know exactly how many times the backedge will be taken if it's
13462 // taken at least once, then the backedge count will either be that or
13463 // zero.
13464 ConstantMaxBECount = BECountIfBackedgeTaken;
13465 MaxOrZero = true;
13466 } else {
13467 ConstantMaxBECount = computeMaxBECountForLT(
13468 Start, Stride, RHS, getTypeSizeInBits(LHS->getType()), IsSigned);
13469 }
13470
13471 if (isa<SCEVCouldNotCompute>(ConstantMaxBECount) &&
13472 !isa<SCEVCouldNotCompute>(BECount))
13473 ConstantMaxBECount = getConstant(getUnsignedRangeMax(BECount));
13474
13475 const SCEV *SymbolicMaxBECount =
13476 isa<SCEVCouldNotCompute>(BECount) ? ConstantMaxBECount : BECount;
13477 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, MaxOrZero,
13478 Predicates);
13479}
13480
13481ScalarEvolution::ExitLimit ScalarEvolution::howManyGreaterThans(
13482 const SCEV *LHS, const SCEV *RHS, const Loop *L, bool IsSigned,
13483 bool ControlsOnlyExit, bool AllowPredicates) {
13485 // We handle only IV > Invariant
13486 if (!isLoopInvariant(RHS, L))
13487 return getCouldNotCompute();
13488
13489 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(LHS);
13490 if (!IV && AllowPredicates)
13491 // Try to make this an AddRec using runtime tests, in the first X
13492 // iterations of this loop, where X is the SCEV expression found by the
13493 // algorithm below.
13494 IV = convertSCEVToAddRecWithPredicates(LHS, L, Predicates);
13495
13496 // Avoid weird loops
13497 if (!IV || IV->getLoop() != L || !IV->isAffine())
13498 return getCouldNotCompute();
13499
13500 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13501 bool NoWrap = ControlsOnlyExit && IV->getNoWrapFlags(WrapType);
13503
13504 const SCEV *Stride = getNegativeSCEV(IV->getStepRecurrence(*this));
13505
13506 // Avoid negative or zero stride values
13507 if (!isKnownPositive(Stride))
13508 return getCouldNotCompute();
13509
13510 // Avoid proven overflow cases: this will ensure that the backedge taken count
13511 // will not generate any unsigned overflow. Relaxed no-overflow conditions
13512 // exploit NoWrapFlags, allowing to optimize in presence of undefined
13513 // behaviors like the case of C language.
13514 if (!Stride->isOne() && !NoWrap)
13515 if (canIVOverflowOnGT(RHS, Stride, IsSigned))
13516 return getCouldNotCompute();
13517
13518 const SCEV *Start = IV->getStart();
13519 const SCEV *End = RHS;
13520 if (!isLoopEntryGuardedByCond(L, Cond, getAddExpr(Start, Stride), RHS)) {
13521 // If we know that Start >= RHS in the context of loop, then we know that
13522 // min(RHS, Start) = RHS at this point.
13524 L, IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, Start, RHS))
13525 End = RHS;
13526 else
13527 End = IsSigned ? getSMinExpr(RHS, Start) : getUMinExpr(RHS, Start);
13528 }
13529
13530 if (Start->getType()->isPointerTy()) {
13532 if (isa<SCEVCouldNotCompute>(Start))
13533 return Start;
13534 }
13535 if (End->getType()->isPointerTy()) {
13536 End = getLosslessPtrToIntExpr(End);
13537 if (isa<SCEVCouldNotCompute>(End))
13538 return End;
13539 }
13540
13541 // Compute ((Start - End) + (Stride - 1)) / Stride.
13542 // FIXME: This can overflow. Holding off on fixing this for now;
13543 // howManyGreaterThans will hopefully be gone soon.
13544 const SCEV *One = getOne(Stride->getType());
13545 const SCEV *BECount = getUDivExpr(
13546 getAddExpr(getMinusSCEV(Start, End), getMinusSCEV(Stride, One)), Stride);
13547
13548 APInt MaxStart = IsSigned ? getSignedRangeMax(Start)
13550
13551 APInt MinStride = IsSigned ? getSignedRangeMin(Stride)
13552 : getUnsignedRangeMin(Stride);
13553
13554 unsigned BitWidth = getTypeSizeInBits(LHS->getType());
13555 APInt Limit = IsSigned ? APInt::getSignedMinValue(BitWidth) + (MinStride - 1)
13556 : APInt::getMinValue(BitWidth) + (MinStride - 1);
13557
13558 // Although End can be a MIN expression we estimate MinEnd considering only
13559 // the case End = RHS. This is safe because in the other case (Start - End)
13560 // is zero, leading to a zero maximum backedge taken count.
13561 APInt MinEnd =
13562 IsSigned ? APIntOps::smax(getSignedRangeMin(RHS), Limit)
13563 : APIntOps::umax(getUnsignedRangeMin(RHS), Limit);
13564
13565 const SCEV *ConstantMaxBECount =
13566 isa<SCEVConstant>(BECount)
13567 ? BECount
13568 : getUDivCeilSCEV(getConstant(MaxStart - MinEnd),
13569 getConstant(MinStride));
13570
13571 if (isa<SCEVCouldNotCompute>(ConstantMaxBECount))
13572 ConstantMaxBECount = BECount;
13573 const SCEV *SymbolicMaxBECount =
13574 isa<SCEVCouldNotCompute>(BECount) ? ConstantMaxBECount : BECount;
13575
13576 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
13577 Predicates);
13578}
13579
13581 ScalarEvolution &SE) const {
13582 if (Range.isFullSet()) // Infinite loop.
13583 return SE.getCouldNotCompute();
13584
13585 // If the start is a non-zero constant, shift the range to simplify things.
13586 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(getStart()))
13587 if (!SC->getValue()->isZero()) {
13589 Operands[0] = SE.getZero(SC->getType());
13590 const SCEV *Shifted = SE.getAddRecExpr(Operands, getLoop(),
13592 if (const auto *ShiftedAddRec = dyn_cast<SCEVAddRecExpr>(Shifted))
13593 return ShiftedAddRec->getNumIterationsInRange(
13594 Range.subtract(SC->getAPInt()), SE);
13595 // This is strange and shouldn't happen.
13596 return SE.getCouldNotCompute();
13597 }
13598
13599 // The only time we can solve this is when we have all constant indices.
13600 // Otherwise, we cannot determine the overflow conditions.
13601 if (any_of(operands(), [](const SCEV *Op) { return !isa<SCEVConstant>(Op); }))
13602 return SE.getCouldNotCompute();
13603
13604 // Okay at this point we know that all elements of the chrec are constants and
13605 // that the start element is zero.
13606
13607 // First check to see if the range contains zero. If not, the first
13608 // iteration exits.
13609 unsigned BitWidth = SE.getTypeSizeInBits(getType());
13610 if (!Range.contains(APInt(BitWidth, 0)))
13611 return SE.getZero(getType());
13612
13613 if (isAffine()) {
13614 // If this is an affine expression then we have this situation:
13615 // Solve {0,+,A} in Range === Ax in Range
13616
13617 // We know that zero is in the range. If A is positive then we know that
13618 // the upper value of the range must be the first possible exit value.
13619 // If A is negative then the lower of the range is the last possible loop
13620 // value. Also note that we already checked for a full range.
13621 APInt A = cast<SCEVConstant>(getOperand(1))->getAPInt();
13622 APInt End = A.sge(1) ? (Range.getUpper() - 1) : Range.getLower();
13623
13624 // The exit value should be (End+A)/A.
13625 APInt ExitVal = (End + A).udiv(A);
13626 ConstantInt *ExitValue = ConstantInt::get(SE.getContext(), ExitVal);
13627
13628 // Evaluate at the exit value. If we really did fall out of the valid
13629 // range, then we computed our trip count, otherwise wrap around or other
13630 // things must have happened.
13631 ConstantInt *Val = EvaluateConstantChrecAtConstant(this, ExitValue, SE);
13632 if (Range.contains(Val->getValue()))
13633 return SE.getCouldNotCompute(); // Something strange happened
13634
13635 // Ensure that the previous value is in the range.
13636 assert(Range.contains(
13638 ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) &&
13639 "Linear scev computation is off in a bad way!");
13640 return SE.getConstant(ExitValue);
13641 }
13642
13643 if (isQuadratic()) {
13644 if (auto S = SolveQuadraticAddRecRange(this, Range, SE))
13645 return SE.getConstant(*S);
13646 }
13647
13648 return SE.getCouldNotCompute();
13649}
13650
13651const SCEVAddRecExpr *
13653 assert(getNumOperands() > 1 && "AddRec with zero step?");
13654 // There is a temptation to just call getAddExpr(this, getStepRecurrence(SE)),
13655 // but in this case we cannot guarantee that the value returned will be an
13656 // AddRec because SCEV does not have a fixed point where it stops
13657 // simplification: it is legal to return ({rec1} + {rec2}). For example, it
13658 // may happen if we reach arithmetic depth limit while simplifying. So we
13659 // construct the returned value explicitly.
13661 // If this is {A,+,B,+,C,...,+,N}, then its step is {B,+,C,+,...,+,N}, and
13662 // (this + Step) is {A+B,+,B+C,+...,+,N}.
13663 for (unsigned i = 0, e = getNumOperands() - 1; i < e; ++i)
13664 Ops.push_back(SE.getAddExpr(getOperand(i), getOperand(i + 1)));
13665 // We know that the last operand is not a constant zero (otherwise it would
13666 // have been popped out earlier). This guarantees us that if the result has
13667 // the same last operand, then it will also not be popped out, meaning that
13668 // the returned value will be an AddRec.
13669 const SCEV *Last = getOperand(getNumOperands() - 1);
13670 assert(!Last->isZero() && "Recurrency with zero step?");
13671 Ops.push_back(Last);
13674}
13675
13676// Return true when S contains at least an undef value.
13678 return SCEVExprContains(S, [](const SCEV *S) {
13679 if (const auto *SU = dyn_cast<SCEVUnknown>(S))
13680 return isa<UndefValue>(SU->getValue());
13681 return false;
13682 });
13683}
13684
13685// Return true when S contains a value that is a nullptr.
13687 return SCEVExprContains(S, [](const SCEV *S) {
13688 if (const auto *SU = dyn_cast<SCEVUnknown>(S))
13689 return SU->getValue() == nullptr;
13690 return false;
13691 });
13692}
13693
13694/// Return the size of an element read or written by Inst.
13696 Type *Ty;
13697 if (StoreInst *Store = dyn_cast<StoreInst>(Inst))
13698 Ty = Store->getValueOperand()->getType();
13699 else if (LoadInst *Load = dyn_cast<LoadInst>(Inst))
13700 Ty = Load->getType();
13701 else
13702 return nullptr;
13703
13705 return getSizeOfExpr(ETy, Ty);
13706}
13707
13708//===----------------------------------------------------------------------===//
13709// SCEVCallbackVH Class Implementation
13710//===----------------------------------------------------------------------===//
13711
13713 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
13714 if (PHINode *PN = dyn_cast<PHINode>(getValPtr()))
13715 SE->ConstantEvolutionLoopExitValue.erase(PN);
13716 SE->eraseValueFromMap(getValPtr());
13717 // this now dangles!
13718}
13719
13720void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) {
13721 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
13722
13723 // Forget all the expressions associated with users of the old value,
13724 // so that future queries will recompute the expressions using the new
13725 // value.
13726 SE->forgetValue(getValPtr());
13727 // this now dangles!
13728}
13729
13730ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se)
13731 : CallbackVH(V), SE(se) {}
13732
13733//===----------------------------------------------------------------------===//
13734// ScalarEvolution Class Implementation
13735//===----------------------------------------------------------------------===//
13736
13739 LoopInfo &LI)
13740 : F(F), DL(F.getDataLayout()), TLI(TLI), AC(AC), DT(DT), LI(LI),
13741 CouldNotCompute(new SCEVCouldNotCompute()), ValuesAtScopes(64),
13742 LoopDispositions(64), BlockDispositions(64) {
13743 // To use guards for proving predicates, we need to scan every instruction in
13744 // relevant basic blocks, and not just terminators. Doing this is a waste of
13745 // time if the IR does not actually contain any calls to
13746 // @llvm.experimental.guard, so do a quick check and remember this beforehand.
13747 //
13748 // This pessimizes the case where a pass that preserves ScalarEvolution wants
13749 // to _add_ guards to the module when there weren't any before, and wants
13750 // ScalarEvolution to optimize based on those guards. For now we prefer to be
13751 // efficient in lieu of being smart in that rather obscure case.
13752
13753 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
13754 F.getParent(), Intrinsic::experimental_guard);
13755 HasGuards = GuardDecl && !GuardDecl->use_empty();
13756}
13757
13759 : F(Arg.F), DL(Arg.DL), HasGuards(Arg.HasGuards), TLI(Arg.TLI), AC(Arg.AC),
13760 DT(Arg.DT), LI(Arg.LI), CouldNotCompute(std::move(Arg.CouldNotCompute)),
13761 ValueExprMap(std::move(Arg.ValueExprMap)),
13762 PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)),
13763 PendingPhiRanges(std::move(Arg.PendingPhiRanges)),
13764 PendingMerges(std::move(Arg.PendingMerges)),
13765 ConstantMultipleCache(std::move(Arg.ConstantMultipleCache)),
13766 BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)),
13767 PredicatedBackedgeTakenCounts(
13768 std::move(Arg.PredicatedBackedgeTakenCounts)),
13769 BECountUsers(std::move(Arg.BECountUsers)),
13770 ConstantEvolutionLoopExitValue(
13771 std::move(Arg.ConstantEvolutionLoopExitValue)),
13772 ValuesAtScopes(std::move(Arg.ValuesAtScopes)),
13773 ValuesAtScopesUsers(std::move(Arg.ValuesAtScopesUsers)),
13774 LoopDispositions(std::move(Arg.LoopDispositions)),
13775 LoopPropertiesCache(std::move(Arg.LoopPropertiesCache)),
13776 BlockDispositions(std::move(Arg.BlockDispositions)),
13777 SCEVUsers(std::move(Arg.SCEVUsers)),
13778 UnsignedRanges(std::move(Arg.UnsignedRanges)),
13779 SignedRanges(std::move(Arg.SignedRanges)),
13780 UniqueSCEVs(std::move(Arg.UniqueSCEVs)),
13781 UniquePreds(std::move(Arg.UniquePreds)),
13782 SCEVAllocator(std::move(Arg.SCEVAllocator)),
13783 LoopUsers(std::move(Arg.LoopUsers)),
13784 PredicatedSCEVRewrites(std::move(Arg.PredicatedSCEVRewrites)),
13785 FirstUnknown(Arg.FirstUnknown) {
13786 Arg.FirstUnknown = nullptr;
13787}
13788
13790 // Iterate through all the SCEVUnknown instances and call their
13791 // destructors, so that they release their references to their values.
13792 for (SCEVUnknown *U = FirstUnknown; U;) {
13793 SCEVUnknown *Tmp = U;
13794 U = U->Next;
13795 Tmp->~SCEVUnknown();
13796 }
13797 FirstUnknown = nullptr;
13798
13799 ExprValueMap.clear();
13800 ValueExprMap.clear();
13801 HasRecMap.clear();
13802 BackedgeTakenCounts.clear();
13803 PredicatedBackedgeTakenCounts.clear();
13804
13805 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage");
13806 assert(PendingPhiRanges.empty() && "getRangeRef garbage");
13807 assert(PendingMerges.empty() && "isImpliedViaMerge garbage");
13808 assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!");
13809 assert(!ProvingSplitPredicate && "ProvingSplitPredicate garbage!");
13810}
13811
13815
13816/// When printing a top-level SCEV for trip counts, it's helpful to include
13817/// a type for constants which are otherwise hard to disambiguate.
13818static void PrintSCEVWithTypeHint(raw_ostream &OS, const SCEV* S) {
13819 if (isa<SCEVConstant>(S))
13820 OS << *S->getType() << " ";
13821 OS << *S;
13822}
13823
13825 const Loop *L) {
13826 // Print all inner loops first
13827 for (Loop *I : *L)
13828 PrintLoopInfo(OS, SE, I);
13829
13830 OS << "Loop ";
13831 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
13832 OS << ": ";
13833
13834 SmallVector<BasicBlock *, 8> ExitingBlocks;
13835 L->getExitingBlocks(ExitingBlocks);
13836 if (ExitingBlocks.size() != 1)
13837 OS << "<multiple exits> ";
13838
13839 auto *BTC = SE->getBackedgeTakenCount(L);
13840 if (!isa<SCEVCouldNotCompute>(BTC)) {
13841 OS << "backedge-taken count is ";
13842 PrintSCEVWithTypeHint(OS, BTC);
13843 } else
13844 OS << "Unpredictable backedge-taken count.";
13845 OS << "\n";
13846
13847 if (ExitingBlocks.size() > 1)
13848 for (BasicBlock *ExitingBlock : ExitingBlocks) {
13849 OS << " exit count for " << ExitingBlock->getName() << ": ";
13850 const SCEV *EC = SE->getExitCount(L, ExitingBlock);
13851 PrintSCEVWithTypeHint(OS, EC);
13852 if (isa<SCEVCouldNotCompute>(EC)) {
13853 // Retry with predicates.
13855 EC = SE->getPredicatedExitCount(L, ExitingBlock, &Predicates);
13856 if (!isa<SCEVCouldNotCompute>(EC)) {
13857 OS << "\n predicated exit count for " << ExitingBlock->getName()
13858 << ": ";
13859 PrintSCEVWithTypeHint(OS, EC);
13860 OS << "\n Predicates:\n";
13861 for (const auto *P : Predicates)
13862 P->print(OS, 4);
13863 }
13864 }
13865 OS << "\n";
13866 }
13867
13868 OS << "Loop ";
13869 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
13870 OS << ": ";
13871
13872 auto *ConstantBTC = SE->getConstantMaxBackedgeTakenCount(L);
13873 if (!isa<SCEVCouldNotCompute>(ConstantBTC)) {
13874 OS << "constant max backedge-taken count is ";
13875 PrintSCEVWithTypeHint(OS, ConstantBTC);
13877 OS << ", actual taken count either this or zero.";
13878 } else {
13879 OS << "Unpredictable constant max backedge-taken count. ";
13880 }
13881
13882 OS << "\n"
13883 "Loop ";
13884 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
13885 OS << ": ";
13886
13887 auto *SymbolicBTC = SE->getSymbolicMaxBackedgeTakenCount(L);
13888 if (!isa<SCEVCouldNotCompute>(SymbolicBTC)) {
13889 OS << "symbolic max backedge-taken count is ";
13890 PrintSCEVWithTypeHint(OS, SymbolicBTC);
13892 OS << ", actual taken count either this or zero.";
13893 } else {
13894 OS << "Unpredictable symbolic max backedge-taken count. ";
13895 }
13896 OS << "\n";
13897
13898 if (ExitingBlocks.size() > 1)
13899 for (BasicBlock *ExitingBlock : ExitingBlocks) {
13900 OS << " symbolic max exit count for " << ExitingBlock->getName() << ": ";
13901 auto *ExitBTC = SE->getExitCount(L, ExitingBlock,
13903 PrintSCEVWithTypeHint(OS, ExitBTC);
13904 if (isa<SCEVCouldNotCompute>(ExitBTC)) {
13905 // Retry with predicates.
13907 ExitBTC = SE->getPredicatedExitCount(L, ExitingBlock, &Predicates,
13909 if (!isa<SCEVCouldNotCompute>(ExitBTC)) {
13910 OS << "\n predicated symbolic max exit count for "
13911 << ExitingBlock->getName() << ": ";
13912 PrintSCEVWithTypeHint(OS, ExitBTC);
13913 OS << "\n Predicates:\n";
13914 for (const auto *P : Predicates)
13915 P->print(OS, 4);
13916 }
13917 }
13918 OS << "\n";
13919 }
13920
13922 auto *PBT = SE->getPredicatedBackedgeTakenCount(L, Preds);
13923 if (PBT != BTC) {
13924 assert(!Preds.empty() && "Different predicated BTC, but no predicates");
13925 OS << "Loop ";
13926 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
13927 OS << ": ";
13928 if (!isa<SCEVCouldNotCompute>(PBT)) {
13929 OS << "Predicated backedge-taken count is ";
13930 PrintSCEVWithTypeHint(OS, PBT);
13931 } else
13932 OS << "Unpredictable predicated backedge-taken count.";
13933 OS << "\n";
13934 OS << " Predicates:\n";
13935 for (const auto *P : Preds)
13936 P->print(OS, 4);
13937 }
13938 Preds.clear();
13939
13940 auto *PredConstantMax =
13942 if (PredConstantMax != ConstantBTC) {
13943 assert(!Preds.empty() &&
13944 "different predicated constant max BTC but no predicates");
13945 OS << "Loop ";
13946 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
13947 OS << ": ";
13948 if (!isa<SCEVCouldNotCompute>(PredConstantMax)) {
13949 OS << "Predicated constant max backedge-taken count is ";
13950 PrintSCEVWithTypeHint(OS, PredConstantMax);
13951 } else
13952 OS << "Unpredictable predicated constant max backedge-taken count.";
13953 OS << "\n";
13954 OS << " Predicates:\n";
13955 for (const auto *P : Preds)
13956 P->print(OS, 4);
13957 }
13958 Preds.clear();
13959
13960 auto *PredSymbolicMax =
13962 if (SymbolicBTC != PredSymbolicMax) {
13963 assert(!Preds.empty() &&
13964 "Different predicated symbolic max BTC, but no predicates");
13965 OS << "Loop ";
13966 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
13967 OS << ": ";
13968 if (!isa<SCEVCouldNotCompute>(PredSymbolicMax)) {
13969 OS << "Predicated symbolic max backedge-taken count is ";
13970 PrintSCEVWithTypeHint(OS, PredSymbolicMax);
13971 } else
13972 OS << "Unpredictable predicated symbolic max backedge-taken count.";
13973 OS << "\n";
13974 OS << " Predicates:\n";
13975 for (const auto *P : Preds)
13976 P->print(OS, 4);
13977 }
13978
13980 OS << "Loop ";
13981 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
13982 OS << ": ";
13983 OS << "Trip multiple is " << SE->getSmallConstantTripMultiple(L) << "\n";
13984 }
13985}
13986
13987namespace llvm {
13989 switch (LD) {
13991 OS << "Variant";
13992 break;
13994 OS << "Invariant";
13995 break;
13997 OS << "Computable";
13998 break;
13999 }
14000 return OS;
14001}
14002
14004 switch (BD) {
14006 OS << "DoesNotDominate";
14007 break;
14009 OS << "Dominates";
14010 break;
14012 OS << "ProperlyDominates";
14013 break;
14014 }
14015 return OS;
14016}
14017} // namespace llvm
14018
14020 // ScalarEvolution's implementation of the print method is to print
14021 // out SCEV values of all instructions that are interesting. Doing
14022 // this potentially causes it to create new SCEV objects though,
14023 // which technically conflicts with the const qualifier. This isn't
14024 // observable from outside the class though, so casting away the
14025 // const isn't dangerous.
14026 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14027
14028 if (ClassifyExpressions) {
14029 OS << "Classifying expressions for: ";
14030 F.printAsOperand(OS, /*PrintType=*/false);
14031 OS << "\n";
14032 for (Instruction &I : instructions(F))
14033 if (isSCEVable(I.getType()) && !isa<CmpInst>(I)) {
14034 OS << I << '\n';
14035 OS << " --> ";
14036 const SCEV *SV = SE.getSCEV(&I);
14037 SV->print(OS);
14038 if (!isa<SCEVCouldNotCompute>(SV)) {
14039 OS << " U: ";
14040 SE.getUnsignedRange(SV).print(OS);
14041 OS << " S: ";
14042 SE.getSignedRange(SV).print(OS);
14043 }
14044
14045 const Loop *L = LI.getLoopFor(I.getParent());
14046
14047 const SCEV *AtUse = SE.getSCEVAtScope(SV, L);
14048 if (AtUse != SV) {
14049 OS << " --> ";
14050 AtUse->print(OS);
14051 if (!isa<SCEVCouldNotCompute>(AtUse)) {
14052 OS << " U: ";
14053 SE.getUnsignedRange(AtUse).print(OS);
14054 OS << " S: ";
14055 SE.getSignedRange(AtUse).print(OS);
14056 }
14057 }
14058
14059 if (L) {
14060 OS << "\t\t" "Exits: ";
14061 const SCEV *ExitValue = SE.getSCEVAtScope(SV, L->getParentLoop());
14062 if (!SE.isLoopInvariant(ExitValue, L)) {
14063 OS << "<<Unknown>>";
14064 } else {
14065 OS << *ExitValue;
14066 }
14067
14068 bool First = true;
14069 for (const auto *Iter = L; Iter; Iter = Iter->getParentLoop()) {
14070 if (First) {
14071 OS << "\t\t" "LoopDispositions: { ";
14072 First = false;
14073 } else {
14074 OS << ", ";
14075 }
14076
14077 Iter->getHeader()->printAsOperand(OS, /*PrintType=*/false);
14078 OS << ": " << SE.getLoopDisposition(SV, Iter);
14079 }
14080
14081 for (const auto *InnerL : depth_first(L)) {
14082 if (InnerL == L)
14083 continue;
14084 if (First) {
14085 OS << "\t\t" "LoopDispositions: { ";
14086 First = false;
14087 } else {
14088 OS << ", ";
14089 }
14090
14091 InnerL->getHeader()->printAsOperand(OS, /*PrintType=*/false);
14092 OS << ": " << SE.getLoopDisposition(SV, InnerL);
14093 }
14094
14095 OS << " }";
14096 }
14097
14098 OS << "\n";
14099 }
14100 }
14101
14102 OS << "Determining loop execution counts for: ";
14103 F.printAsOperand(OS, /*PrintType=*/false);
14104 OS << "\n";
14105 for (Loop *I : LI)
14106 PrintLoopInfo(OS, &SE, I);
14107}
14108
14111 auto &Values = LoopDispositions[S];
14112 for (auto &V : Values) {
14113 if (V.getPointer() == L)
14114 return V.getInt();
14115 }
14116 Values.emplace_back(L, LoopVariant);
14117 LoopDisposition D = computeLoopDisposition(S, L);
14118 auto &Values2 = LoopDispositions[S];
14119 for (auto &V : llvm::reverse(Values2)) {
14120 if (V.getPointer() == L) {
14121 V.setInt(D);
14122 break;
14123 }
14124 }
14125 return D;
14126}
14127
14129ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) {
14130 switch (S->getSCEVType()) {
14131 case scConstant:
14132 case scVScale:
14133 return LoopInvariant;
14134 case scAddRecExpr: {
14135 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S);
14136
14137 // If L is the addrec's loop, it's computable.
14138 if (AR->getLoop() == L)
14139 return LoopComputable;
14140
14141 // Add recurrences are never invariant in the function-body (null loop).
14142 if (!L)
14143 return LoopVariant;
14144
14145 // Everything that is not defined at loop entry is variant.
14146 if (DT.dominates(L->getHeader(), AR->getLoop()->getHeader()))
14147 return LoopVariant;
14148 assert(!L->contains(AR->getLoop()) && "Containing loop's header does not"
14149 " dominate the contained loop's header?");
14150
14151 // This recurrence is invariant w.r.t. L if AR's loop contains L.
14152 if (AR->getLoop()->contains(L))
14153 return LoopInvariant;
14154
14155 // This recurrence is variant w.r.t. L if any of its operands
14156 // are variant.
14157 for (const auto *Op : AR->operands())
14158 if (!isLoopInvariant(Op, L))
14159 return LoopVariant;
14160
14161 // Otherwise it's loop-invariant.
14162 return LoopInvariant;
14163 }
14164 case scTruncate:
14165 case scZeroExtend:
14166 case scSignExtend:
14167 case scPtrToInt:
14168 case scAddExpr:
14169 case scMulExpr:
14170 case scUDivExpr:
14171 case scUMaxExpr:
14172 case scSMaxExpr:
14173 case scUMinExpr:
14174 case scSMinExpr:
14175 case scSequentialUMinExpr: {
14176 bool HasVarying = false;
14177 for (const auto *Op : S->operands()) {
14179 if (D == LoopVariant)
14180 return LoopVariant;
14181 if (D == LoopComputable)
14182 HasVarying = true;
14183 }
14184 return HasVarying ? LoopComputable : LoopInvariant;
14185 }
14186 case scUnknown:
14187 // All non-instruction values are loop invariant. All instructions are loop
14188 // invariant if they are not contained in the specified loop.
14189 // Instructions are never considered invariant in the function body
14190 // (null loop) because they are defined within the "loop".
14191 if (auto *I = dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue()))
14192 return (L && !L->contains(I)) ? LoopInvariant : LoopVariant;
14193 return LoopInvariant;
14194 case scCouldNotCompute:
14195 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14196 }
14197 llvm_unreachable("Unknown SCEV kind!");
14198}
14199
14201 return getLoopDisposition(S, L) == LoopInvariant;
14202}
14203
14205 return getLoopDisposition(S, L) == LoopComputable;
14206}
14207
14210 auto &Values = BlockDispositions[S];
14211 for (auto &V : Values) {
14212 if (V.getPointer() == BB)
14213 return V.getInt();
14214 }
14215 Values.emplace_back(BB, DoesNotDominateBlock);
14216 BlockDisposition D = computeBlockDisposition(S, BB);
14217 auto &Values2 = BlockDispositions[S];
14218 for (auto &V : llvm::reverse(Values2)) {
14219 if (V.getPointer() == BB) {
14220 V.setInt(D);
14221 break;
14222 }
14223 }
14224 return D;
14225}
14226
14228ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14229 switch (S->getSCEVType()) {
14230 case scConstant:
14231 case scVScale:
14233 case scAddRecExpr: {
14234 // This uses a "dominates" query instead of "properly dominates" query
14235 // to test for proper dominance too, because the instruction which
14236 // produces the addrec's value is a PHI, and a PHI effectively properly
14237 // dominates its entire containing block.
14238 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S);
14239 if (!DT.dominates(AR->getLoop()->getHeader(), BB))
14240 return DoesNotDominateBlock;
14241
14242 // Fall through into SCEVNAryExpr handling.
14243 [[fallthrough]];
14244 }
14245 case scTruncate:
14246 case scZeroExtend:
14247 case scSignExtend:
14248 case scPtrToInt:
14249 case scAddExpr:
14250 case scMulExpr:
14251 case scUDivExpr:
14252 case scUMaxExpr:
14253 case scSMaxExpr:
14254 case scUMinExpr:
14255 case scSMinExpr:
14256 case scSequentialUMinExpr: {
14257 bool Proper = true;
14258 for (const SCEV *NAryOp : S->operands()) {
14260 if (D == DoesNotDominateBlock)
14261 return DoesNotDominateBlock;
14262 if (D == DominatesBlock)
14263 Proper = false;
14264 }
14265 return Proper ? ProperlyDominatesBlock : DominatesBlock;
14266 }
14267 case scUnknown:
14268 if (Instruction *I =
14269 dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) {
14270 if (I->getParent() == BB)
14271 return DominatesBlock;
14272 if (DT.properlyDominates(I->getParent(), BB))
14274 return DoesNotDominateBlock;
14275 }
14277 case scCouldNotCompute:
14278 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14279 }
14280 llvm_unreachable("Unknown SCEV kind!");
14281}
14282
14283bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) {
14284 return getBlockDisposition(S, BB) >= DominatesBlock;
14285}
14286
14289}
14290
14291bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const {
14292 return SCEVExprContains(S, [&](const SCEV *Expr) { return Expr == Op; });
14293}
14294
14295void ScalarEvolution::forgetBackedgeTakenCounts(const Loop *L,
14296 bool Predicated) {
14297 auto &BECounts =
14298 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14299 auto It = BECounts.find(L);
14300 if (It != BECounts.end()) {
14301 for (const ExitNotTakenInfo &ENT : It->second.ExitNotTaken) {
14302 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14303 if (!isa<SCEVConstant>(S)) {
14304 auto UserIt = BECountUsers.find(S);
14305 assert(UserIt != BECountUsers.end());
14306 UserIt->second.erase({L, Predicated});
14307 }
14308 }
14309 }
14310 BECounts.erase(It);
14311 }
14312}
14313
14314void ScalarEvolution::forgetMemoizedResults(ArrayRef<const SCEV *> SCEVs) {
14315 SmallPtrSet<const SCEV *, 8> ToForget(llvm::from_range, SCEVs);
14316 SmallVector<const SCEV *, 8> Worklist(ToForget.begin(), ToForget.end());
14317
14318 while (!Worklist.empty()) {
14319 const SCEV *Curr = Worklist.pop_back_val();
14320 auto Users = SCEVUsers.find(Curr);
14321 if (Users != SCEVUsers.end())
14322 for (const auto *User : Users->second)
14323 if (ToForget.insert(User).second)
14324 Worklist.push_back(User);
14325 }
14326
14327 for (const auto *S : ToForget)
14328 forgetMemoizedResultsImpl(S);
14329
14330 for (auto I = PredicatedSCEVRewrites.begin();
14331 I != PredicatedSCEVRewrites.end();) {
14332 std::pair<const SCEV *, const Loop *> Entry = I->first;
14333 if (ToForget.count(Entry.first))
14334 PredicatedSCEVRewrites.erase(I++);
14335 else
14336 ++I;
14337 }
14338}
14339
14340void ScalarEvolution::forgetMemoizedResultsImpl(const SCEV *S) {
14341 LoopDispositions.erase(S);
14342 BlockDispositions.erase(S);
14343 UnsignedRanges.erase(S);
14344 SignedRanges.erase(S);
14345 HasRecMap.erase(S);
14346 ConstantMultipleCache.erase(S);
14347
14348 if (auto *AR = dyn_cast<SCEVAddRecExpr>(S)) {
14349 UnsignedWrapViaInductionTried.erase(AR);
14350 SignedWrapViaInductionTried.erase(AR);
14351 }
14352
14353 auto ExprIt = ExprValueMap.find(S);
14354 if (ExprIt != ExprValueMap.end()) {
14355 for (Value *V : ExprIt->second) {
14356 auto ValueIt = ValueExprMap.find_as(V);
14357 if (ValueIt != ValueExprMap.end())
14358 ValueExprMap.erase(ValueIt);
14359 }
14360 ExprValueMap.erase(ExprIt);
14361 }
14362
14363 auto ScopeIt = ValuesAtScopes.find(S);
14364 if (ScopeIt != ValuesAtScopes.end()) {
14365 for (const auto &Pair : ScopeIt->second)
14366 if (!isa_and_nonnull<SCEVConstant>(Pair.second))
14367 llvm::erase(ValuesAtScopesUsers[Pair.second],
14368 std::make_pair(Pair.first, S));
14369 ValuesAtScopes.erase(ScopeIt);
14370 }
14371
14372 auto ScopeUserIt = ValuesAtScopesUsers.find(S);
14373 if (ScopeUserIt != ValuesAtScopesUsers.end()) {
14374 for (const auto &Pair : ScopeUserIt->second)
14375 llvm::erase(ValuesAtScopes[Pair.second], std::make_pair(Pair.first, S));
14376 ValuesAtScopesUsers.erase(ScopeUserIt);
14377 }
14378
14379 auto BEUsersIt = BECountUsers.find(S);
14380 if (BEUsersIt != BECountUsers.end()) {
14381 // Work on a copy, as forgetBackedgeTakenCounts() will modify the original.
14382 auto Copy = BEUsersIt->second;
14383 for (const auto &Pair : Copy)
14384 forgetBackedgeTakenCounts(Pair.getPointer(), Pair.getInt());
14385 BECountUsers.erase(BEUsersIt);
14386 }
14387
14388 auto FoldUser = FoldCacheUser.find(S);
14389 if (FoldUser != FoldCacheUser.end())
14390 for (auto &KV : FoldUser->second)
14391 FoldCache.erase(KV);
14392 FoldCacheUser.erase(S);
14393}
14394
14395void
14396ScalarEvolution::getUsedLoops(const SCEV *S,
14397 SmallPtrSetImpl<const Loop *> &LoopsUsed) {
14398 struct FindUsedLoops {
14399 FindUsedLoops(SmallPtrSetImpl<const Loop *> &LoopsUsed)
14400 : LoopsUsed(LoopsUsed) {}
14401 SmallPtrSetImpl<const Loop *> &LoopsUsed;
14402 bool follow(const SCEV *S) {
14403 if (auto *AR = dyn_cast<SCEVAddRecExpr>(S))
14404 LoopsUsed.insert(AR->getLoop());
14405 return true;
14406 }
14407
14408 bool isDone() const { return false; }
14409 };
14410
14411 FindUsedLoops F(LoopsUsed);
14412 SCEVTraversal<FindUsedLoops>(F).visitAll(S);
14413}
14414
14415void ScalarEvolution::getReachableBlocks(
14418 Worklist.push_back(&F.getEntryBlock());
14419 while (!Worklist.empty()) {
14420 BasicBlock *BB = Worklist.pop_back_val();
14421 if (!Reachable.insert(BB).second)
14422 continue;
14423
14424 Value *Cond;
14425 BasicBlock *TrueBB, *FalseBB;
14426 if (match(BB->getTerminator(), m_Br(m_Value(Cond), m_BasicBlock(TrueBB),
14427 m_BasicBlock(FalseBB)))) {
14428 if (auto *C = dyn_cast<ConstantInt>(Cond)) {
14429 Worklist.push_back(C->isOne() ? TrueBB : FalseBB);
14430 continue;
14431 }
14432
14433 if (auto *Cmp = dyn_cast<ICmpInst>(Cond)) {
14434 const SCEV *L = getSCEV(Cmp->getOperand(0));
14435 const SCEV *R = getSCEV(Cmp->getOperand(1));
14436 if (isKnownPredicateViaConstantRanges(Cmp->getCmpPredicate(), L, R)) {
14437 Worklist.push_back(TrueBB);
14438 continue;
14439 }
14440 if (isKnownPredicateViaConstantRanges(Cmp->getInverseCmpPredicate(), L,
14441 R)) {
14442 Worklist.push_back(FalseBB);
14443 continue;
14444 }
14445 }
14446 }
14447
14448 append_range(Worklist, successors(BB));
14449 }
14450}
14451
14453 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14454 ScalarEvolution SE2(F, TLI, AC, DT, LI);
14455
14456 SmallVector<Loop *, 8> LoopStack(LI.begin(), LI.end());
14457
14458 // Map's SCEV expressions from one ScalarEvolution "universe" to another.
14459 struct SCEVMapper : public SCEVRewriteVisitor<SCEVMapper> {
14460 SCEVMapper(ScalarEvolution &SE) : SCEVRewriteVisitor<SCEVMapper>(SE) {}
14461
14462 const SCEV *visitConstant(const SCEVConstant *Constant) {
14463 return SE.getConstant(Constant->getAPInt());
14464 }
14465
14466 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
14467 return SE.getUnknown(Expr->getValue());
14468 }
14469
14470 const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) {
14471 return SE.getCouldNotCompute();
14472 }
14473 };
14474
14475 SCEVMapper SCM(SE2);
14476 SmallPtrSet<BasicBlock *, 16> ReachableBlocks;
14477 SE2.getReachableBlocks(ReachableBlocks, F);
14478
14479 auto GetDelta = [&](const SCEV *Old, const SCEV *New) -> const SCEV * {
14480 if (containsUndefs(Old) || containsUndefs(New)) {
14481 // SCEV treats "undef" as an unknown but consistent value (i.e. it does
14482 // not propagate undef aggressively). This means we can (and do) fail
14483 // verification in cases where a transform makes a value go from "undef"
14484 // to "undef+1" (say). The transform is fine, since in both cases the
14485 // result is "undef", but SCEV thinks the value increased by 1.
14486 return nullptr;
14487 }
14488
14489 // Unless VerifySCEVStrict is set, we only compare constant deltas.
14490 const SCEV *Delta = SE2.getMinusSCEV(Old, New);
14491 if (!VerifySCEVStrict && !isa<SCEVConstant>(Delta))
14492 return nullptr;
14493
14494 return Delta;
14495 };
14496
14497 while (!LoopStack.empty()) {
14498 auto *L = LoopStack.pop_back_val();
14499 llvm::append_range(LoopStack, *L);
14500
14501 // Only verify BECounts in reachable loops. For an unreachable loop,
14502 // any BECount is legal.
14503 if (!ReachableBlocks.contains(L->getHeader()))
14504 continue;
14505
14506 // Only verify cached BECounts. Computing new BECounts may change the
14507 // results of subsequent SCEV uses.
14508 auto It = BackedgeTakenCounts.find(L);
14509 if (It == BackedgeTakenCounts.end())
14510 continue;
14511
14512 auto *CurBECount =
14513 SCM.visit(It->second.getExact(L, const_cast<ScalarEvolution *>(this)));
14514 auto *NewBECount = SE2.getBackedgeTakenCount(L);
14515
14516 if (CurBECount == SE2.getCouldNotCompute() ||
14517 NewBECount == SE2.getCouldNotCompute()) {
14518 // NB! This situation is legal, but is very suspicious -- whatever pass
14519 // change the loop to make a trip count go from could not compute to
14520 // computable or vice-versa *should have* invalidated SCEV. However, we
14521 // choose not to assert here (for now) since we don't want false
14522 // positives.
14523 continue;
14524 }
14525
14526 if (SE.getTypeSizeInBits(CurBECount->getType()) >
14527 SE.getTypeSizeInBits(NewBECount->getType()))
14528 NewBECount = SE2.getZeroExtendExpr(NewBECount, CurBECount->getType());
14529 else if (SE.getTypeSizeInBits(CurBECount->getType()) <
14530 SE.getTypeSizeInBits(NewBECount->getType()))
14531 CurBECount = SE2.getZeroExtendExpr(CurBECount, NewBECount->getType());
14532
14533 const SCEV *Delta = GetDelta(CurBECount, NewBECount);
14534 if (Delta && !Delta->isZero()) {
14535 dbgs() << "Trip Count for " << *L << " Changed!\n";
14536 dbgs() << "Old: " << *CurBECount << "\n";
14537 dbgs() << "New: " << *NewBECount << "\n";
14538 dbgs() << "Delta: " << *Delta << "\n";
14539 std::abort();
14540 }
14541 }
14542
14543 // Collect all valid loops currently in LoopInfo.
14544 SmallPtrSet<Loop *, 32> ValidLoops;
14545 SmallVector<Loop *, 32> Worklist(LI.begin(), LI.end());
14546 while (!Worklist.empty()) {
14547 Loop *L = Worklist.pop_back_val();
14548 if (ValidLoops.insert(L).second)
14549 Worklist.append(L->begin(), L->end());
14550 }
14551 for (const auto &KV : ValueExprMap) {
14552#ifndef NDEBUG
14553 // Check for SCEV expressions referencing invalid/deleted loops.
14554 if (auto *AR = dyn_cast<SCEVAddRecExpr>(KV.second)) {
14555 assert(ValidLoops.contains(AR->getLoop()) &&
14556 "AddRec references invalid loop");
14557 }
14558#endif
14559
14560 // Check that the value is also part of the reverse map.
14561 auto It = ExprValueMap.find(KV.second);
14562 if (It == ExprValueMap.end() || !It->second.contains(KV.first)) {
14563 dbgs() << "Value " << *KV.first
14564 << " is in ValueExprMap but not in ExprValueMap\n";
14565 std::abort();
14566 }
14567
14568 if (auto *I = dyn_cast<Instruction>(&*KV.first)) {
14569 if (!ReachableBlocks.contains(I->getParent()))
14570 continue;
14571 const SCEV *OldSCEV = SCM.visit(KV.second);
14572 const SCEV *NewSCEV = SE2.getSCEV(I);
14573 const SCEV *Delta = GetDelta(OldSCEV, NewSCEV);
14574 if (Delta && !Delta->isZero()) {
14575 dbgs() << "SCEV for value " << *I << " changed!\n"
14576 << "Old: " << *OldSCEV << "\n"
14577 << "New: " << *NewSCEV << "\n"
14578 << "Delta: " << *Delta << "\n";
14579 std::abort();
14580 }
14581 }
14582 }
14583
14584 for (const auto &KV : ExprValueMap) {
14585 for (Value *V : KV.second) {
14586 const SCEV *S = ValueExprMap.lookup(V);
14587 if (!S) {
14588 dbgs() << "Value " << *V
14589 << " is in ExprValueMap but not in ValueExprMap\n";
14590 std::abort();
14591 }
14592 if (S != KV.first) {
14593 dbgs() << "Value " << *V << " mapped to " << *S << " rather than "
14594 << *KV.first << "\n";
14595 std::abort();
14596 }
14597 }
14598 }
14599
14600 // Verify integrity of SCEV users.
14601 for (const auto &S : UniqueSCEVs) {
14602 for (const auto *Op : S.operands()) {
14603 // We do not store dependencies of constants.
14604 if (isa<SCEVConstant>(Op))
14605 continue;
14606 auto It = SCEVUsers.find(Op);
14607 if (It != SCEVUsers.end() && It->second.count(&S))
14608 continue;
14609 dbgs() << "Use of operand " << *Op << " by user " << S
14610 << " is not being tracked!\n";
14611 std::abort();
14612 }
14613 }
14614
14615 // Verify integrity of ValuesAtScopes users.
14616 for (const auto &ValueAndVec : ValuesAtScopes) {
14617 const SCEV *Value = ValueAndVec.first;
14618 for (const auto &LoopAndValueAtScope : ValueAndVec.second) {
14619 const Loop *L = LoopAndValueAtScope.first;
14620 const SCEV *ValueAtScope = LoopAndValueAtScope.second;
14621 if (!isa<SCEVConstant>(ValueAtScope)) {
14622 auto It = ValuesAtScopesUsers.find(ValueAtScope);
14623 if (It != ValuesAtScopesUsers.end() &&
14624 is_contained(It->second, std::make_pair(L, Value)))
14625 continue;
14626 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14627 << *ValueAtScope << " missing in ValuesAtScopesUsers\n";
14628 std::abort();
14629 }
14630 }
14631 }
14632
14633 for (const auto &ValueAtScopeAndVec : ValuesAtScopesUsers) {
14634 const SCEV *ValueAtScope = ValueAtScopeAndVec.first;
14635 for (const auto &LoopAndValue : ValueAtScopeAndVec.second) {
14636 const Loop *L = LoopAndValue.first;
14637 const SCEV *Value = LoopAndValue.second;
14639 auto It = ValuesAtScopes.find(Value);
14640 if (It != ValuesAtScopes.end() &&
14641 is_contained(It->second, std::make_pair(L, ValueAtScope)))
14642 continue;
14643 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14644 << *ValueAtScope << " missing in ValuesAtScopes\n";
14645 std::abort();
14646 }
14647 }
14648
14649 // Verify integrity of BECountUsers.
14650 auto VerifyBECountUsers = [&](bool Predicated) {
14651 auto &BECounts =
14652 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14653 for (const auto &LoopAndBEInfo : BECounts) {
14654 for (const ExitNotTakenInfo &ENT : LoopAndBEInfo.second.ExitNotTaken) {
14655 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14656 if (!isa<SCEVConstant>(S)) {
14657 auto UserIt = BECountUsers.find(S);
14658 if (UserIt != BECountUsers.end() &&
14659 UserIt->second.contains({ LoopAndBEInfo.first, Predicated }))
14660 continue;
14661 dbgs() << "Value " << *S << " for loop " << *LoopAndBEInfo.first
14662 << " missing from BECountUsers\n";
14663 std::abort();
14664 }
14665 }
14666 }
14667 }
14668 };
14669 VerifyBECountUsers(/* Predicated */ false);
14670 VerifyBECountUsers(/* Predicated */ true);
14671
14672 // Verify intergity of loop disposition cache.
14673 for (auto &[S, Values] : LoopDispositions) {
14674 for (auto [Loop, CachedDisposition] : Values) {
14675 const auto RecomputedDisposition = SE2.getLoopDisposition(S, Loop);
14676 if (CachedDisposition != RecomputedDisposition) {
14677 dbgs() << "Cached disposition of " << *S << " for loop " << *Loop
14678 << " is incorrect: cached " << CachedDisposition << ", actual "
14679 << RecomputedDisposition << "\n";
14680 std::abort();
14681 }
14682 }
14683 }
14684
14685 // Verify integrity of the block disposition cache.
14686 for (auto &[S, Values] : BlockDispositions) {
14687 for (auto [BB, CachedDisposition] : Values) {
14688 const auto RecomputedDisposition = SE2.getBlockDisposition(S, BB);
14689 if (CachedDisposition != RecomputedDisposition) {
14690 dbgs() << "Cached disposition of " << *S << " for block %"
14691 << BB->getName() << " is incorrect: cached " << CachedDisposition
14692 << ", actual " << RecomputedDisposition << "\n";
14693 std::abort();
14694 }
14695 }
14696 }
14697
14698 // Verify FoldCache/FoldCacheUser caches.
14699 for (auto [FoldID, Expr] : FoldCache) {
14700 auto I = FoldCacheUser.find(Expr);
14701 if (I == FoldCacheUser.end()) {
14702 dbgs() << "Missing entry in FoldCacheUser for cached expression " << *Expr
14703 << "!\n";
14704 std::abort();
14705 }
14706 if (!is_contained(I->second, FoldID)) {
14707 dbgs() << "Missing FoldID in cached users of " << *Expr << "!\n";
14708 std::abort();
14709 }
14710 }
14711 for (auto [Expr, IDs] : FoldCacheUser) {
14712 for (auto &FoldID : IDs) {
14713 const SCEV *S = FoldCache.lookup(FoldID);
14714 if (!S) {
14715 dbgs() << "Missing entry in FoldCache for expression " << *Expr
14716 << "!\n";
14717 std::abort();
14718 }
14719 if (S != Expr) {
14720 dbgs() << "Entry in FoldCache doesn't match FoldCacheUser: " << *S
14721 << " != " << *Expr << "!\n";
14722 std::abort();
14723 }
14724 }
14725 }
14726
14727 // Verify that ConstantMultipleCache computations are correct. We check that
14728 // cached multiples and recomputed multiples are multiples of each other to
14729 // verify correctness. It is possible that a recomputed multiple is different
14730 // from the cached multiple due to strengthened no wrap flags or changes in
14731 // KnownBits computations.
14732 for (auto [S, Multiple] : ConstantMultipleCache) {
14733 APInt RecomputedMultiple = SE2.getConstantMultiple(S);
14734 if ((Multiple != 0 && RecomputedMultiple != 0 &&
14735 Multiple.urem(RecomputedMultiple) != 0 &&
14736 RecomputedMultiple.urem(Multiple) != 0)) {
14737 dbgs() << "Incorrect cached computation in ConstantMultipleCache for "
14738 << *S << " : Computed " << RecomputedMultiple
14739 << " but cache contains " << Multiple << "!\n";
14740 std::abort();
14741 }
14742 }
14743}
14744
14746 Function &F, const PreservedAnalyses &PA,
14747 FunctionAnalysisManager::Invalidator &Inv) {
14748 // Invalidate the ScalarEvolution object whenever it isn't preserved or one
14749 // of its dependencies is invalidated.
14750 auto PAC = PA.getChecker<ScalarEvolutionAnalysis>();
14751 return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>()) ||
14752 Inv.invalidate<AssumptionAnalysis>(F, PA) ||
14753 Inv.invalidate<DominatorTreeAnalysis>(F, PA) ||
14754 Inv.invalidate<LoopAnalysis>(F, PA);
14755}
14756
14757AnalysisKey ScalarEvolutionAnalysis::Key;
14758
14761 auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
14762 auto &AC = AM.getResult<AssumptionAnalysis>(F);
14763 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
14764 auto &LI = AM.getResult<LoopAnalysis>(F);
14765 return ScalarEvolution(F, TLI, AC, DT, LI);
14766}
14767
14773
14776 // For compatibility with opt's -analyze feature under legacy pass manager
14777 // which was not ported to NPM. This keeps tests using
14778 // update_analyze_test_checks.py working.
14779 OS << "Printing analysis 'Scalar Evolution Analysis' for function '"
14780 << F.getName() << "':\n";
14782 return PreservedAnalyses::all();
14783}
14784
14786 "Scalar Evolution Analysis", false, true)
14792 "Scalar Evolution Analysis", false, true)
14793
14795
14797
14799 SE.reset(new ScalarEvolution(
14801 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
14803 getAnalysis<LoopInfoWrapperPass>().getLoopInfo()));
14804 return false;
14805}
14806
14808
14810 SE->print(OS);
14811}
14812
14814 if (!VerifySCEV)
14815 return;
14816
14817 SE->verify();
14818}
14819
14827
14829 const SCEV *RHS) {
14830 return getComparePredicate(ICmpInst::ICMP_EQ, LHS, RHS);
14831}
14832
14833const SCEVPredicate *
14835 const SCEV *LHS, const SCEV *RHS) {
14837 assert(LHS->getType() == RHS->getType() &&
14838 "Type mismatch between LHS and RHS");
14839 // Unique this node based on the arguments
14840 ID.AddInteger(SCEVPredicate::P_Compare);
14841 ID.AddInteger(Pred);
14842 ID.AddPointer(LHS);
14843 ID.AddPointer(RHS);
14844 void *IP = nullptr;
14845 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP))
14846 return S;
14847 SCEVComparePredicate *Eq = new (SCEVAllocator)
14848 SCEVComparePredicate(ID.Intern(SCEVAllocator), Pred, LHS, RHS);
14849 UniquePreds.InsertNode(Eq, IP);
14850 return Eq;
14851}
14852
14854 const SCEVAddRecExpr *AR,
14857 // Unique this node based on the arguments
14858 ID.AddInteger(SCEVPredicate::P_Wrap);
14859 ID.AddPointer(AR);
14860 ID.AddInteger(AddedFlags);
14861 void *IP = nullptr;
14862 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP))
14863 return S;
14864 auto *OF = new (SCEVAllocator)
14865 SCEVWrapPredicate(ID.Intern(SCEVAllocator), AR, AddedFlags);
14866 UniquePreds.InsertNode(OF, IP);
14867 return OF;
14868}
14869
14870namespace {
14871
14872class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> {
14873public:
14874
14875 /// Rewrites \p S in the context of a loop L and the SCEV predication
14876 /// infrastructure.
14877 ///
14878 /// If \p Pred is non-null, the SCEV expression is rewritten to respect the
14879 /// equivalences present in \p Pred.
14880 ///
14881 /// If \p NewPreds is non-null, rewrite is free to add further predicates to
14882 /// \p NewPreds such that the result will be an AddRecExpr.
14883 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
14885 const SCEVPredicate *Pred) {
14886 SCEVPredicateRewriter Rewriter(L, SE, NewPreds, Pred);
14887 return Rewriter.visit(S);
14888 }
14889
14890 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
14891 if (Pred) {
14892 if (auto *U = dyn_cast<SCEVUnionPredicate>(Pred)) {
14893 for (const auto *Pred : U->getPredicates())
14894 if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Pred))
14895 if (IPred->getLHS() == Expr &&
14896 IPred->getPredicate() == ICmpInst::ICMP_EQ)
14897 return IPred->getRHS();
14898 } else if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Pred)) {
14899 if (IPred->getLHS() == Expr &&
14900 IPred->getPredicate() == ICmpInst::ICMP_EQ)
14901 return IPred->getRHS();
14902 }
14903 }
14904 return convertToAddRecWithPreds(Expr);
14905 }
14906
14907 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
14908 const SCEV *Operand = visit(Expr->getOperand());
14909 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Operand);
14910 if (AR && AR->getLoop() == L && AR->isAffine()) {
14911 // This couldn't be folded because the operand didn't have the nuw
14912 // flag. Add the nusw flag as an assumption that we could make.
14913 const SCEV *Step = AR->getStepRecurrence(SE);
14914 Type *Ty = Expr->getType();
14915 if (addOverflowAssumption(AR, SCEVWrapPredicate::IncrementNUSW))
14916 return SE.getAddRecExpr(SE.getZeroExtendExpr(AR->getStart(), Ty),
14917 SE.getSignExtendExpr(Step, Ty), L,
14918 AR->getNoWrapFlags());
14919 }
14920 return SE.getZeroExtendExpr(Operand, Expr->getType());
14921 }
14922
14923 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
14924 const SCEV *Operand = visit(Expr->getOperand());
14925 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Operand);
14926 if (AR && AR->getLoop() == L && AR->isAffine()) {
14927 // This couldn't be folded because the operand didn't have the nsw
14928 // flag. Add the nssw flag as an assumption that we could make.
14929 const SCEV *Step = AR->getStepRecurrence(SE);
14930 Type *Ty = Expr->getType();
14931 if (addOverflowAssumption(AR, SCEVWrapPredicate::IncrementNSSW))
14932 return SE.getAddRecExpr(SE.getSignExtendExpr(AR->getStart(), Ty),
14933 SE.getSignExtendExpr(Step, Ty), L,
14934 AR->getNoWrapFlags());
14935 }
14936 return SE.getSignExtendExpr(Operand, Expr->getType());
14937 }
14938
14939private:
14940 explicit SCEVPredicateRewriter(
14941 const Loop *L, ScalarEvolution &SE,
14942 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
14943 const SCEVPredicate *Pred)
14944 : SCEVRewriteVisitor(SE), NewPreds(NewPreds), Pred(Pred), L(L) {}
14945
14946 bool addOverflowAssumption(const SCEVPredicate *P) {
14947 if (!NewPreds) {
14948 // Check if we've already made this assumption.
14949 return Pred && Pred->implies(P, SE);
14950 }
14951 NewPreds->push_back(P);
14952 return true;
14953 }
14954
14955 bool addOverflowAssumption(const SCEVAddRecExpr *AR,
14957 auto *A = SE.getWrapPredicate(AR, AddedFlags);
14958 return addOverflowAssumption(A);
14959 }
14960
14961 // If \p Expr represents a PHINode, we try to see if it can be represented
14962 // as an AddRec, possibly under a predicate (PHISCEVPred). If it is possible
14963 // to add this predicate as a runtime overflow check, we return the AddRec.
14964 // If \p Expr does not meet these conditions (is not a PHI node, or we
14965 // couldn't create an AddRec for it, or couldn't add the predicate), we just
14966 // return \p Expr.
14967 const SCEV *convertToAddRecWithPreds(const SCEVUnknown *Expr) {
14968 if (!isa<PHINode>(Expr->getValue()))
14969 return Expr;
14970 std::optional<
14971 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
14972 PredicatedRewrite = SE.createAddRecFromPHIWithCasts(Expr);
14973 if (!PredicatedRewrite)
14974 return Expr;
14975 for (const auto *P : PredicatedRewrite->second){
14976 // Wrap predicates from outer loops are not supported.
14977 if (auto *WP = dyn_cast<const SCEVWrapPredicate>(P)) {
14978 if (L != WP->getExpr()->getLoop())
14979 return Expr;
14980 }
14981 if (!addOverflowAssumption(P))
14982 return Expr;
14983 }
14984 return PredicatedRewrite->first;
14985 }
14986
14987 SmallVectorImpl<const SCEVPredicate *> *NewPreds;
14988 const SCEVPredicate *Pred;
14989 const Loop *L;
14990};
14991
14992} // end anonymous namespace
14993
14994const SCEV *
14996 const SCEVPredicate &Preds) {
14997 return SCEVPredicateRewriter::rewrite(S, L, *this, nullptr, &Preds);
14998}
14999
15001 const SCEV *S, const Loop *L,
15004 S = SCEVPredicateRewriter::rewrite(S, L, *this, &TransformPreds, nullptr);
15005 auto *AddRec = dyn_cast<SCEVAddRecExpr>(S);
15006
15007 if (!AddRec)
15008 return nullptr;
15009
15010 // Check if any of the transformed predicates is known to be false. In that
15011 // case, it doesn't make sense to convert to a predicated AddRec, as the
15012 // versioned loop will never execute.
15013 for (const SCEVPredicate *Pred : TransformPreds) {
15014 auto *WrapPred = dyn_cast<SCEVWrapPredicate>(Pred);
15015 if (!WrapPred || WrapPred->getFlags() != SCEVWrapPredicate::IncrementNSSW)
15016 continue;
15017
15018 const SCEVAddRecExpr *AddRecToCheck = WrapPred->getExpr();
15019 const SCEV *ExitCount = getBackedgeTakenCount(AddRecToCheck->getLoop());
15020 if (isa<SCEVCouldNotCompute>(ExitCount))
15021 continue;
15022
15023 const SCEV *Step = AddRecToCheck->getStepRecurrence(*this);
15024 if (!Step->isOne())
15025 continue;
15026
15027 ExitCount = getTruncateOrSignExtend(ExitCount, Step->getType());
15028 const SCEV *Add = getAddExpr(AddRecToCheck->getStart(), ExitCount);
15029 if (isKnownPredicate(CmpInst::ICMP_SLT, Add, AddRecToCheck->getStart()))
15030 return nullptr;
15031 }
15032
15033 // Since the transformation was successful, we can now transfer the SCEV
15034 // predicates.
15035 Preds.append(TransformPreds.begin(), TransformPreds.end());
15036
15037 return AddRec;
15038}
15039
15040/// SCEV predicates
15044
15046 const ICmpInst::Predicate Pred,
15047 const SCEV *LHS, const SCEV *RHS)
15048 : SCEVPredicate(ID, P_Compare), Pred(Pred), LHS(LHS), RHS(RHS) {
15049 assert(LHS->getType() == RHS->getType() && "LHS and RHS types don't match");
15050 assert(LHS != RHS && "LHS and RHS are the same SCEV");
15051}
15052
15054 ScalarEvolution &SE) const {
15055 const auto *Op = dyn_cast<SCEVComparePredicate>(N);
15056
15057 if (!Op)
15058 return false;
15059
15060 if (Pred != ICmpInst::ICMP_EQ)
15061 return false;
15062
15063 return Op->LHS == LHS && Op->RHS == RHS;
15064}
15065
15066bool SCEVComparePredicate::isAlwaysTrue() const { return false; }
15067
15069 if (Pred == ICmpInst::ICMP_EQ)
15070 OS.indent(Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n";
15071 else
15072 OS.indent(Depth) << "Compare predicate: " << *LHS << " " << Pred << ") "
15073 << *RHS << "\n";
15074
15075}
15076
15078 const SCEVAddRecExpr *AR,
15079 IncrementWrapFlags Flags)
15080 : SCEVPredicate(ID, P_Wrap), AR(AR), Flags(Flags) {}
15081
15082const SCEVAddRecExpr *SCEVWrapPredicate::getExpr() const { return AR; }
15083
15085 ScalarEvolution &SE) const {
15086 const auto *Op = dyn_cast<SCEVWrapPredicate>(N);
15087 if (!Op || setFlags(Flags, Op->Flags) != Flags)
15088 return false;
15089
15090 if (Op->AR == AR)
15091 return true;
15092
15093 if (Flags != SCEVWrapPredicate::IncrementNSSW &&
15095 return false;
15096
15097 const SCEV *Start = AR->getStart();
15098 const SCEV *OpStart = Op->AR->getStart();
15099 if (Start->getType()->isPointerTy() != OpStart->getType()->isPointerTy())
15100 return false;
15101
15102 // Reject pointers to different address spaces.
15103 if (Start->getType()->isPointerTy() && Start->getType() != OpStart->getType())
15104 return false;
15105
15106 const SCEV *Step = AR->getStepRecurrence(SE);
15107 const SCEV *OpStep = Op->AR->getStepRecurrence(SE);
15108 if (!SE.isKnownPositive(Step) || !SE.isKnownPositive(OpStep))
15109 return false;
15110
15111 // If both steps are positive, this implies N, if N's start and step are
15112 // ULE/SLE (for NSUW/NSSW) than this'.
15113 Type *WiderTy = SE.getWiderType(Step->getType(), OpStep->getType());
15114 Step = SE.getNoopOrZeroExtend(Step, WiderTy);
15115 OpStep = SE.getNoopOrZeroExtend(OpStep, WiderTy);
15116
15117 bool IsNUW = Flags == SCEVWrapPredicate::IncrementNUSW;
15118 OpStart = IsNUW ? SE.getNoopOrZeroExtend(OpStart, WiderTy)
15119 : SE.getNoopOrSignExtend(OpStart, WiderTy);
15120 Start = IsNUW ? SE.getNoopOrZeroExtend(Start, WiderTy)
15121 : SE.getNoopOrSignExtend(Start, WiderTy);
15123 return SE.isKnownPredicate(Pred, OpStep, Step) &&
15124 SE.isKnownPredicate(Pred, OpStart, Start);
15125}
15126
15128 SCEV::NoWrapFlags ScevFlags = AR->getNoWrapFlags();
15129 IncrementWrapFlags IFlags = Flags;
15130
15131 if (ScalarEvolution::setFlags(ScevFlags, SCEV::FlagNSW) == ScevFlags)
15132 IFlags = clearFlags(IFlags, IncrementNSSW);
15133
15134 return IFlags == IncrementAnyWrap;
15135}
15136
15137void SCEVWrapPredicate::print(raw_ostream &OS, unsigned Depth) const {
15138 OS.indent(Depth) << *getExpr() << " Added Flags: ";
15140 OS << "<nusw>";
15142 OS << "<nssw>";
15143 OS << "\n";
15144}
15145
15148 ScalarEvolution &SE) {
15149 IncrementWrapFlags ImpliedFlags = IncrementAnyWrap;
15150 SCEV::NoWrapFlags StaticFlags = AR->getNoWrapFlags();
15151
15152 // We can safely transfer the NSW flag as NSSW.
15153 if (ScalarEvolution::setFlags(StaticFlags, SCEV::FlagNSW) == StaticFlags)
15154 ImpliedFlags = IncrementNSSW;
15155
15156 if (ScalarEvolution::setFlags(StaticFlags, SCEV::FlagNUW) == StaticFlags) {
15157 // If the increment is positive, the SCEV NUW flag will also imply the
15158 // WrapPredicate NUSW flag.
15159 if (const auto *Step = dyn_cast<SCEVConstant>(AR->getStepRecurrence(SE)))
15160 if (Step->getValue()->getValue().isNonNegative())
15161 ImpliedFlags = setFlags(ImpliedFlags, IncrementNUSW);
15162 }
15163
15164 return ImpliedFlags;
15165}
15166
15167/// Union predicates don't get cached so create a dummy set ID for it.
15169 ScalarEvolution &SE)
15170 : SCEVPredicate(FoldingSetNodeIDRef(nullptr, 0), P_Union) {
15171 for (const auto *P : Preds)
15172 add(P, SE);
15173}
15174
15176 return all_of(Preds,
15177 [](const SCEVPredicate *I) { return I->isAlwaysTrue(); });
15178}
15179
15181 ScalarEvolution &SE) const {
15182 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(N))
15183 return all_of(Set->Preds, [this, &SE](const SCEVPredicate *I) {
15184 return this->implies(I, SE);
15185 });
15186
15187 return any_of(Preds,
15188 [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); });
15189}
15190
15192 for (const auto *Pred : Preds)
15193 Pred->print(OS, Depth);
15194}
15195
15196void SCEVUnionPredicate::add(const SCEVPredicate *N, ScalarEvolution &SE) {
15197 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(N)) {
15198 for (const auto *Pred : Set->Preds)
15199 add(Pred, SE);
15200 return;
15201 }
15202
15203 // Implication checks are quadratic in the number of predicates. Stop doing
15204 // them if there are many predicates, as they should be too expensive to use
15205 // anyway at that point.
15206 bool CheckImplies = Preds.size() < 16;
15207
15208 // Only add predicate if it is not already implied by this union predicate.
15209 if (CheckImplies && implies(N, SE))
15210 return;
15211
15212 // Build a new vector containing the current predicates, except the ones that
15213 // are implied by the new predicate N.
15215 for (auto *P : Preds) {
15216 if (CheckImplies && N->implies(P, SE))
15217 continue;
15218 PrunedPreds.push_back(P);
15219 }
15220 Preds = std::move(PrunedPreds);
15221 Preds.push_back(N);
15222}
15223
15225 Loop &L)
15226 : SE(SE), L(L) {
15228 Preds = std::make_unique<SCEVUnionPredicate>(Empty, SE);
15229}
15230
15233 for (const auto *Op : Ops)
15234 // We do not expect that forgetting cached data for SCEVConstants will ever
15235 // open any prospects for sharpening or introduce any correctness issues,
15236 // so we don't bother storing their dependencies.
15237 if (!isa<SCEVConstant>(Op))
15238 SCEVUsers[Op].insert(User);
15239}
15240
15242 const SCEV *Expr = SE.getSCEV(V);
15243 RewriteEntry &Entry = RewriteMap[Expr];
15244
15245 // If we already have an entry and the version matches, return it.
15246 if (Entry.second && Generation == Entry.first)
15247 return Entry.second;
15248
15249 // We found an entry but it's stale. Rewrite the stale entry
15250 // according to the current predicate.
15251 if (Entry.second)
15252 Expr = Entry.second;
15253
15254 const SCEV *NewSCEV = SE.rewriteUsingPredicate(Expr, &L, *Preds);
15255 Entry = {Generation, NewSCEV};
15256
15257 return NewSCEV;
15258}
15259
15261 if (!BackedgeCount) {
15263 BackedgeCount = SE.getPredicatedBackedgeTakenCount(&L, Preds);
15264 for (const auto *P : Preds)
15265 addPredicate(*P);
15266 }
15267 return BackedgeCount;
15268}
15269
15271 if (!SymbolicMaxBackedgeCount) {
15273 SymbolicMaxBackedgeCount =
15274 SE.getPredicatedSymbolicMaxBackedgeTakenCount(&L, Preds);
15275 for (const auto *P : Preds)
15276 addPredicate(*P);
15277 }
15278 return SymbolicMaxBackedgeCount;
15279}
15280
15282 if (!SmallConstantMaxTripCount) {
15284 SmallConstantMaxTripCount = SE.getSmallConstantMaxTripCount(&L, &Preds);
15285 for (const auto *P : Preds)
15286 addPredicate(*P);
15287 }
15288 return *SmallConstantMaxTripCount;
15289}
15290
15292 if (Preds->implies(&Pred, SE))
15293 return;
15294
15295 SmallVector<const SCEVPredicate *, 4> NewPreds(Preds->getPredicates());
15296 NewPreds.push_back(&Pred);
15297 Preds = std::make_unique<SCEVUnionPredicate>(NewPreds, SE);
15298 updateGeneration();
15299}
15300
15302 return *Preds;
15303}
15304
15305void PredicatedScalarEvolution::updateGeneration() {
15306 // If the generation number wrapped recompute everything.
15307 if (++Generation == 0) {
15308 for (auto &II : RewriteMap) {
15309 const SCEV *Rewritten = II.second.second;
15310 II.second = {Generation, SE.rewriteUsingPredicate(Rewritten, &L, *Preds)};
15311 }
15312 }
15313}
15314
15317 const SCEV *Expr = getSCEV(V);
15318 const auto *AR = cast<SCEVAddRecExpr>(Expr);
15319
15320 auto ImpliedFlags = SCEVWrapPredicate::getImpliedFlags(AR, SE);
15321
15322 // Clear the statically implied flags.
15323 Flags = SCEVWrapPredicate::clearFlags(Flags, ImpliedFlags);
15324 addPredicate(*SE.getWrapPredicate(AR, Flags));
15325
15326 auto II = FlagsMap.insert({V, Flags});
15327 if (!II.second)
15328 II.first->second = SCEVWrapPredicate::setFlags(Flags, II.first->second);
15329}
15330
15333 const SCEV *Expr = getSCEV(V);
15334 const auto *AR = cast<SCEVAddRecExpr>(Expr);
15335
15337 Flags, SCEVWrapPredicate::getImpliedFlags(AR, SE));
15338
15339 auto II = FlagsMap.find(V);
15340
15341 if (II != FlagsMap.end())
15342 Flags = SCEVWrapPredicate::clearFlags(Flags, II->second);
15343
15345}
15346
15348 const SCEV *Expr = this->getSCEV(V);
15350 auto *New = SE.convertSCEVToAddRecWithPredicates(Expr, &L, NewPreds);
15351
15352 if (!New)
15353 return nullptr;
15354
15355 for (const auto *P : NewPreds)
15356 addPredicate(*P);
15357
15358 RewriteMap[SE.getSCEV(V)] = {Generation, New};
15359 return New;
15360}
15361
15364 : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L),
15365 Preds(std::make_unique<SCEVUnionPredicate>(Init.Preds->getPredicates(),
15366 SE)),
15367 Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) {
15368 for (auto I : Init.FlagsMap)
15369 FlagsMap.insert(I);
15370}
15371
15373 // For each block.
15374 for (auto *BB : L.getBlocks())
15375 for (auto &I : *BB) {
15376 if (!SE.isSCEVable(I.getType()))
15377 continue;
15378
15379 auto *Expr = SE.getSCEV(&I);
15380 auto II = RewriteMap.find(Expr);
15381
15382 if (II == RewriteMap.end())
15383 continue;
15384
15385 // Don't print things that are not interesting.
15386 if (II->second.second == Expr)
15387 continue;
15388
15389 OS.indent(Depth) << "[PSE]" << I << ":\n";
15390 OS.indent(Depth + 2) << *Expr << "\n";
15391 OS.indent(Depth + 2) << "--> " << *II->second.second << "\n";
15392 }
15393}
15394
15397 BasicBlock *Header = L->getHeader();
15398 BasicBlock *Pred = L->getLoopPredecessor();
15399 LoopGuards Guards(SE);
15400 if (!Pred)
15401 return Guards;
15403 collectFromBlock(SE, Guards, Header, Pred, VisitedBlocks);
15404 return Guards;
15405}
15406
15407void ScalarEvolution::LoopGuards::collectFromPHI(
15411 unsigned Depth) {
15412 if (!SE.isSCEVable(Phi.getType()))
15413 return;
15414
15415 using MinMaxPattern = std::pair<const SCEVConstant *, SCEVTypes>;
15416 auto GetMinMaxConst = [&](unsigned IncomingIdx) -> MinMaxPattern {
15417 const BasicBlock *InBlock = Phi.getIncomingBlock(IncomingIdx);
15418 if (!VisitedBlocks.insert(InBlock).second)
15419 return {nullptr, scCouldNotCompute};
15420
15421 // Avoid analyzing unreachable blocks so that we don't get trapped
15422 // traversing cycles with ill-formed dominance or infinite cycles
15423 if (!SE.DT.isReachableFromEntry(InBlock))
15424 return {nullptr, scCouldNotCompute};
15425
15426 auto [G, Inserted] = IncomingGuards.try_emplace(InBlock, LoopGuards(SE));
15427 if (Inserted)
15428 collectFromBlock(SE, G->second, Phi.getParent(), InBlock, VisitedBlocks,
15429 Depth + 1);
15430 auto &RewriteMap = G->second.RewriteMap;
15431 if (RewriteMap.empty())
15432 return {nullptr, scCouldNotCompute};
15433 auto S = RewriteMap.find(SE.getSCEV(Phi.getIncomingValue(IncomingIdx)));
15434 if (S == RewriteMap.end())
15435 return {nullptr, scCouldNotCompute};
15436 auto *SM = dyn_cast_if_present<SCEVMinMaxExpr>(S->second);
15437 if (!SM)
15438 return {nullptr, scCouldNotCompute};
15439 if (const SCEVConstant *C0 = dyn_cast<SCEVConstant>(SM->getOperand(0)))
15440 return {C0, SM->getSCEVType()};
15441 return {nullptr, scCouldNotCompute};
15442 };
15443 auto MergeMinMaxConst = [](MinMaxPattern P1,
15444 MinMaxPattern P2) -> MinMaxPattern {
15445 auto [C1, T1] = P1;
15446 auto [C2, T2] = P2;
15447 if (!C1 || !C2 || T1 != T2)
15448 return {nullptr, scCouldNotCompute};
15449 switch (T1) {
15450 case scUMaxExpr:
15451 return {C1->getAPInt().ult(C2->getAPInt()) ? C1 : C2, T1};
15452 case scSMaxExpr:
15453 return {C1->getAPInt().slt(C2->getAPInt()) ? C1 : C2, T1};
15454 case scUMinExpr:
15455 return {C1->getAPInt().ugt(C2->getAPInt()) ? C1 : C2, T1};
15456 case scSMinExpr:
15457 return {C1->getAPInt().sgt(C2->getAPInt()) ? C1 : C2, T1};
15458 default:
15459 llvm_unreachable("Trying to merge non-MinMaxExpr SCEVs.");
15460 }
15461 };
15462 auto P = GetMinMaxConst(0);
15463 for (unsigned int In = 1; In < Phi.getNumIncomingValues(); In++) {
15464 if (!P.first)
15465 break;
15466 P = MergeMinMaxConst(P, GetMinMaxConst(In));
15467 }
15468 if (P.first) {
15469 const SCEV *LHS = SE.getSCEV(const_cast<PHINode *>(&Phi));
15471 const SCEV *RHS = SE.getMinMaxExpr(P.second, Ops);
15472 Guards.RewriteMap.insert({LHS, RHS});
15473 }
15474}
15475
15476// Return a new SCEV that modifies \p Expr to the closest number divides by
15477// \p Divisor and less or equal than Expr. For now, only handle constant
15478// Expr.
15480 const APInt &DivisorVal,
15481 ScalarEvolution &SE) {
15482 const APInt *ExprVal;
15483 if (!match(Expr, m_scev_APInt(ExprVal)) || ExprVal->isNegative() ||
15484 DivisorVal.isNonPositive())
15485 return Expr;
15486 APInt Rem = ExprVal->urem(DivisorVal);
15487 // return the SCEV: Expr - Expr % Divisor
15488 return SE.getConstant(*ExprVal - Rem);
15489}
15490
15491// Return a new SCEV that modifies \p Expr to the closest number divides by
15492// \p Divisor and greater or equal than Expr. For now, only handle constant
15493// Expr.
15494static const SCEV *getNextSCEVDivisibleByDivisor(const SCEV *Expr,
15495 const APInt &DivisorVal,
15496 ScalarEvolution &SE) {
15497 const APInt *ExprVal;
15498 if (!match(Expr, m_scev_APInt(ExprVal)) || ExprVal->isNegative() ||
15499 DivisorVal.isNonPositive())
15500 return Expr;
15501 APInt Rem = ExprVal->urem(DivisorVal);
15502 if (Rem.isZero())
15503 return Expr;
15504 // return the SCEV: Expr + Divisor - Expr % Divisor
15505 return SE.getConstant(*ExprVal + DivisorVal - Rem);
15506}
15507
15508void ScalarEvolution::LoopGuards::collectFromBlock(
15509 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15510 const BasicBlock *Block, const BasicBlock *Pred,
15511 SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks, unsigned Depth) {
15512
15514
15515 SmallVector<const SCEV *> ExprsToRewrite;
15516 auto CollectCondition = [&](ICmpInst::Predicate Predicate, const SCEV *LHS,
15517 const SCEV *RHS,
15518 DenseMap<const SCEV *, const SCEV *>
15519 &RewriteMap) {
15520 // WARNING: It is generally unsound to apply any wrap flags to the proposed
15521 // replacement SCEV which isn't directly implied by the structure of that
15522 // SCEV. In particular, using contextual facts to imply flags is *NOT*
15523 // legal. See the scoping rules for flags in the header to understand why.
15524
15525 // If LHS is a constant, apply information to the other expression.
15526 if (isa<SCEVConstant>(LHS)) {
15527 std::swap(LHS, RHS);
15529 }
15530
15531 // Check for a condition of the form (-C1 + X < C2). InstCombine will
15532 // create this form when combining two checks of the form (X u< C2 + C1) and
15533 // (X >=u C1).
15534 auto MatchRangeCheckIdiom = [&SE, Predicate, LHS, RHS, &RewriteMap,
15535 &ExprsToRewrite]() {
15536 const SCEVConstant *C1;
15537 const SCEVUnknown *LHSUnknown;
15538 auto *C2 = dyn_cast<SCEVConstant>(RHS);
15539 if (!match(LHS,
15540 m_scev_Add(m_SCEVConstant(C1), m_SCEVUnknown(LHSUnknown))) ||
15541 !C2)
15542 return false;
15543
15544 auto ExactRegion =
15545 ConstantRange::makeExactICmpRegion(Predicate, C2->getAPInt())
15546 .sub(C1->getAPInt());
15547
15548 // Bail out, unless we have a non-wrapping, monotonic range.
15549 if (ExactRegion.isWrappedSet() || ExactRegion.isFullSet())
15550 return false;
15551 auto [I, Inserted] = RewriteMap.try_emplace(LHSUnknown);
15552 const SCEV *RewrittenLHS = Inserted ? LHSUnknown : I->second;
15553 I->second = SE.getUMaxExpr(
15554 SE.getConstant(ExactRegion.getUnsignedMin()),
15555 SE.getUMinExpr(RewrittenLHS,
15556 SE.getConstant(ExactRegion.getUnsignedMax())));
15557 ExprsToRewrite.push_back(LHSUnknown);
15558 return true;
15559 };
15560 if (MatchRangeCheckIdiom())
15561 return;
15562
15563 // Return true if \p Expr is a MinMax SCEV expression with a non-negative
15564 // constant operand. If so, return in \p SCTy the SCEV type and in \p RHS
15565 // the non-constant operand and in \p LHS the constant operand.
15566 auto IsMinMaxSCEVWithNonNegativeConstant =
15567 [&](const SCEV *Expr, SCEVTypes &SCTy, const SCEV *&LHS,
15568 const SCEV *&RHS) {
15569 const APInt *C;
15570 SCTy = Expr->getSCEVType();
15571 return match(Expr, m_scev_MinMax(m_SCEV(LHS), m_SCEV(RHS))) &&
15572 match(LHS, m_scev_APInt(C)) && C->isNonNegative();
15573 };
15574
15575 // Apply divisibilty by \p Divisor on MinMaxExpr with constant values,
15576 // recursively. This is done by aligning up/down the constant value to the
15577 // Divisor.
15578 std::function<const SCEV *(const SCEV *, const SCEV *)>
15579 ApplyDivisibiltyOnMinMaxExpr = [&](const SCEV *MinMaxExpr,
15580 const SCEV *Divisor) {
15581 auto *ConstDivisor = dyn_cast<SCEVConstant>(Divisor);
15582 if (!ConstDivisor)
15583 return MinMaxExpr;
15584 const APInt &DivisorVal = ConstDivisor->getAPInt();
15585
15586 const SCEV *MinMaxLHS = nullptr, *MinMaxRHS = nullptr;
15587 SCEVTypes SCTy;
15588 if (!IsMinMaxSCEVWithNonNegativeConstant(MinMaxExpr, SCTy, MinMaxLHS,
15589 MinMaxRHS))
15590 return MinMaxExpr;
15591 auto IsMin =
15592 isa<SCEVSMinExpr>(MinMaxExpr) || isa<SCEVUMinExpr>(MinMaxExpr);
15593 assert(SE.isKnownNonNegative(MinMaxLHS) &&
15594 "Expected non-negative operand!");
15595 auto *DivisibleExpr =
15596 IsMin
15597 ? getPreviousSCEVDivisibleByDivisor(MinMaxLHS, DivisorVal, SE)
15598 : getNextSCEVDivisibleByDivisor(MinMaxLHS, DivisorVal, SE);
15600 ApplyDivisibiltyOnMinMaxExpr(MinMaxRHS, Divisor), DivisibleExpr};
15601 return SE.getMinMaxExpr(SCTy, Ops);
15602 };
15603
15604 // If we have LHS == 0, check if LHS is computing a property of some unknown
15605 // SCEV %v which we can rewrite %v to express explicitly.
15606 if (Predicate == CmpInst::ICMP_EQ && match(RHS, m_scev_Zero())) {
15607 // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to
15608 // explicitly express that.
15609 const SCEVUnknown *URemLHS = nullptr;
15610 const SCEV *URemRHS = nullptr;
15611 if (match(LHS,
15612 m_scev_URem(m_SCEVUnknown(URemLHS), m_SCEV(URemRHS), SE))) {
15613 auto I = RewriteMap.find(URemLHS);
15614 const SCEV *RewrittenLHS = I != RewriteMap.end() ? I->second : URemLHS;
15615 RewrittenLHS = ApplyDivisibiltyOnMinMaxExpr(RewrittenLHS, URemRHS);
15616 const auto *Multiple =
15617 SE.getMulExpr(SE.getUDivExpr(RewrittenLHS, URemRHS), URemRHS);
15618 RewriteMap[URemLHS] = Multiple;
15619 ExprsToRewrite.push_back(URemLHS);
15620 return;
15621 }
15622 }
15623
15624 // Do not apply information for constants or if RHS contains an AddRec.
15626 return;
15627
15628 // If RHS is SCEVUnknown, make sure the information is applied to it.
15630 std::swap(LHS, RHS);
15632 }
15633
15634 // Puts rewrite rule \p From -> \p To into the rewrite map. Also if \p From
15635 // and \p FromRewritten are the same (i.e. there has been no rewrite
15636 // registered for \p From), then puts this value in the list of rewritten
15637 // expressions.
15638 auto AddRewrite = [&](const SCEV *From, const SCEV *FromRewritten,
15639 const SCEV *To) {
15640 if (From == FromRewritten)
15641 ExprsToRewrite.push_back(From);
15642 RewriteMap[From] = To;
15643 };
15644
15645 // Checks whether \p S has already been rewritten. In that case returns the
15646 // existing rewrite because we want to chain further rewrites onto the
15647 // already rewritten value. Otherwise returns \p S.
15648 auto GetMaybeRewritten = [&](const SCEV *S) {
15649 return RewriteMap.lookup_or(S, S);
15650 };
15651
15652 const SCEV *RewrittenLHS = GetMaybeRewritten(LHS);
15653 const APInt &DividesBy = SE.getConstantMultiple(RewrittenLHS);
15654
15655 // Collect rewrites for LHS and its transitive operands based on the
15656 // condition.
15657 // For min/max expressions, also apply the guard to its operands:
15658 // 'min(a, b) >= c' -> '(a >= c) and (b >= c)',
15659 // 'min(a, b) > c' -> '(a > c) and (b > c)',
15660 // 'max(a, b) <= c' -> '(a <= c) and (b <= c)',
15661 // 'max(a, b) < c' -> '(a < c) and (b < c)'.
15662
15663 // We cannot express strict predicates in SCEV, so instead we replace them
15664 // with non-strict ones against plus or minus one of RHS depending on the
15665 // predicate.
15666 const SCEV *One = SE.getOne(RHS->getType());
15667 switch (Predicate) {
15668 case CmpInst::ICMP_ULT:
15669 if (RHS->getType()->isPointerTy())
15670 return;
15671 RHS = SE.getUMaxExpr(RHS, One);
15672 [[fallthrough]];
15673 case CmpInst::ICMP_SLT: {
15674 RHS = SE.getMinusSCEV(RHS, One);
15675 RHS = getPreviousSCEVDivisibleByDivisor(RHS, DividesBy, SE);
15676 break;
15677 }
15678 case CmpInst::ICMP_UGT:
15679 case CmpInst::ICMP_SGT:
15680 RHS = SE.getAddExpr(RHS, One);
15681 RHS = getNextSCEVDivisibleByDivisor(RHS, DividesBy, SE);
15682 break;
15683 case CmpInst::ICMP_ULE:
15684 case CmpInst::ICMP_SLE:
15685 RHS = getPreviousSCEVDivisibleByDivisor(RHS, DividesBy, SE);
15686 break;
15687 case CmpInst::ICMP_UGE:
15688 case CmpInst::ICMP_SGE:
15689 RHS = getNextSCEVDivisibleByDivisor(RHS, DividesBy, SE);
15690 break;
15691 default:
15692 break;
15693 }
15694
15696 SmallPtrSet<const SCEV *, 16> Visited;
15697
15698 auto EnqueueOperands = [&Worklist](const SCEVNAryExpr *S) {
15699 append_range(Worklist, S->operands());
15700 };
15701
15702 while (!Worklist.empty()) {
15703 const SCEV *From = Worklist.pop_back_val();
15704 if (isa<SCEVConstant>(From))
15705 continue;
15706 if (!Visited.insert(From).second)
15707 continue;
15708 const SCEV *FromRewritten = GetMaybeRewritten(From);
15709 const SCEV *To = nullptr;
15710
15711 switch (Predicate) {
15712 case CmpInst::ICMP_ULT:
15713 case CmpInst::ICMP_ULE:
15714 To = SE.getUMinExpr(FromRewritten, RHS);
15715 if (auto *UMax = dyn_cast<SCEVUMaxExpr>(FromRewritten))
15716 EnqueueOperands(UMax);
15717 break;
15718 case CmpInst::ICMP_SLT:
15719 case CmpInst::ICMP_SLE:
15720 To = SE.getSMinExpr(FromRewritten, RHS);
15721 if (auto *SMax = dyn_cast<SCEVSMaxExpr>(FromRewritten))
15722 EnqueueOperands(SMax);
15723 break;
15724 case CmpInst::ICMP_UGT:
15725 case CmpInst::ICMP_UGE:
15726 To = SE.getUMaxExpr(FromRewritten, RHS);
15727 if (auto *UMin = dyn_cast<SCEVUMinExpr>(FromRewritten))
15728 EnqueueOperands(UMin);
15729 break;
15730 case CmpInst::ICMP_SGT:
15731 case CmpInst::ICMP_SGE:
15732 To = SE.getSMaxExpr(FromRewritten, RHS);
15733 if (auto *SMin = dyn_cast<SCEVSMinExpr>(FromRewritten))
15734 EnqueueOperands(SMin);
15735 break;
15736 case CmpInst::ICMP_EQ:
15738 To = RHS;
15739 break;
15740 case CmpInst::ICMP_NE:
15741 if (match(RHS, m_scev_Zero())) {
15742 const SCEV *OneAlignedUp =
15743 getNextSCEVDivisibleByDivisor(One, DividesBy, SE);
15744 To = SE.getUMaxExpr(FromRewritten, OneAlignedUp);
15745 } else {
15746 // LHS != RHS can be rewritten as (LHS - RHS) = UMax(1, LHS - RHS),
15747 // but creating the subtraction eagerly is expensive. Track the
15748 // inequalities in a separate map, and materialize the rewrite lazily
15749 // when encountering a suitable subtraction while re-writing.
15750 if (LHS->getType()->isPointerTy()) {
15754 break;
15755 }
15756 const SCEVConstant *C;
15757 const SCEV *A, *B;
15760 RHS = A;
15761 LHS = B;
15762 }
15763 if (LHS > RHS)
15764 std::swap(LHS, RHS);
15765 Guards.NotEqual.insert({LHS, RHS});
15766 continue;
15767 }
15768 break;
15769 default:
15770 break;
15771 }
15772
15773 if (To)
15774 AddRewrite(From, FromRewritten, To);
15775 }
15776 };
15777
15779 // First, collect information from assumptions dominating the loop.
15780 for (auto &AssumeVH : SE.AC.assumptions()) {
15781 if (!AssumeVH)
15782 continue;
15783 auto *AssumeI = cast<CallInst>(AssumeVH);
15784 if (!SE.DT.dominates(AssumeI, Block))
15785 continue;
15786 Terms.emplace_back(AssumeI->getOperand(0), true);
15787 }
15788
15789 // Second, collect information from llvm.experimental.guards dominating the loop.
15790 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
15791 SE.F.getParent(), Intrinsic::experimental_guard);
15792 if (GuardDecl)
15793 for (const auto *GU : GuardDecl->users())
15794 if (const auto *Guard = dyn_cast<IntrinsicInst>(GU))
15795 if (Guard->getFunction() == Block->getParent() &&
15796 SE.DT.dominates(Guard, Block))
15797 Terms.emplace_back(Guard->getArgOperand(0), true);
15798
15799 // Third, collect conditions from dominating branches. Starting at the loop
15800 // predecessor, climb up the predecessor chain, as long as there are
15801 // predecessors that can be found that have unique successors leading to the
15802 // original header.
15803 // TODO: share this logic with isLoopEntryGuardedByCond.
15804 unsigned NumCollectedConditions = 0;
15806 std::pair<const BasicBlock *, const BasicBlock *> Pair(Pred, Block);
15807 for (; Pair.first;
15808 Pair = SE.getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
15809 VisitedBlocks.insert(Pair.second);
15810 const BranchInst *LoopEntryPredicate =
15811 dyn_cast<BranchInst>(Pair.first->getTerminator());
15812 if (!LoopEntryPredicate || LoopEntryPredicate->isUnconditional())
15813 continue;
15814
15815 Terms.emplace_back(LoopEntryPredicate->getCondition(),
15816 LoopEntryPredicate->getSuccessor(0) == Pair.second);
15817 NumCollectedConditions++;
15818
15819 // If we are recursively collecting guards stop after 2
15820 // conditions to limit compile-time impact for now.
15821 if (Depth > 0 && NumCollectedConditions == 2)
15822 break;
15823 }
15824 // Finally, if we stopped climbing the predecessor chain because
15825 // there wasn't a unique one to continue, try to collect conditions
15826 // for PHINodes by recursively following all of their incoming
15827 // blocks and try to merge the found conditions to build a new one
15828 // for the Phi.
15829 if (Pair.second->hasNPredecessorsOrMore(2) &&
15831 SmallDenseMap<const BasicBlock *, LoopGuards> IncomingGuards;
15832 for (auto &Phi : Pair.second->phis())
15833 collectFromPHI(SE, Guards, Phi, VisitedBlocks, IncomingGuards, Depth);
15834 }
15835
15836 // Now apply the information from the collected conditions to
15837 // Guards.RewriteMap. Conditions are processed in reverse order, so the
15838 // earliest conditions is processed first. This ensures the SCEVs with the
15839 // shortest dependency chains are constructed first.
15840 for (auto [Term, EnterIfTrue] : reverse(Terms)) {
15841 SmallVector<Value *, 8> Worklist;
15842 SmallPtrSet<Value *, 8> Visited;
15843 Worklist.push_back(Term);
15844 while (!Worklist.empty()) {
15845 Value *Cond = Worklist.pop_back_val();
15846 if (!Visited.insert(Cond).second)
15847 continue;
15848
15849 if (auto *Cmp = dyn_cast<ICmpInst>(Cond)) {
15850 auto Predicate =
15851 EnterIfTrue ? Cmp->getPredicate() : Cmp->getInversePredicate();
15852 const auto *LHS = SE.getSCEV(Cmp->getOperand(0));
15853 const auto *RHS = SE.getSCEV(Cmp->getOperand(1));
15854 CollectCondition(Predicate, LHS, RHS, Guards.RewriteMap);
15855 continue;
15856 }
15857
15858 Value *L, *R;
15859 if (EnterIfTrue ? match(Cond, m_LogicalAnd(m_Value(L), m_Value(R)))
15860 : match(Cond, m_LogicalOr(m_Value(L), m_Value(R)))) {
15861 Worklist.push_back(L);
15862 Worklist.push_back(R);
15863 }
15864 }
15865 }
15866
15867 // Let the rewriter preserve NUW/NSW flags if the unsigned/signed ranges of
15868 // the replacement expressions are contained in the ranges of the replaced
15869 // expressions.
15870 Guards.PreserveNUW = true;
15871 Guards.PreserveNSW = true;
15872 for (const SCEV *Expr : ExprsToRewrite) {
15873 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
15874 Guards.PreserveNUW &=
15875 SE.getUnsignedRange(Expr).contains(SE.getUnsignedRange(RewriteTo));
15876 Guards.PreserveNSW &=
15877 SE.getSignedRange(Expr).contains(SE.getSignedRange(RewriteTo));
15878 }
15879
15880 // Now that all rewrite information is collect, rewrite the collected
15881 // expressions with the information in the map. This applies information to
15882 // sub-expressions.
15883 if (ExprsToRewrite.size() > 1) {
15884 for (const SCEV *Expr : ExprsToRewrite) {
15885 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
15886 Guards.RewriteMap.erase(Expr);
15887 Guards.RewriteMap.insert({Expr, Guards.rewrite(RewriteTo)});
15888 }
15889 }
15890}
15891
15893 /// A rewriter to replace SCEV expressions in Map with the corresponding entry
15894 /// in the map. It skips AddRecExpr because we cannot guarantee that the
15895 /// replacement is loop invariant in the loop of the AddRec.
15896 class SCEVLoopGuardRewriter
15897 : public SCEVRewriteVisitor<SCEVLoopGuardRewriter> {
15900
15902
15903 public:
15904 SCEVLoopGuardRewriter(ScalarEvolution &SE,
15905 const ScalarEvolution::LoopGuards &Guards)
15906 : SCEVRewriteVisitor(SE), Map(Guards.RewriteMap),
15907 NotEqual(Guards.NotEqual) {
15908 if (Guards.PreserveNUW)
15909 FlagMask = ScalarEvolution::setFlags(FlagMask, SCEV::FlagNUW);
15910 if (Guards.PreserveNSW)
15911 FlagMask = ScalarEvolution::setFlags(FlagMask, SCEV::FlagNSW);
15912 }
15913
15914 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
15915
15916 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
15917 return Map.lookup_or(Expr, Expr);
15918 }
15919
15920 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
15921 if (const SCEV *S = Map.lookup(Expr))
15922 return S;
15923
15924 // If we didn't find the extact ZExt expr in the map, check if there's
15925 // an entry for a smaller ZExt we can use instead.
15926 Type *Ty = Expr->getType();
15927 const SCEV *Op = Expr->getOperand(0);
15928 unsigned Bitwidth = Ty->getScalarSizeInBits() / 2;
15929 while (Bitwidth % 8 == 0 && Bitwidth >= 8 &&
15930 Bitwidth > Op->getType()->getScalarSizeInBits()) {
15931 Type *NarrowTy = IntegerType::get(SE.getContext(), Bitwidth);
15932 auto *NarrowExt = SE.getZeroExtendExpr(Op, NarrowTy);
15933 if (const SCEV *S = Map.lookup(NarrowExt))
15934 return SE.getZeroExtendExpr(S, Ty);
15935 Bitwidth = Bitwidth / 2;
15936 }
15937
15939 Expr);
15940 }
15941
15942 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
15943 if (const SCEV *S = Map.lookup(Expr))
15944 return S;
15946 Expr);
15947 }
15948
15949 const SCEV *visitUMinExpr(const SCEVUMinExpr *Expr) {
15950 if (const SCEV *S = Map.lookup(Expr))
15951 return S;
15953 }
15954
15955 const SCEV *visitSMinExpr(const SCEVSMinExpr *Expr) {
15956 if (const SCEV *S = Map.lookup(Expr))
15957 return S;
15959 }
15960
15961 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
15962 // Helper to check if S is a subtraction (A - B) where A != B, and if so,
15963 // return UMax(S, 1).
15964 auto RewriteSubtraction = [&](const SCEV *S) -> const SCEV * {
15965 const SCEV *LHS, *RHS;
15966 if (MatchBinarySub(S, LHS, RHS)) {
15967 if (LHS > RHS)
15968 std::swap(LHS, RHS);
15969 if (NotEqual.contains({LHS, RHS})) {
15970 const SCEV *OneAlignedUp = getNextSCEVDivisibleByDivisor(
15971 SE.getOne(S->getType()), SE.getConstantMultiple(S), SE);
15972 return SE.getUMaxExpr(OneAlignedUp, S);
15973 }
15974 }
15975 return nullptr;
15976 };
15977
15978 // Check if Expr itself is a subtraction pattern with guard info.
15979 if (const SCEV *Rewritten = RewriteSubtraction(Expr))
15980 return Rewritten;
15981
15982 // Trip count expressions sometimes consist of adding 3 operands, i.e.
15983 // (Const + A + B). There may be guard info for A + B, and if so, apply
15984 // it.
15985 // TODO: Could more generally apply guards to Add sub-expressions.
15986 if (isa<SCEVConstant>(Expr->getOperand(0)) &&
15987 Expr->getNumOperands() == 3) {
15988 const SCEV *Add =
15989 SE.getAddExpr(Expr->getOperand(1), Expr->getOperand(2));
15990 if (const SCEV *Rewritten = RewriteSubtraction(Add))
15991 return SE.getAddExpr(
15992 Expr->getOperand(0), Rewritten,
15993 ScalarEvolution::maskFlags(Expr->getNoWrapFlags(), FlagMask));
15994 if (const SCEV *S = Map.lookup(Add))
15995 return SE.getAddExpr(Expr->getOperand(0), S);
15996 }
15998 bool Changed = false;
15999 for (const auto *Op : Expr->operands()) {
16000 Operands.push_back(
16002 Changed |= Op != Operands.back();
16003 }
16004 // We are only replacing operands with equivalent values, so transfer the
16005 // flags from the original expression.
16006 return !Changed ? Expr
16007 : SE.getAddExpr(Operands,
16009 Expr->getNoWrapFlags(), FlagMask));
16010 }
16011
16012 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
16014 bool Changed = false;
16015 for (const auto *Op : Expr->operands()) {
16016 Operands.push_back(
16018 Changed |= Op != Operands.back();
16019 }
16020 // We are only replacing operands with equivalent values, so transfer the
16021 // flags from the original expression.
16022 return !Changed ? Expr
16023 : SE.getMulExpr(Operands,
16025 Expr->getNoWrapFlags(), FlagMask));
16026 }
16027 };
16028
16029 if (RewriteMap.empty() && NotEqual.empty())
16030 return Expr;
16031
16032 SCEVLoopGuardRewriter Rewriter(SE, *this);
16033 return Rewriter.visit(Expr);
16034}
16035
16036const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) {
16037 return applyLoopGuards(Expr, LoopGuards::collect(L, *this));
16038}
16039
16041 const LoopGuards &Guards) {
16042 return Guards.rewrite(Expr);
16043}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
constexpr LLT S1
Rewrite undef for PHI
This file implements a class to represent arbitrary precision integral constant values and operations...
@ PostInc
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition Compiler.h:638
This file contains the declarations for the subclasses of Constant, which represent the different fla...
SmallPtrSet< const BasicBlock *, 8 > VisitedBlocks
This file defines the DenseMap class.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
static bool isSigned(unsigned int Opcode)
This file defines a hash set that can be used to remove duplication of nodes in a graph.
#define op(i)
Hexagon Common GEP
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This defines the Use class.
iv Induction Variable Users
Definition IVUsers.cpp:48
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:539
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
#define G(x, y, z)
Definition MD5.cpp:56
#define T
#define T1
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
#define P(N)
ppc ctr loops verify
PowerPC Reduce CR logical Operation
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
R600 Clause Merge
const SmallVectorImpl< MachineOperand > & Cond
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
SI optimize exec mask operations pre RA
void visit(MachineFunction &MF, MachineBasicBlock &Start, std::function< void(MachineBasicBlock *)> op)
This file contains some templates that are useful if you are working with the STL at all.
This file provides utility classes that use RAII to save and restore values.
bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind, SCEVTypes RootKind)
static cl::opt< unsigned > MaxAddRecSize("scalar-evolution-max-add-rec-size", cl::Hidden, cl::desc("Max coefficients in AddRec during evolving"), cl::init(8))
static cl::opt< unsigned > RangeIterThreshold("scev-range-iter-threshold", cl::Hidden, cl::desc("Threshold for switching to iteratively computing SCEV ranges"), cl::init(32))
static const Loop * isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI)
static unsigned getConstantTripCount(const SCEVConstant *ExitCount)
static int CompareValueComplexity(const LoopInfo *const LI, Value *LV, Value *RV, unsigned Depth)
Compare the two values LV and RV in terms of their "complexity" where "complexity" is a partial (and ...
static const SCEV * getNextSCEVDivisibleByDivisor(const SCEV *Expr, const APInt &DivisorVal, ScalarEvolution &SE)
static void PushLoopPHIs(const Loop *L, SmallVectorImpl< Instruction * > &Worklist, SmallPtrSetImpl< Instruction * > &Visited)
Push PHI nodes in the header of the given loop onto the given Worklist.
static void insertFoldCacheEntry(const ScalarEvolution::FoldID &ID, const SCEV *S, DenseMap< ScalarEvolution::FoldID, const SCEV * > &FoldCache, DenseMap< const SCEV *, SmallVector< ScalarEvolution::FoldID, 2 > > &FoldCacheUser)
static cl::opt< bool > ClassifyExpressions("scalar-evolution-classify-expressions", cl::Hidden, cl::init(true), cl::desc("When printing analysis, include information on every instruction"))
static bool CanConstantFold(const Instruction *I)
Return true if we can constant fold an instruction of the specified type, assuming that all operands ...
static cl::opt< unsigned > AddOpsInlineThreshold("scev-addops-inline-threshold", cl::Hidden, cl::desc("Threshold for inlining addition operands into a SCEV"), cl::init(500))
static cl::opt< unsigned > MaxLoopGuardCollectionDepth("scalar-evolution-max-loop-guard-collection-depth", cl::Hidden, cl::desc("Maximum depth for recursive loop guard collection"), cl::init(1))
static cl::opt< bool > VerifyIR("scev-verify-ir", cl::Hidden, cl::desc("Verify IR correctness when making sensitive SCEV queries (slow)"), cl::init(false))
static bool BrPHIToSelect(DominatorTree &DT, BranchInst *BI, PHINode *Merge, Value *&C, Value *&LHS, Value *&RHS)
static const SCEV * getPreStartForExtend(const SCEVAddRecExpr *AR, Type *Ty, ScalarEvolution *SE, unsigned Depth)
static std::optional< APInt > MinOptional(std::optional< APInt > X, std::optional< APInt > Y)
Helper function to compare optional APInts: (a) if X and Y both exist, return min(X,...
static cl::opt< unsigned > MulOpsInlineThreshold("scev-mulops-inline-threshold", cl::Hidden, cl::desc("Threshold for inlining multiplication operands into a SCEV"), cl::init(32))
static void GroupByComplexity(SmallVectorImpl< const SCEV * > &Ops, LoopInfo *LI, DominatorTree &DT)
Given a list of SCEV objects, order them by their complexity, and group objects of the same complexit...
static const SCEV * constantFoldAndGroupOps(ScalarEvolution &SE, LoopInfo &LI, DominatorTree &DT, SmallVectorImpl< const SCEV * > &Ops, FoldT Fold, IsIdentityT IsIdentity, IsAbsorberT IsAbsorber)
Performs a number of common optimizations on the passed Ops.
static std::optional< const SCEV * > createNodeForSelectViaUMinSeq(ScalarEvolution *SE, const SCEV *CondExpr, const SCEV *TrueExpr, const SCEV *FalseExpr)
static Constant * BuildConstantFromSCEV(const SCEV *V)
This builds up a Constant using the ConstantExpr interface.
static ConstantInt * EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C, ScalarEvolution &SE)
static const SCEV * BinomialCoefficient(const SCEV *It, unsigned K, ScalarEvolution &SE, Type *ResultTy)
Compute BC(It, K). The result has width W. Assume, K > 0.
static cl::opt< unsigned > MaxCastDepth("scalar-evolution-max-cast-depth", cl::Hidden, cl::desc("Maximum depth of recursive SExt/ZExt/Trunc"), cl::init(8))
static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr, const SCEV *Candidate)
Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
static PHINode * getConstantEvolvingPHI(Value *V, const Loop *L)
getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node in the loop that V is deri...
static const SCEV * SolveLinEquationWithOverflow(const APInt &A, const SCEV *B, SmallVectorImpl< const SCEVPredicate * > *Predicates, ScalarEvolution &SE, const Loop *L)
Finds the minimum unsigned root of the following equation:
static cl::opt< unsigned > MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden, cl::desc("Maximum number of iterations SCEV will " "symbolically execute a constant " "derived loop"), cl::init(100))
static bool MatchBinarySub(const SCEV *S, const SCEV *&LHS, const SCEV *&RHS)
static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow)
static void PrintSCEVWithTypeHint(raw_ostream &OS, const SCEV *S)
When printing a top-level SCEV for trip counts, it's helpful to include a type for constants which ar...
static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, const Loop *L)
static bool containsConstantInAddMulChain(const SCEV *StartExpr)
Determine if any of the operands in this SCEV are a constant or if any of the add or multiply express...
static const SCEV * getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty, ScalarEvolution *SE, unsigned Depth)
static bool hasHugeExpression(ArrayRef< const SCEV * > Ops)
Returns true if Ops contains a huge SCEV (the subtree of S contains at least HugeExprThreshold nodes)...
static cl::opt< unsigned > MaxPhiSCCAnalysisSize("scalar-evolution-max-scc-analysis-depth", cl::Hidden, cl::desc("Maximum amount of nodes to process while searching SCEVUnknown " "Phi strongly connected components"), cl::init(8))
static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
static cl::opt< unsigned > MaxSCEVOperationsImplicationDepth("scalar-evolution-max-scev-operations-implication-depth", cl::Hidden, cl::desc("Maximum depth of recursive SCEV operations implication analysis"), cl::init(2))
static void PushDefUseChildren(Instruction *I, SmallVectorImpl< Instruction * > &Worklist, SmallPtrSetImpl< Instruction * > &Visited)
Push users of the given Instruction onto the given Worklist.
static std::optional< APInt > SolveQuadraticAddRecRange(const SCEVAddRecExpr *AddRec, const ConstantRange &Range, ScalarEvolution &SE)
Let c(n) be the value of the quadratic chrec {0,+,M,+,N} after n iterations.
static cl::opt< bool > UseContextForNoWrapFlagInference("scalar-evolution-use-context-for-no-wrap-flag-strenghening", cl::Hidden, cl::desc("Infer nuw/nsw flags using context where suitable"), cl::init(true))
static cl::opt< bool > EnableFiniteLoopControl("scalar-evolution-finite-loop", cl::Hidden, cl::desc("Handle <= and >= in finite loops"), cl::init(true))
static std::optional< std::tuple< APInt, APInt, APInt, APInt, unsigned > > GetQuadraticEquation(const SCEVAddRecExpr *AddRec)
For a given quadratic addrec, generate coefficients of the corresponding quadratic equation,...
static bool isKnownPredicateExtendIdiom(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
static std::optional< BinaryOp > MatchBinaryOp(Value *V, const DataLayout &DL, AssumptionCache &AC, const DominatorTree &DT, const Instruction *CxtI)
Try to map V into a BinaryOp, and return std::nullopt on failure.
static std::optional< APInt > SolveQuadraticAddRecExact(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE)
Let c(n) be the value of the quadratic chrec {L,+,M,+,N} after n iterations.
static std::optional< APInt > TruncIfPossible(std::optional< APInt > X, unsigned BitWidth)
Helper function to truncate an optional APInt to a given BitWidth.
static cl::opt< unsigned > MaxSCEVCompareDepth("scalar-evolution-max-scev-compare-depth", cl::Hidden, cl::desc("Maximum depth of recursive SCEV complexity comparisons"), cl::init(32))
static APInt extractConstantWithoutWrapping(ScalarEvolution &SE, const SCEVConstant *ConstantTerm, const SCEVAddExpr *WholeAddExpr)
static cl::opt< unsigned > MaxConstantEvolvingDepth("scalar-evolution-max-constant-evolving-depth", cl::Hidden, cl::desc("Maximum depth of recursive constant evolving"), cl::init(32))
static ConstantRange getRangeForAffineARHelper(APInt Step, const ConstantRange &StartRange, const APInt &MaxBECount, bool Signed)
static std::optional< ConstantRange > GetRangeFromMetadata(Value *V)
Helper method to assign a range to V from metadata present in the IR.
static cl::opt< unsigned > HugeExprThreshold("scalar-evolution-huge-expr-threshold", cl::Hidden, cl::desc("Size of the expression which is considered huge"), cl::init(4096))
static Type * isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI, bool &Signed, ScalarEvolution &SE)
Helper function to createAddRecFromPHIWithCasts.
static Constant * EvaluateExpression(Value *V, const Loop *L, DenseMap< Instruction *, Constant * > &Vals, const DataLayout &DL, const TargetLibraryInfo *TLI)
EvaluateExpression - Given an expression that passes the getConstantEvolvingPHI predicate,...
static const SCEV * getPreviousSCEVDivisibleByDivisor(const SCEV *Expr, const APInt &DivisorVal, ScalarEvolution &SE)
static const SCEV * MatchNotExpr(const SCEV *Expr)
If Expr computes ~A, return A else return nullptr.
static cl::opt< unsigned > MaxValueCompareDepth("scalar-evolution-max-value-compare-depth", cl::Hidden, cl::desc("Maximum depth of recursive value complexity comparisons"), cl::init(2))
static cl::opt< bool, true > VerifySCEVOpt("verify-scev", cl::Hidden, cl::location(VerifySCEV), cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"))
static const SCEV * getSignedOverflowLimitForStep(const SCEV *Step, ICmpInst::Predicate *Pred, ScalarEvolution *SE)
static SCEV::NoWrapFlags StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type, const ArrayRef< const SCEV * > Ops, SCEV::NoWrapFlags Flags)
static cl::opt< unsigned > MaxArithDepth("scalar-evolution-max-arith-depth", cl::Hidden, cl::desc("Maximum depth of recursive arithmetics"), cl::init(32))
static bool HasSameValue(const SCEV *A, const SCEV *B)
SCEV structural equivalence is usually sufficient for testing whether two expressions are equal,...
static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow)
Compute the result of "n choose k", the binomial coefficient.
static std::optional< int > CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS, const SCEV *RHS, DominatorTree &DT, unsigned Depth=0)
static bool CollectAddOperandsWithScales(SmallDenseMap< const SCEV *, APInt, 16 > &M, SmallVectorImpl< const SCEV * > &NewOps, APInt &AccumulatedConstant, ArrayRef< const SCEV * > Ops, const APInt &Scale, ScalarEvolution &SE)
Process the given Ops list, which is a list of operands to be added under the given scale,...
static bool canConstantEvolve(Instruction *I, const Loop *L)
Determine whether this instruction can constant evolve within this loop assuming its operands can all...
static PHINode * getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L, DenseMap< Instruction *, PHINode * > &PHIMap, unsigned Depth)
getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by recursing through each instructi...
static bool scevUnconditionallyPropagatesPoisonFromOperands(SCEVTypes Kind)
static cl::opt< bool > VerifySCEVStrict("verify-scev-strict", cl::Hidden, cl::desc("Enable stricter verification with -verify-scev is passed"))
static Constant * getOtherIncomingValue(PHINode *PN, BasicBlock *BB)
static cl::opt< bool > UseExpensiveRangeSharpening("scalar-evolution-use-expensive-range-sharpening", cl::Hidden, cl::init(false), cl::desc("Use more powerful methods of sharpening expression ranges. May " "be costly in terms of compile time"))
static const SCEV * getUnsignedOverflowLimitForStep(const SCEV *Step, ICmpInst::Predicate *Pred, ScalarEvolution *SE)
static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Is LHS Pred RHS true on the virtue of LHS or RHS being a Min or Max expression?
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
static bool InBlock(const Value *V, const BasicBlock *BB)
Provides some synthesis utilities to produce sequences of values.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
This file contains some functions that are useful when dealing with strings.
#define LLVM_DEBUG(...)
Definition Debug.h:114
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition VPlanSLP.cpp:247
static std::optional< bool > isImpliedCondOperands(CmpInst::Predicate Pred, const Value *ALHS, const Value *ARHS, const Value *BLHS, const Value *BRHS)
Return true if "icmp Pred BLHS BRHS" is true whenever "icmp PredALHS ARHS" is true.
Virtual Register Rewriter
Value * RHS
Value * LHS
BinaryOperator * Mul
static const uint32_t IV[8]
Definition blake3_impl.h:83
Class for arbitrary precision integers.
Definition APInt.h:78
LLVM_ABI APInt umul_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1971
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1012
bool isMinSignedValue() const
Determine if this is the smallest signed value.
Definition APInt.h:423
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1540
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
Definition APInt.h:1391
LLVM_ABI APInt getHiBits(unsigned numBits) const
Compute an APInt containing numBits highbits from this APInt.
Definition APInt.cpp:639
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1512
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:936
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition APInt.h:206
APInt abs() const
Get the absolute value.
Definition APInt.h:1795
bool sgt(const APInt &RHS) const
Signed greater than comparison.
Definition APInt.h:1201
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition APInt.h:371
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition APInt.h:1182
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:380
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
Definition APInt.h:466
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition APInt.cpp:1666
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1488
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition APInt.h:1111
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition APInt.h:209
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
Definition APInt.h:216
bool isNegative() const
Determine sign of this APInt.
Definition APInt.h:329
bool sle(const APInt &RHS) const
Signed less or equal comparison.
Definition APInt.h:1166
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition APInt.h:219
bool isNonPositive() const
Determine if this APInt Value is non-positive (<= 0).
Definition APInt.h:361
unsigned countTrailingZeros() const
Definition APInt.h:1647
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
Definition APInt.h:356
unsigned logBase2() const
Definition APInt.h:1761
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition APInt.h:827
LLVM_ABI APInt multiplicativeInverse() const
Definition APInt.cpp:1274
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition APInt.h:1150
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
Definition APInt.cpp:985
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition APInt.h:873
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:440
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:306
bool isSignBitSet() const
Determine if sign bit of this APInt is set.
Definition APInt.h:341
bool slt(const APInt &RHS) const
Signed less than comparison.
Definition APInt.h:1130
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition APInt.h:200
bool isIntN(unsigned N) const
Check if this APInt has an N-bits unsigned integer value.
Definition APInt.h:432
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition APInt.h:239
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition APInt.h:1221
This templated class represents "all analyses that operate over <aparticular IR unit>" (e....
Definition Analysis.h:50
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
void setPreservesAll()
Set by analyses that do not transform their input at all.
AnalysisUsage & addRequiredTransitive()
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
iterator end() const
Definition ArrayRef.h:136
size_t size() const
size - Get the array size.
Definition ArrayRef.h:147
iterator begin() const
Definition ArrayRef.h:135
A function analysis which provides an AssumptionCache.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
MutableArrayRef< WeakVH > assumptions()
Access the list of assumption handles currently tracked for this function.
LLVM_ABI bool isSingleEdge() const
Check if this is the only edge between Start and End.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator begin()
Instruction iterator methods.
Definition BasicBlock.h:459
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction & front() const
Definition BasicBlock.h:482
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition BasicBlock.h:233
LLVM_ABI unsigned getNoWrapKind() const
Returns one of OBO::NoSignedWrap or OBO::NoUnsignedWrap.
LLVM_ABI Instruction::BinaryOps getBinaryOp() const
Returns the binary operation underlying the intrinsic.
BinaryOps getOpcode() const
Definition InstrTypes.h:374
Conditional or Unconditional Branch instruction.
bool isConditional() const
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
LLVM_ATTRIBUTE_RETURNS_NONNULL void * Allocate(size_t Size, Align Alignment)
Allocate space at the specified alignment.
Definition Allocator.h:149
This class represents a function call, abstracting a target machine's calling convention.
virtual void deleted()
Callback for Value destruction.
void setValPtr(Value *P)
bool isFalseWhenEqual() const
This is just a convenience.
Definition InstrTypes.h:948
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:676
@ ICMP_SLT
signed less than
Definition InstrTypes.h:705
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:706
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:700
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:699
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:703
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:701
@ ICMP_NE
not equal
Definition InstrTypes.h:698
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:704
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:702
bool isSigned() const
Definition InstrTypes.h:930
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition InstrTypes.h:827
bool isTrueWhenEqual() const
This is just a convenience.
Definition InstrTypes.h:942
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition InstrTypes.h:789
bool isUnsigned() const
Definition InstrTypes.h:936
bool isRelational() const
Return true if the predicate is relational (not EQ or NE).
Definition InstrTypes.h:926
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
static LLVM_ABI std::optional< CmpPredicate > getMatching(CmpPredicate A, CmpPredicate B)
Compares two CmpPredicates taking samesign into account and returns the canonicalized CmpPredicate if...
LLVM_ABI CmpInst::Predicate getPreferredSignedPredicate() const
Attempts to return a signed CmpInst::Predicate from the CmpPredicate.
CmpInst::Predicate dropSameSign() const
Drops samesign information.
static LLVM_ABI Constant * getNot(Constant *C)
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, GEPNoWrapFlags NW=GEPNoWrapFlags::none(), std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
Definition Constants.h:1274
static LLVM_ABI Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static LLVM_ABI Constant * getNeg(Constant *C, bool HasNSW=false)
static LLVM_ABI Constant * getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:214
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:163
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:154
static LLVM_ABI ConstantInt * getBool(LLVMContext &Context, bool V)
This class represents a range of values.
LLVM_ABI ConstantRange add(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an addition of a value in this ran...
LLVM_ABI ConstantRange zextOrTrunc(uint32_t BitWidth) const
Make this range have the bit width given by BitWidth.
PreferredRangeType
If represented precisely, the result of some range operations may consist of multiple disjoint ranges...
LLVM_ABI bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const
Set up Pred and RHS such that ConstantRange::makeExactICmpRegion(Pred, RHS) == *this.
const APInt & getLower() const
Return the lower value for this range.
LLVM_ABI bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
LLVM_ABI bool icmp(CmpInst::Predicate Pred, const ConstantRange &Other) const
Does the predicate Pred hold between ranges this and Other?
LLVM_ABI bool isEmptySet() const
Return true if this set contains no members.
LLVM_ABI ConstantRange zeroExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
LLVM_ABI bool isSignWrappedSet() const
Return true if this set wraps around the signed domain.
LLVM_ABI APInt getSignedMin() const
Return the smallest signed value contained in the ConstantRange.
LLVM_ABI bool isWrappedSet() const
Return true if this set wraps around the unsigned domain.
LLVM_ABI void print(raw_ostream &OS) const
Print out the bounds to a stream.
LLVM_ABI ConstantRange truncate(uint32_t BitWidth, unsigned NoWrapKind=0) const
Return a new range in the specified integer type, which must be strictly smaller than the current typ...
LLVM_ABI ConstantRange signExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
const APInt & getUpper() const
Return the upper value for this range.
LLVM_ABI ConstantRange unionWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the union of this range with another range.
static LLVM_ABI ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
LLVM_ABI bool contains(const APInt &Val) const
Return true if the specified value is in the set.
LLVM_ABI APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
LLVM_ABI ConstantRange intersectWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the intersection of this range with another range.
LLVM_ABI APInt getSignedMax() const
Return the largest signed value contained in the ConstantRange.
static ConstantRange getNonEmpty(APInt Lower, APInt Upper)
Create non-empty constant range with the given bounds.
static LLVM_ABI ConstantRange makeGuaranteedNoWrapRegion(Instruction::BinaryOps BinOp, const ConstantRange &Other, unsigned NoWrapKind)
Produce the largest range containing all X such that "X BinOp Y" is guaranteed not to wrap (overflow)...
LLVM_ABI unsigned getMinSignedBits() const
Compute the maximal number of bits needed to represent every value in this signed range.
uint32_t getBitWidth() const
Get the bit width of this ConstantRange.
LLVM_ABI ConstantRange sub(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a subtraction of a value in this r...
LLVM_ABI ConstantRange sextOrTrunc(uint32_t BitWidth) const
Make this range have the bit width given by BitWidth.
static LLVM_ABI ConstantRange makeExactNoWrapRegion(Instruction::BinaryOps BinOp, const APInt &Other, unsigned NoWrapKind)
Produce the range that contains X if and only if "X BinOp Other" does not wrap.
This is an important base class in LLVM.
Definition Constant.h:43
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:63
LLVM_ABI const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
LLVM_ABI IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
LLVM_ABI unsigned getIndexTypeSizeInBits(Type *Ty) const
The size in bits of the index used in GEP calculation for this type.
LLVM_ABI IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition DataLayout.h:760
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition DenseMap.h:194
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:167
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:237
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT > iterator
Definition DenseMap.h:74
iterator find_as(const LookupKeyT &Val)
Alternate version of find() which allows a different, and possibly less expensive,...
Definition DenseMap.h:180
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition DenseMap.h:163
iterator end()
Definition DenseMap.h:81
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition DenseMap.h:158
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:222
void swap(DenseMap &RHS)
Definition DenseMap.h:747
Analysis pass which computes a DominatorTree.
Definition Dominators.h:284
Legacy analysis pass which computes a DominatorTree.
Definition Dominators.h:322
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:165
LLVM_ABI bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
FoldingSetNodeIDRef - This class describes a reference to an interned FoldingSetNodeID,...
Definition FoldingSet.h:293
FoldingSetNodeID - This class is used to gather all the unique data bits of a node.
Definition FoldingSet.h:330
FunctionPass(char &pid)
Definition Pass.h:316
Represents flags for the getelementptr instruction/expression.
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
static GEPNoWrapFlags none()
static LLVM_ABI Type * getTypeAtIndex(Type *Ty, Value *Idx)
Return the type of the element at the given index of an indexable type.
Module * getParent()
Get the module that this global value is contained inside of...
static bool isPrivateLinkage(LinkageTypes Linkage)
static bool isInternalLinkage(LinkageTypes Linkage)
This instruction compares its operands according to the predicate given to the constructor.
CmpPredicate getCmpPredicate() const
static bool isGE(Predicate P)
Return true if the predicate is SGE or UGE.
CmpPredicate getSwappedCmpPredicate() const
static LLVM_ABI bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
static bool isLT(Predicate P)
Return true if the predicate is SLT or ULT.
CmpPredicate getInverseCmpPredicate() const
Predicate getNonStrictCmpPredicate() const
For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
static bool isGT(Predicate P)
Return true if the predicate is SGT or UGT.
Predicate getFlippedSignednessPredicate() const
For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->EQ.
static CmpPredicate getInverseCmpPredicate(CmpPredicate Pred)
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
bool isRelational() const
Return true if the predicate is relational (not EQ or NE).
static bool isLE(Predicate P)
Return true if the predicate is SLE or ULE.
LLVM_ABI bool hasNoUnsignedWrap() const LLVM_READONLY
Determine whether the no unsigned wrap flag is set.
LLVM_ABI bool hasNoSignedWrap() const LLVM_READONLY
Determine whether the no signed wrap flag is set.
LLVM_ABI bool isIdenticalToWhenDefined(const Instruction *I, bool IntersectAttrs=false) const LLVM_READONLY
This is like isIdenticalTo, except that it ignores the SubclassOptionalData flags,...
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:319
An instruction for reading from memory.
Analysis pass that exposes the LoopInfo for a function.
Definition LoopInfo.h:569
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
BlockT * getHeader() const
unsigned getLoopDepth() const
Return the nesting level of this loop.
BlockT * getLoopPredecessor() const
If the given loop's header has exactly one unique predecessor outside the loop, return it.
LoopT * getParentLoop() const
Return the parent loop if it exists or nullptr for top level loops.
unsigned getLoopDepth(const BlockT *BB) const
Return the loop nesting level of the specified block.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
The legacy pass manager's analysis pass to compute loop information.
Definition LoopInfo.h:596
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
Definition LoopInfo.cpp:61
Metadata node.
Definition Metadata.h:1078
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:67
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
Definition Operator.h:43
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
Definition Operator.h:78
bool hasNoSignedWrap() const
Test whether this operation is known to never undergo signed overflow, aka the nsw property.
Definition Operator.h:111
bool hasNoUnsignedWrap() const
Test whether this operation is known to never undergo unsigned overflow, aka the nuw property.
Definition Operator.h:105
iterator_range< const_block_iterator > blocks() const
op_range incoming_values()
Value * getIncomingValueForBlock(const BasicBlock *BB) const
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
PointerIntPair - This class implements a pair of a pointer and small integer.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
LLVM_ABI void addPredicate(const SCEVPredicate &Pred)
Adds a new predicate.
LLVM_ABI const SCEVPredicate & getPredicate() const
LLVM_ABI bool hasNoOverflow(Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags)
Returns true if we've proved that V doesn't wrap by means of a SCEV predicate.
LLVM_ABI void setNoOverflow(Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags)
Proves that V doesn't overflow by adding SCEV predicate.
LLVM_ABI void print(raw_ostream &OS, unsigned Depth) const
Print the SCEV mappings done by the Predicated Scalar Evolution.
LLVM_ABI bool areAddRecsEqualWithPreds(const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2) const
Check if AR1 and AR2 are equal, while taking into account Equal predicates in Preds.
LLVM_ABI PredicatedScalarEvolution(ScalarEvolution &SE, Loop &L)
LLVM_ABI const SCEVAddRecExpr * getAsAddRec(Value *V)
Attempts to produce an AddRecExpr for V by adding additional SCEV predicates.
LLVM_ABI unsigned getSmallConstantMaxTripCount()
Returns the upper bound of the loop trip count as a normal unsigned value, or 0 if the trip count is ...
LLVM_ABI const SCEV * getBackedgeTakenCount()
Get the (predicated) backedge count for the analyzed loop.
LLVM_ABI const SCEV * getSymbolicMaxBackedgeTakenCount()
Get the (predicated) symbolic max backedge count for the analyzed loop.
LLVM_ABI const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalysisChecker getChecker() const
Build a checker for this PreservedAnalyses and the specified analysis type.
Definition Analysis.h:275
constexpr bool isValid() const
Definition Register.h:107
This node represents an addition of some number of SCEVs.
This node represents a polynomial recurrence on the trip count of the specified loop.
LLVM_ABI const SCEV * evaluateAtIteration(const SCEV *It, ScalarEvolution &SE) const
Return the value of this chain of recurrences at the specified iteration number.
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
Constructs and returns the recurrence indicating how much this expression steps by.
void setNoWrapFlags(NoWrapFlags Flags)
Set flags for a recurrence without clearing any previously set flags.
bool isAffine() const
Return true if this represents an expression A + B*x where A and B are loop invariant values.
bool isQuadratic() const
Return true if this represents an expression A + B*x + C*x^2 where A, B and C are loop invariant valu...
LLVM_ABI const SCEV * getNumIterationsInRange(const ConstantRange &Range, ScalarEvolution &SE) const
Return the number of iterations of this loop that produce values in the specified constant range.
LLVM_ABI const SCEVAddRecExpr * getPostIncExpr(ScalarEvolution &SE) const
Return an expression representing the value of this expression one iteration of the loop ahead.
This is the base class for unary cast operator classes.
const SCEV * getOperand() const
LLVM_ABI SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, const SCEV *op, Type *ty)
void setNoWrapFlags(NoWrapFlags Flags)
Set flags for a non-recurrence without clearing previously set flags.
This class represents an assumption that the expression LHS Pred RHS evaluates to true,...
SCEVComparePredicate(const FoldingSetNodeIDRef ID, const ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
bool isAlwaysTrue() const override
Returns true if the predicate is always true.
void print(raw_ostream &OS, unsigned Depth=0) const override
Prints a textual representation of this predicate with an indentation of Depth.
bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const override
Implementation of the SCEVPredicate interface.
This class represents a constant integer value.
ConstantInt * getValue() const
const APInt & getAPInt() const
This is the base class for unary integral cast operator classes.
LLVM_ABI SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, const SCEV *op, Type *ty)
This node is the base class min/max selections.
static enum SCEVTypes negate(enum SCEVTypes T)
This node represents multiplication of some number of SCEVs.
This node is a base class providing common functionality for n'ary operators.
NoWrapFlags getNoWrapFlags(NoWrapFlags Mask=NoWrapMask) const
const SCEV * getOperand(unsigned i) const
ArrayRef< const SCEV * > operands() const
This class represents an assumption made using SCEV expressions which can be checked at run-time.
SCEVPredicate(const SCEVPredicate &)=default
virtual bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const =0
Returns true if this predicate implies N.
SCEVPredicateKind Kind
This class represents a cast from a pointer to a pointer-sized integer value.
This visitor recursively visits a SCEV expression and re-writes it.
const SCEV * visitSignExtendExpr(const SCEVSignExtendExpr *Expr)
const SCEV * visit(const SCEV *S)
const SCEV * visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr)
const SCEV * visitSMinExpr(const SCEVSMinExpr *Expr)
const SCEV * visitUMinExpr(const SCEVUMinExpr *Expr)
This class represents a signed minimum selection.
This node is the base class for sequential/in-order min/max selections.
static SCEVTypes getEquivalentNonSequentialSCEVType(SCEVTypes Ty)
This class represents a sign extension of a small integer value to a larger integer value.
Visit all nodes in the expression tree using worklist traversal.
This class represents a truncation of an integer value to a smaller integer value.
This class represents a binary unsigned division operation.
This class represents an unsigned minimum selection.
This class represents a composition of other SCEV predicates, and is the class that most clients will...
void print(raw_ostream &OS, unsigned Depth) const override
Prints a textual representation of this predicate with an indentation of Depth.
bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const override
Returns true if this predicate implies N.
SCEVUnionPredicate(ArrayRef< const SCEVPredicate * > Preds, ScalarEvolution &SE)
Union predicates don't get cached so create a dummy set ID for it.
bool isAlwaysTrue() const override
Implementation of the SCEVPredicate interface.
This means that we are dealing with an entirely unknown SCEV value, and only represent it as its LLVM...
This class represents the value of vscale, as used when defining the length of a scalable vector or r...
This class represents an assumption made on an AddRec expression.
IncrementWrapFlags
Similar to SCEV::NoWrapFlags, but with slightly different semantics for FlagNUSW.
SCEVWrapPredicate(const FoldingSetNodeIDRef ID, const SCEVAddRecExpr *AR, IncrementWrapFlags Flags)
bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const override
Returns true if this predicate implies N.
static SCEVWrapPredicate::IncrementWrapFlags setFlags(SCEVWrapPredicate::IncrementWrapFlags Flags, SCEVWrapPredicate::IncrementWrapFlags OnFlags)
void print(raw_ostream &OS, unsigned Depth=0) const override
Prints a textual representation of this predicate with an indentation of Depth.
bool isAlwaysTrue() const override
Returns true if the predicate is always true.
const SCEVAddRecExpr * getExpr() const
Implementation of the SCEVPredicate interface.
static SCEVWrapPredicate::IncrementWrapFlags clearFlags(SCEVWrapPredicate::IncrementWrapFlags Flags, SCEVWrapPredicate::IncrementWrapFlags OffFlags)
Convenient IncrementWrapFlags manipulation methods.
static SCEVWrapPredicate::IncrementWrapFlags getImpliedFlags(const SCEVAddRecExpr *AR, ScalarEvolution &SE)
Returns the set of SCEVWrapPredicate no wrap flags implied by a SCEVAddRecExpr.
IncrementWrapFlags getFlags() const
Returns the set assumed no overflow flags.
This class represents a zero extension of a small integer value to a larger integer value.
This class represents an analyzed expression in the program.
LLVM_ABI ArrayRef< const SCEV * > operands() const
Return operands of this SCEV expression.
unsigned short getExpressionSize() const
LLVM_ABI bool isOne() const
Return true if the expression is a constant one.
LLVM_ABI bool isZero() const
Return true if the expression is a constant zero.
LLVM_ABI void dump() const
This method is used for debugging.
LLVM_ABI bool isAllOnesValue() const
Return true if the expression is a constant all-ones value.
LLVM_ABI bool isNonConstantNegative() const
Return true if the specified scev is negated, but not a constant.
LLVM_ABI void print(raw_ostream &OS) const
Print out the internal representation of this scalar to the specified stream.
SCEV(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, unsigned short ExpressionSize)
SCEVTypes getSCEVType() const
LLVM_ABI Type * getType() const
Return the LLVM type of this SCEV expression.
NoWrapFlags
NoWrapFlags are bitfield indices into SubclassData.
Analysis pass that exposes the ScalarEvolution for a function.
LLVM_ABI ScalarEvolution run(Function &F, FunctionAnalysisManager &AM)
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
void print(raw_ostream &OS, const Module *=nullptr) const override
print - Print out the internal state of the pass.
bool runOnFunction(Function &F) override
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
void releaseMemory() override
releaseMemory() - This member can be implemented by a pass if it wants to be able to release its memo...
void verifyAnalysis() const override
verifyAnalysis() - This member can be implemented by a analysis pass to check state of analysis infor...
static LLVM_ABI LoopGuards collect(const Loop *L, ScalarEvolution &SE)
Collect rewrite map for loop guards for loop L, together with flags indicating if NUW and NSW can be ...
LLVM_ABI const SCEV * rewrite(const SCEV *Expr) const
Try to apply the collected loop guards to Expr.
The main scalar evolution driver.
const SCEV * getConstantMaxBackedgeTakenCount(const Loop *L)
When successful, this returns a SCEVConstant that is greater than or equal to (i.e.
static bool hasFlags(SCEV::NoWrapFlags Flags, SCEV::NoWrapFlags TestFlags)
const DataLayout & getDataLayout() const
Return the DataLayout associated with the module this SCEV instance is operating on.
LLVM_ABI bool isKnownNonNegative(const SCEV *S)
Test if the given expression is known to be non-negative.
LLVM_ABI bool isKnownOnEveryIteration(CmpPredicate Pred, const SCEVAddRecExpr *LHS, const SCEV *RHS)
Test if the condition described by Pred, LHS, RHS is known to be true on every iteration of the loop ...
LLVM_ABI const SCEV * getNegativeSCEV(const SCEV *V, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Return the SCEV object corresponding to -V.
LLVM_ABI std::optional< LoopInvariantPredicate > getLoopInvariantExitCondDuringFirstIterationsImpl(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L, const Instruction *CtxI, const SCEV *MaxIter)
LLVM_ABI const SCEV * getSMaxExpr(const SCEV *LHS, const SCEV *RHS)
LLVM_ABI const SCEV * getUDivCeilSCEV(const SCEV *N, const SCEV *D)
Compute ceil(N / D).
LLVM_ABI const SCEV * getGEPExpr(GEPOperator *GEP, const SmallVectorImpl< const SCEV * > &IndexExprs)
Returns an expression for a GEP.
LLVM_ABI std::optional< LoopInvariantPredicate > getLoopInvariantExitCondDuringFirstIterations(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L, const Instruction *CtxI, const SCEV *MaxIter)
If the result of the predicate LHS Pred RHS is loop invariant with respect to L at given Context duri...
LLVM_ABI Type * getWiderType(Type *Ty1, Type *Ty2) const
LLVM_ABI const SCEV * getAbsExpr(const SCEV *Op, bool IsNSW)
LLVM_ABI bool isKnownNonPositive(const SCEV *S)
Test if the given expression is known to be non-positive.
LLVM_ABI const SCEV * getURemExpr(const SCEV *LHS, const SCEV *RHS)
Represents an unsigned remainder expression based on unsigned division.
LLVM_ABI bool isKnownNegative(const SCEV *S)
Test if the given expression is known to be negative.
LLVM_ABI const SCEV * getPredicatedConstantMaxBackedgeTakenCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > &Predicates)
Similar to getConstantMaxBackedgeTakenCount, except it will add a set of SCEV predicates to Predicate...
LLVM_ABI const SCEV * removePointerBase(const SCEV *S)
Compute an expression equivalent to S - getPointerBase(S).
LLVM_ABI bool isLoopEntryGuardedByCond(const Loop *L, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test whether entry to the loop is protected by a conditional between LHS and RHS.
LLVM_ABI bool isKnownNonZero(const SCEV *S)
Test if the given expression is known to be non-zero.
LLVM_ABI const SCEV * getSCEVAtScope(const SCEV *S, const Loop *L)
Return a SCEV expression for the specified value at the specified scope in the program.
LLVM_ABI const SCEV * getSMinExpr(const SCEV *LHS, const SCEV *RHS)
LLVM_ABI const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
LLVM_ABI const SCEV * getUMaxExpr(const SCEV *LHS, const SCEV *RHS)
LLVM_ABI void setNoWrapFlags(SCEVAddRecExpr *AddRec, SCEV::NoWrapFlags Flags)
Update no-wrap flags of an AddRec.
LLVM_ABI const SCEV * getUMaxFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS)
Promote the operands to the wider of the types using zero-extension, and then perform a umax operatio...
const SCEV * getZero(Type *Ty)
Return a SCEV for the constant 0 of a specific type.
LLVM_ABI bool willNotOverflow(Instruction::BinaryOps BinOp, bool Signed, const SCEV *LHS, const SCEV *RHS, const Instruction *CtxI=nullptr)
Is operation BinOp between LHS and RHS provably does not have a signed/unsigned overflow (Signed)?
LLVM_ABI ExitLimit computeExitLimitFromCond(const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit, bool AllowPredicates=false)
Compute the number of times the backedge of the specified loop will execute if its exit condition wer...
LLVM_ABI const SCEV * getZeroExtendExprImpl(const SCEV *Op, Type *Ty, unsigned Depth=0)
LLVM_ABI const SCEVPredicate * getEqualPredicate(const SCEV *LHS, const SCEV *RHS)
LLVM_ABI unsigned getSmallConstantTripMultiple(const Loop *L, const SCEV *ExitCount)
Returns the largest constant divisor of the trip count as a normal unsigned value,...
LLVM_ABI uint64_t getTypeSizeInBits(Type *Ty) const
Return the size in bits of the specified type, for which isSCEVable must return true.
LLVM_ABI const SCEV * getConstant(ConstantInt *V)
LLVM_ABI const SCEV * getPredicatedBackedgeTakenCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > &Predicates)
Similar to getBackedgeTakenCount, except it will add a set of SCEV predicates to Predicates that are ...
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
ConstantRange getSignedRange(const SCEV *S)
Determine the signed range for a particular SCEV.
LLVM_ABI const SCEV * getNoopOrSignExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
bool loopHasNoAbnormalExits(const Loop *L)
Return true if the loop has no abnormal exits.
LLVM_ABI const SCEV * getTripCountFromExitCount(const SCEV *ExitCount)
A version of getTripCountFromExitCount below which always picks an evaluation type which can not resu...
LLVM_ABI ScalarEvolution(Function &F, TargetLibraryInfo &TLI, AssumptionCache &AC, DominatorTree &DT, LoopInfo &LI)
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
LLVM_ABI const SCEV * getTruncateOrNoop(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI const SCEV * getCastExpr(SCEVTypes Kind, const SCEV *Op, Type *Ty)
LLVM_ABI const SCEV * getSequentialMinMaxExpr(SCEVTypes Kind, SmallVectorImpl< const SCEV * > &Operands)
LLVM_ABI const SCEV * getLosslessPtrToIntExpr(const SCEV *Op, unsigned Depth=0)
LLVM_ABI std::optional< bool > evaluatePredicateAt(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Instruction *CtxI)
Check whether the condition described by Pred, LHS, and RHS is true or false in the given Context.
LLVM_ABI unsigned getSmallConstantMaxTripCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > *Predicates=nullptr)
Returns the upper bound of the loop trip count as a normal unsigned value.
LLVM_ABI const SCEV * getPtrToIntExpr(const SCEV *Op, Type *Ty)
LLVM_ABI bool isBackedgeTakenCountMaxOrZero(const Loop *L)
Return true if the backedge taken count is either the value returned by getConstantMaxBackedgeTakenCo...
LLVM_ABI void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
LLVM_ABI bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
LLVM_ABI bool isKnownPositive(const SCEV *S)
Test if the given expression is known to be positive.
APInt getUnsignedRangeMin(const SCEV *S)
Determine the min of the unsigned range for a particular SCEV.
LLVM_ABI bool SimplifyICmpOperands(CmpPredicate &Pred, const SCEV *&LHS, const SCEV *&RHS, unsigned Depth=0)
Simplify LHS and RHS in a comparison with predicate Pred.
LLVM_ABI const SCEV * getOffsetOfExpr(Type *IntTy, StructType *STy, unsigned FieldNo)
Return an expression for offsetof on the given field with type IntTy.
LLVM_ABI LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L)
Return the "disposition" of the given SCEV with respect to the given loop.
LLVM_ABI bool containsAddRecurrence(const SCEV *S)
Return true if the SCEV is a scAddRecExpr or it contains scAddRecExpr.
LLVM_ABI const SCEV * getSignExtendExprImpl(const SCEV *Op, Type *Ty, unsigned Depth=0)
LLVM_ABI const SCEV * getAddRecExpr(const SCEV *Start, const SCEV *Step, const Loop *L, SCEV::NoWrapFlags Flags)
Get an add recurrence expression for the specified loop.
LLVM_ABI bool hasOperand(const SCEV *S, const SCEV *Op) const
Test whether the given SCEV has Op as a direct or indirect operand.
LLVM_ABI const SCEV * getUDivExpr(const SCEV *LHS, const SCEV *RHS)
Get a canonical unsigned division expression, or something simpler if possible.
LLVM_ABI const SCEV * getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth=0)
LLVM_ABI bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
LLVM_ABI Type * getEffectiveSCEVType(Type *Ty) const
Return a type with the same bitwidth as the given type and which represents how SCEV will treat the g...
LLVM_ABI const SCEVPredicate * getComparePredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
LLVM_ABI const SCEV * getNotSCEV(const SCEV *V)
Return the SCEV object corresponding to ~V.
LLVM_ABI const SCEV * getElementCount(Type *Ty, ElementCount EC, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
LLVM_ABI bool instructionCouldExistWithOperands(const SCEV *A, const SCEV *B)
Return true if there exists a point in the program at which both A and B could be operands to the sam...
ConstantRange getUnsignedRange(const SCEV *S)
Determine the unsigned range for a particular SCEV.
LLVM_ABI void print(raw_ostream &OS) const
LLVM_ABI const SCEV * getUMinExpr(const SCEV *LHS, const SCEV *RHS, bool Sequential=false)
LLVM_ABI const SCEV * getPredicatedExitCount(const Loop *L, const BasicBlock *ExitingBlock, SmallVectorImpl< const SCEVPredicate * > *Predicates, ExitCountKind Kind=Exact)
Same as above except this uses the predicated backedge taken info and may require predicates.
static SCEV::NoWrapFlags clearFlags(SCEV::NoWrapFlags Flags, SCEV::NoWrapFlags OffFlags)
LLVM_ABI void forgetTopmostLoop(const Loop *L)
LLVM_ABI void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
APInt getSignedRangeMin(const SCEV *S)
Determine the min of the signed range for a particular SCEV.
LLVM_ABI const SCEV * getNoopOrAnyExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI void forgetBlockAndLoopDispositions(Value *V=nullptr)
Called when the client has changed the disposition of values in a loop or block.
LLVM_ABI const SCEV * getTruncateExpr(const SCEV *Op, Type *Ty, unsigned Depth=0)
LLVM_ABI std::optional< LoopInvariantPredicate > getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L, const Instruction *CtxI=nullptr)
If the result of the predicate LHS Pred RHS is loop invariant with respect to L, return a LoopInvaria...
LLVM_ABI const SCEV * getStoreSizeOfExpr(Type *IntTy, Type *StoreTy)
Return an expression for the store size of StoreTy that is type IntTy.
LLVM_ABI const SCEVPredicate * getWrapPredicate(const SCEVAddRecExpr *AR, SCEVWrapPredicate::IncrementWrapFlags AddedFlags)
LLVM_ABI bool isLoopBackedgeGuardedByCond(const Loop *L, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test whether the backedge of the loop is protected by a conditional between LHS and RHS.
LLVM_ABI const SCEV * getMinusSCEV(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Return LHS-RHS.
LLVM_ABI APInt getNonZeroConstantMultiple(const SCEV *S)
const SCEV * getMinusOne(Type *Ty)
Return a SCEV for the constant -1 of a specific type.
static SCEV::NoWrapFlags setFlags(SCEV::NoWrapFlags Flags, SCEV::NoWrapFlags OnFlags)
LLVM_ABI bool hasLoopInvariantBackedgeTakenCount(const Loop *L)
Return true if the specified loop has an analyzable loop-invariant backedge-taken count.
LLVM_ABI BlockDisposition getBlockDisposition(const SCEV *S, const BasicBlock *BB)
Return the "disposition" of the given SCEV with respect to the given block.
LLVM_ABI const SCEV * getNoopOrZeroExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA, FunctionAnalysisManager::Invalidator &Inv)
LLVM_ABI const SCEV * getUMinFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS, bool Sequential=false)
Promote the operands to the wider of the types using zero-extension, and then perform a umin operatio...
LLVM_ABI bool loopIsFiniteByAssumption(const Loop *L)
Return true if this loop is finite by assumption.
LLVM_ABI const SCEV * getExistingSCEV(Value *V)
Return an existing SCEV for V if there is one, otherwise return nullptr.
LLVM_ABI APInt getConstantMultiple(const SCEV *S, const Instruction *CtxI=nullptr)
Returns the max constant multiple of S.
LoopDisposition
An enum describing the relationship between a SCEV and a loop.
@ LoopComputable
The SCEV varies predictably with the loop.
@ LoopVariant
The SCEV is loop-variant (unknown).
@ LoopInvariant
The SCEV is loop-invariant.
LLVM_ABI bool isKnownMultipleOf(const SCEV *S, uint64_t M, SmallVectorImpl< const SCEVPredicate * > &Assumptions)
Check that S is a multiple of M.
LLVM_ABI const SCEV * getAnyExtendExpr(const SCEV *Op, Type *Ty)
getAnyExtendExpr - Return a SCEV for the given operand extended with unspecified bits out to the give...
LLVM_ABI bool isKnownToBeAPowerOfTwo(const SCEV *S, bool OrZero=false, bool OrNegative=false)
Test if the given expression is known to be a power of 2.
LLVM_ABI std::optional< SCEV::NoWrapFlags > getStrengthenedNoWrapFlagsFromBinOp(const OverflowingBinaryOperator *OBO)
Parse NSW/NUW flags from add/sub/mul IR binary operation Op into SCEV no-wrap flags,...
LLVM_ABI void forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V)
Forget LCSSA phi node V of loop L to which a new predecessor was added, such that it may no longer be...
LLVM_ABI bool containsUndefs(const SCEV *S) const
Return true if the SCEV expression contains an undef value.
LLVM_ABI std::optional< MonotonicPredicateType > getMonotonicPredicateType(const SCEVAddRecExpr *LHS, ICmpInst::Predicate Pred)
If, for all loop invariant X, the predicate "LHS `Pred` X" is monotonically increasing or decreasing,...
LLVM_ABI const SCEV * getCouldNotCompute()
LLVM_ABI bool isAvailableAtLoopEntry(const SCEV *S, const Loop *L)
Determine if the SCEV can be evaluated at loop's entry.
LLVM_ABI uint32_t getMinTrailingZeros(const SCEV *S, const Instruction *CtxI=nullptr)
Determine the minimum number of zero bits that S is guaranteed to end in (at every loop iteration).
BlockDisposition
An enum describing the relationship between a SCEV and a basic block.
@ DominatesBlock
The SCEV dominates the block.
@ ProperlyDominatesBlock
The SCEV properly dominates the block.
@ DoesNotDominateBlock
The SCEV does not dominate the block.
LLVM_ABI const SCEV * getExitCount(const Loop *L, const BasicBlock *ExitingBlock, ExitCountKind Kind=Exact)
Return the number of times the backedge executes before the given exit would be taken; if not exactly...
LLVM_ABI const SCEV * getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth=0)
LLVM_ABI void getPoisonGeneratingValues(SmallPtrSetImpl< const Value * > &Result, const SCEV *S)
Return the set of Values that, if poison, will definitively result in S being poison as well.
LLVM_ABI void forgetLoopDispositions()
Called when the client has changed the disposition of values in this loop.
LLVM_ABI const SCEV * getVScale(Type *Ty)
LLVM_ABI unsigned getSmallConstantTripCount(const Loop *L)
Returns the exact trip count of the loop if we can compute it, and the result is a small constant.
LLVM_ABI bool hasComputableLoopEvolution(const SCEV *S, const Loop *L)
Return true if the given SCEV changes value in a known way in the specified loop.
LLVM_ABI const SCEV * getPointerBase(const SCEV *V)
Transitively follow the chain of pointer-type operands until reaching a SCEV that does not have a sin...
LLVM_ABI const SCEV * getMinMaxExpr(SCEVTypes Kind, SmallVectorImpl< const SCEV * > &Operands)
LLVM_ABI void forgetAllLoops()
LLVM_ABI bool dominates(const SCEV *S, const BasicBlock *BB)
Return true if elements that makes up the given SCEV dominate the specified basic block.
APInt getUnsignedRangeMax(const SCEV *S)
Determine the max of the unsigned range for a particular SCEV.
ExitCountKind
The terms "backedge taken count" and "exit count" are used interchangeably to refer to the number of ...
@ SymbolicMaximum
An expression which provides an upper bound on the exact trip count.
@ ConstantMaximum
A constant which provides an upper bound on the exact trip count.
@ Exact
An expression exactly describing the number of times the backedge has executed when a loop is exited.
LLVM_ABI const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
LLVM_ABI const SCEV * getMulExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical multiply expression, or something simpler if possible.
LLVM_ABI const SCEVAddRecExpr * convertSCEVToAddRecWithPredicates(const SCEV *S, const Loop *L, SmallVectorImpl< const SCEVPredicate * > &Preds)
Tries to convert the S expression to an AddRec expression, adding additional predicates to Preds as r...
LLVM_ABI const SCEV * getElementSize(Instruction *Inst)
Return the size of an element read or written by Inst.
LLVM_ABI const SCEV * getSizeOfExpr(Type *IntTy, TypeSize Size)
Return an expression for a TypeSize.
LLVM_ABI std::optional< bool > evaluatePredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Check whether the condition described by Pred, LHS, and RHS is true or false.
LLVM_ABI const SCEV * getUnknown(Value *V)
LLVM_ABI std::optional< std::pair< const SCEV *, SmallVector< const SCEVPredicate *, 3 > > > createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI)
Checks if SymbolicPHI can be rewritten as an AddRecExpr under some Predicates.
LLVM_ABI const SCEV * getTruncateOrZeroExtend(const SCEV *V, Type *Ty, unsigned Depth=0)
Return a SCEV corresponding to a conversion of the input value to the specified type.
static SCEV::NoWrapFlags maskFlags(SCEV::NoWrapFlags Flags, int Mask)
Convenient NoWrapFlags manipulation that hides enum casts and is visible in the ScalarEvolution name ...
LLVM_ABI std::optional< APInt > computeConstantDifference(const SCEV *LHS, const SCEV *RHS)
Compute LHS - RHS and returns the result as an APInt if it is a constant, and std::nullopt if it isn'...
LLVM_ABI bool properlyDominates(const SCEV *S, const BasicBlock *BB)
Return true if elements that makes up the given SCEV properly dominate the specified basic block.
LLVM_ABI const SCEV * rewriteUsingPredicate(const SCEV *S, const Loop *L, const SCEVPredicate &A)
Re-writes the SCEV according to the Predicates in A.
LLVM_ABI std::pair< const SCEV *, const SCEV * > SplitIntoInitAndPostInc(const Loop *L, const SCEV *S)
Splits SCEV expression S into two SCEVs.
LLVM_ABI bool canReuseInstruction(const SCEV *S, Instruction *I, SmallVectorImpl< Instruction * > &DropPoisonGeneratingInsts)
Check whether it is poison-safe to represent the expression S using the instruction I.
LLVM_ABI bool isKnownPredicateAt(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Instruction *CtxI)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
LLVM_ABI const SCEV * getPredicatedSymbolicMaxBackedgeTakenCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > &Predicates)
Similar to getSymbolicMaxBackedgeTakenCount, except it will add a set of SCEV predicates to Predicate...
LLVM_ABI const SCEV * getUDivExactExpr(const SCEV *LHS, const SCEV *RHS)
Get a canonical unsigned division expression, or something simpler if possible.
LLVM_ABI void registerUser(const SCEV *User, ArrayRef< const SCEV * > Ops)
Notify this ScalarEvolution that User directly uses SCEVs in Ops.
LLVM_ABI const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
LLVM_ABI bool isBasicBlockEntryGuardedByCond(const BasicBlock *BB, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test whether entry to the basic block is protected by a conditional between LHS and RHS.
LLVM_ABI const SCEV * getTruncateOrSignExtend(const SCEV *V, Type *Ty, unsigned Depth=0)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI bool containsErasedValue(const SCEV *S) const
Return true if the SCEV expression contains a Value that has been optimised out and is now a nullptr.
LLVM_ABI bool isKnownPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
LLVM_ABI bool isKnownViaInduction(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
We'd like to check the predicate on every iteration of the most dominated loop between loops used in ...
const SCEV * getSymbolicMaxBackedgeTakenCount(const Loop *L)
When successful, this returns a SCEV that is greater than or equal to (i.e.
APInt getSignedRangeMax(const SCEV *S)
Determine the max of the signed range for a particular SCEV.
LLVM_ABI void verify() const
LLVMContext & getContext() const
Implements a dense probed hash-table based set with some number of buckets stored inline.
Definition DenseSet.h:291
size_type size() const
Definition SmallPtrSet.h:99
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition DataLayout.h:712
TypeSize getElementOffset(unsigned Idx) const
Definition DataLayout.h:743
TypeSize getSizeInBits() const
Definition DataLayout.h:723
Class to represent struct types.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:297
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:267
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:295
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:198
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:294
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
Definition Type.h:255
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:240
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:301
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
op_range operands()
Definition User.h:292
Use & Op()
Definition User.h:196
Value * getOperand(unsigned i) const
Definition User.h:232
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256
unsigned getValueID() const
Return an ID for the concrete type of this object.
Definition Value.h:543
LLVM_ABI void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.cpp:1099
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
const ParentTy * getParent() const
Definition ilist_node.h:34
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
raw_ostream & indent(unsigned NumSpaces)
indent - Insert 'NumSpaces' spaces.
Changed
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
const APInt & smin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be signed.
Definition APInt.h:2248
const APInt & smax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be signed.
Definition APInt.h:2253
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be unsigned.
Definition APInt.h:2258
LLVM_ABI std::optional< APInt > SolveQuadraticEquationWrap(APInt A, APInt B, APInt C, unsigned RangeWidth)
Let q(n) = An^2 + Bn + C, and BW = bit width of the value range (e.g.
Definition APInt.cpp:2812
const APInt & umax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be unsigned.
Definition APInt.h:2263
LLVM_ABI APInt GreatestCommonDivisor(APInt A, APInt B)
Compute GCD of two unsigned APInt values.
Definition APInt.cpp:798
@ Entry
Definition COFF.h:862
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
int getMinValue(MCInstrInfo const &MCII, MCInst const &MCI)
Return the minimum value of an extendable operand.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:81
LLVM_ABI Function * getDeclarationIfExists(const Module *M, ID id)
Look up the Function declaration of the intrinsic id in the Module M and return it if it exists.
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
bool match(Val *V, const Pattern &P)
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
IntrinsicID_match m_Intrinsic()
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
ExtractValue_match< Ind, Val_t > m_ExtractValue(const Val_t &V)
Match a single index ExtractValue instruction.
bind_ty< WithOverflowInst > m_WithOverflowInst(WithOverflowInst *&I)
Match a with overflow intrinsic, capturing it if we match.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
brc_match< Cond_t, bind_ty< BasicBlock >, bind_ty< BasicBlock > > m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F)
BinaryOp_match< LHS, RHS, Instruction::SDiv > m_SDiv(const LHS &L, const RHS &R)
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
class_match< BasicBlock > m_BasicBlock()
Match an arbitrary basic block value and ignore it.
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
class_match< const SCEVVScale > m_SCEVVScale()
bind_cst_ty m_scev_APInt(const APInt *&C)
Match an SCEV constant and bind it to an APInt.
cst_pred_ty< is_all_ones > m_scev_AllOnes()
Match an integer with all bits set.
SCEVUnaryExpr_match< SCEVZeroExtendExpr, Op0_t > m_scev_ZExt(const Op0_t &Op0)
SCEVBinaryExpr_match< SCEVMinMaxExpr, Op0_t, Op1_t > m_scev_MinMax(const Op0_t &Op0, const Op1_t &Op1)
class_match< const SCEVConstant > m_SCEVConstant()
cst_pred_ty< is_one > m_scev_One()
Match an integer 1.
specificloop_ty m_SpecificLoop(const Loop *L)
SCEVAffineAddRec_match< Op0_t, Op1_t, class_match< const Loop > > m_scev_AffineAddRec(const Op0_t &Op0, const Op1_t &Op1)
bind_ty< const SCEVMulExpr > m_scev_Mul(const SCEVMulExpr *&V)
SCEVUnaryExpr_match< SCEVSignExtendExpr, Op0_t > m_scev_SExt(const Op0_t &Op0)
cst_pred_ty< is_zero > m_scev_Zero()
Match an integer 0.
SCEVUnaryExpr_match< SCEVTruncateExpr, Op0_t > m_scev_Trunc(const Op0_t &Op0)
bool match(const SCEV *S, const Pattern &P)
SCEVBinaryExpr_match< SCEVUDivExpr, Op0_t, Op1_t > m_scev_UDiv(const Op0_t &Op0, const Op1_t &Op1)
specificscev_ty m_scev_Specific(const SCEV *S)
Match if we have a specific specified SCEV.
SCEVBinaryExpr_match< SCEVMulExpr, Op0_t, Op1_t, SCEV::FlagNUW, true > m_scev_c_NUWMul(const Op0_t &Op0, const Op1_t &Op1)
class_match< const Loop > m_Loop()
bind_ty< const SCEVAddExpr > m_scev_Add(const SCEVAddExpr *&V)
bind_ty< const SCEVUnknown > m_SCEVUnknown(const SCEVUnknown *&V)
SCEVBinaryExpr_match< SCEVMulExpr, Op0_t, Op1_t, SCEV::FlagAnyWrap, true > m_scev_c_Mul(const Op0_t &Op0, const Op1_t &Op1)
SCEVBinaryExpr_match< SCEVSMaxExpr, Op0_t, Op1_t > m_scev_SMax(const Op0_t &Op0, const Op1_t &Op1)
SCEVURem_match< Op0_t, Op1_t > m_scev_URem(Op0_t LHS, Op1_t RHS, ScalarEvolution &SE)
Match the mathematical pattern A - (A / B) * B, where A and B can be arbitrary expressions.
class_match< const SCEV > m_SCEV()
initializer< Ty > init(const Ty &Val)
LocationClass< Ty > location(Ty &L)
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
Definition CoroShape.h:31
constexpr double e
NodeAddr< PhiNode * > Phi
Definition RDFGraph.h:390
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
void visitAll(const SCEV *Root, SV &Visitor)
Use SCEVTraversal to visit all nodes in the given expression tree.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:477
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt gcd(const DynamicAPInt &A, const DynamicAPInt &B)
void stable_sort(R &&Range)
Definition STLExtras.h:2058
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1725
SaveAndRestore(T &) -> SaveAndRestore< T >
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
LLVM_ABI bool canCreatePoison(const Operator *Op, bool ConsiderFlagsAndMetadata=true)
LLVM_ABI bool mustTriggerUB(const Instruction *I, const SmallPtrSetImpl< const Value * > &KnownPoison)
Return true if the given instruction must trigger undefined behavior when I is executed with any oper...
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition ScopeExit.h:59
LLVM_ABI bool canConstantFoldCallTo(const CallBase *Call, const Function *F)
canConstantFoldCallTo - Return true if its even possible to fold a call to the specified function.
InterleavedRange< Range > interleaved(const Range &R, StringRef Separator=", ", StringRef Prefix="", StringRef Suffix="")
Output range R as a sequence of interleaved elements.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
auto successors(const MachineBasicBlock *BB)
constexpr from_range_t from_range
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
bool set_is_subset(const S1Ty &S1, const S2Ty &S2)
set_is_subset(A, B) - Return true iff A in B
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2136
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition MathExtras.h:243
LLVM_ABI Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
unsigned short computeExpressionSize(ArrayRef< const SCEV * > Args)
void * PointerTy
LLVM_ABI bool VerifySCEV
auto uninitialized_copy(R &&Src, IterTy Dst)
Definition STLExtras.h:2053
bool isa_and_nonnull(const Y &Val)
Definition Casting.h:676
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:202
LLVM_ABI Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q)
See if we can compute a simplified version of this instruction.
LLVM_ABI bool isOverflowIntrinsicNoWrap(const WithOverflowInst *WO, const DominatorTree &DT)
Returns true if the arithmetic part of the WO 's result is used only along the paths control dependen...
DomTreeNodeBase< BasicBlock > DomTreeNode
Definition Dominators.h:95
LLVM_ABI bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start, Value *&Step)
Attempt to match a simple first order recurrence cycle of the form: iv = phi Ty [Start,...
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
Definition STLExtras.h:2128
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1732
iterator_range< pointee_iterator< WrappedIteratorT > > make_pointee_range(RangeT &&Range)
Definition iterator.h:336
auto reverse(ContainerTy &&C)
Definition STLExtras.h:406
LLVM_ABI bool isMustProgress(const Loop *L)
Return true if this loop can be assumed to make progress.
LLVM_ABI bool impliesPoison(const Value *ValAssumedPoison, const Value *V)
Return true if V is poison given that ValAssumedPoison is already poison.
LLVM_ABI bool isFinite(const Loop *L)
Return true if this loop can be assumed to run for a finite number of iterations.
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
LLVM_ABI bool programUndefinedIfPoison(const Instruction *Inst)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
bool isPointerTy(const Type *T)
Definition SPIRVUtils.h:339
FunctionAddr VTableAddr Count
Definition InstrProf.h:139
LLVM_ABI ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:71
LLVM_ABI bool propagatesPoison(const Use &PoisonOp)
Return true if PoisonOp's user yields poison or raises UB if its operand PoisonOp is poison.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Mul
Product of integers.
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ SMin
Signed integer min implemented in terms of select(cmp()).
@ Add
Sum of integers.
@ UMax
Unsigned integer max implemented in terms of select(cmp()).
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition STLExtras.h:1954
DWARFExpression::Operation Op
auto max_element(R &&Range)
Provide wrappers to std::max_element which take ranges instead of having to pass begin/end explicitly...
Definition STLExtras.h:2030
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI unsigned ComputeNumSignBits(const Value *Op, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Return the number of times the sign bit of the register is replicated into the other bits.
constexpr unsigned BitWidth
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1867
LLVM_ABI bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition STLExtras.h:1961
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
Definition MathExtras.h:248
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1897
iterator_range< df_iterator< T > > depth_first(const T &G)
auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:305
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI bool isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Returns true if V cannot be poison, but may be undef.
LLVM_ABI Constant * ConstantFoldInstOperands(const Instruction *I, ArrayRef< Constant * > Ops, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, bool AllowNonDeterministic=true)
ConstantFoldInstOperands - Attempt to constant fold an instruction with the specified operands.
bool SCEVExprContains(const SCEV *Root, PredTy Pred)
Return true if any node in Root satisfies the predicate Pred.
Implement std::hash so that hash_code can be used in STL containers.
Definition BitVector.h:867
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:869
#define N
#define NC
Definition regutils.h:42
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition Analysis.h:29
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
Definition KnownBits.h:301
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition KnownBits.h:108
static LLVM_ABI KnownBits ashr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for ashr(LHS, RHS).
unsigned getBitWidth() const
Get the bit width of this value.
Definition KnownBits.h:44
static LLVM_ABI KnownBits lshr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for lshr(LHS, RHS).
KnownBits zextOrTrunc(unsigned BitWidth) const
Return known bits for a zero extension or truncation of the value we're tracking.
Definition KnownBits.h:196
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition KnownBits.h:145
APInt getMinValue() const
Return the minimal unsigned value possible given these KnownBits.
Definition KnownBits.h:129
bool isNegative() const
Returns true if this value is known to be negative.
Definition KnownBits.h:105
static LLVM_ABI KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
An object of this class is returned by queries that could not be answered.
static LLVM_ABI bool classof(const SCEV *S)
Methods for support type inquiry through isa, cast, and dyn_cast:
This class defines a simple visitor class that may be used for various SCEV analysis purposes.
A utility class that uses RAII to save and restore the value of a variable.
Information about the number of loop iterations for which a loop exit's branch condition evaluates to...
LLVM_ABI ExitLimit(const SCEV *E)
Construct either an exact exit limit from a constant, or an unknown one from a SCEVCouldNotCompute.
SmallVector< const SCEVPredicate *, 4 > Predicates
A vector of predicate guards for this ExitLimit.