Bug Summary

File:llvm/lib/Analysis/ScalarEvolution.cpp
Warning:line 7602, column 35
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name ScalarEvolution.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Analysis -I /build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/llvm/lib/Analysis -I include -I /build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-01-19-134126-35450-1 -x c++ /build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/llvm/lib/Analysis/ScalarEvolution.cpp

/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/llvm/lib/Analysis/ScalarEvolution.cpp

1//===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the scalar evolution analysis
10// engine, which is used primarily to analyze expressions involving induction
11// variables in loops.
12//
13// There are several aspects to this library. First is the representation of
14// scalar expressions, which are represented as subclasses of the SCEV class.
15// These classes are used to represent certain types of subexpressions that we
16// can handle. We only create one SCEV of a particular shape, so
17// pointer-comparisons for equality are legal.
18//
19// One important aspect of the SCEV objects is that they are never cyclic, even
20// if there is a cycle in the dataflow for an expression (ie, a PHI node). If
21// the PHI node is one of the idioms that we can represent (e.g., a polynomial
22// recurrence) then we represent it directly as a recurrence node, otherwise we
23// represent it as a SCEVUnknown node.
24//
25// In addition to being able to represent expressions of various types, we also
26// have folders that are used to build the *canonical* representation for a
27// particular expression. These folders are capable of using a variety of
28// rewrite rules to simplify the expressions.
29//
30// Once the folders are defined, we can implement the more interesting
31// higher-level code, such as the code that recognizes PHI nodes of various
32// types, computes the execution count of a loop, etc.
33//
34// TODO: We should use these routines and value representations to implement
35// dependence analysis!
36//
37//===----------------------------------------------------------------------===//
38//
39// There are several good references for the techniques used in this analysis.
40//
41// Chains of recurrences -- a method to expedite the evaluation
42// of closed-form functions
43// Olaf Bachmann, Paul S. Wang, Eugene V. Zima
44//
45// On computational properties of chains of recurrences
46// Eugene V. Zima
47//
48// Symbolic Evaluation of Chains of Recurrences for Loop Optimization
49// Robert A. van Engelen
50//
51// Efficient Symbolic Analysis for Optimizing Compilers
52// Robert A. van Engelen
53//
54// Using the chains of recurrences algebra for data dependence testing and
55// induction variable substitution
56// MS Thesis, Johnie Birch
57//
58//===----------------------------------------------------------------------===//
59
60#include "llvm/Analysis/ScalarEvolution.h"
61#include "llvm/ADT/APInt.h"
62#include "llvm/ADT/ArrayRef.h"
63#include "llvm/ADT/DenseMap.h"
64#include "llvm/ADT/DepthFirstIterator.h"
65#include "llvm/ADT/EquivalenceClasses.h"
66#include "llvm/ADT/FoldingSet.h"
67#include "llvm/ADT/None.h"
68#include "llvm/ADT/Optional.h"
69#include "llvm/ADT/STLExtras.h"
70#include "llvm/ADT/ScopeExit.h"
71#include "llvm/ADT/Sequence.h"
72#include "llvm/ADT/SetVector.h"
73#include "llvm/ADT/SmallPtrSet.h"
74#include "llvm/ADT/SmallSet.h"
75#include "llvm/ADT/SmallVector.h"
76#include "llvm/ADT/Statistic.h"
77#include "llvm/ADT/StringRef.h"
78#include "llvm/Analysis/AssumptionCache.h"
79#include "llvm/Analysis/ConstantFolding.h"
80#include "llvm/Analysis/InstructionSimplify.h"
81#include "llvm/Analysis/LoopInfo.h"
82#include "llvm/Analysis/ScalarEvolutionDivision.h"
83#include "llvm/Analysis/ScalarEvolutionExpressions.h"
84#include "llvm/Analysis/TargetLibraryInfo.h"
85#include "llvm/Analysis/ValueTracking.h"
86#include "llvm/Config/llvm-config.h"
87#include "llvm/IR/Argument.h"
88#include "llvm/IR/BasicBlock.h"
89#include "llvm/IR/CFG.h"
90#include "llvm/IR/Constant.h"
91#include "llvm/IR/ConstantRange.h"
92#include "llvm/IR/Constants.h"
93#include "llvm/IR/DataLayout.h"
94#include "llvm/IR/DerivedTypes.h"
95#include "llvm/IR/Dominators.h"
96#include "llvm/IR/Function.h"
97#include "llvm/IR/GlobalAlias.h"
98#include "llvm/IR/GlobalValue.h"
99#include "llvm/IR/GlobalVariable.h"
100#include "llvm/IR/InstIterator.h"
101#include "llvm/IR/InstrTypes.h"
102#include "llvm/IR/Instruction.h"
103#include "llvm/IR/Instructions.h"
104#include "llvm/IR/IntrinsicInst.h"
105#include "llvm/IR/Intrinsics.h"
106#include "llvm/IR/LLVMContext.h"
107#include "llvm/IR/Metadata.h"
108#include "llvm/IR/Operator.h"
109#include "llvm/IR/PatternMatch.h"
110#include "llvm/IR/Type.h"
111#include "llvm/IR/Use.h"
112#include "llvm/IR/User.h"
113#include "llvm/IR/Value.h"
114#include "llvm/IR/Verifier.h"
115#include "llvm/InitializePasses.h"
116#include "llvm/Pass.h"
117#include "llvm/Support/Casting.h"
118#include "llvm/Support/CommandLine.h"
119#include "llvm/Support/Compiler.h"
120#include "llvm/Support/Debug.h"
121#include "llvm/Support/ErrorHandling.h"
122#include "llvm/Support/KnownBits.h"
123#include "llvm/Support/SaveAndRestore.h"
124#include "llvm/Support/raw_ostream.h"
125#include <algorithm>
126#include <cassert>
127#include <climits>
128#include <cstddef>
129#include <cstdint>
130#include <cstdlib>
131#include <map>
132#include <memory>
133#include <tuple>
134#include <utility>
135#include <vector>
136
137using namespace llvm;
138using namespace PatternMatch;
139
140#define DEBUG_TYPE"scalar-evolution" "scalar-evolution"
141
142STATISTIC(NumTripCountsComputed,static llvm::Statistic NumTripCountsComputed = {"scalar-evolution"
, "NumTripCountsComputed", "Number of loops with predictable loop counts"
}
143 "Number of loops with predictable loop counts")static llvm::Statistic NumTripCountsComputed = {"scalar-evolution"
, "NumTripCountsComputed", "Number of loops with predictable loop counts"
}
;
144STATISTIC(NumTripCountsNotComputed,static llvm::Statistic NumTripCountsNotComputed = {"scalar-evolution"
, "NumTripCountsNotComputed", "Number of loops without predictable loop counts"
}
145 "Number of loops without predictable loop counts")static llvm::Statistic NumTripCountsNotComputed = {"scalar-evolution"
, "NumTripCountsNotComputed", "Number of loops without predictable loop counts"
}
;
146STATISTIC(NumBruteForceTripCountsComputed,static llvm::Statistic NumBruteForceTripCountsComputed = {"scalar-evolution"
, "NumBruteForceTripCountsComputed", "Number of loops with trip counts computed by force"
}
147 "Number of loops with trip counts computed by force")static llvm::Statistic NumBruteForceTripCountsComputed = {"scalar-evolution"
, "NumBruteForceTripCountsComputed", "Number of loops with trip counts computed by force"
}
;
148
149static cl::opt<unsigned>
150MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
151 cl::ZeroOrMore,
152 cl::desc("Maximum number of iterations SCEV will "
153 "symbolically execute a constant "
154 "derived loop"),
155 cl::init(100));
156
157// FIXME: Enable this with EXPENSIVE_CHECKS when the test suite is clean.
158static cl::opt<bool> VerifySCEV(
159 "verify-scev", cl::Hidden,
160 cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
161static cl::opt<bool> VerifySCEVStrict(
162 "verify-scev-strict", cl::Hidden,
163 cl::desc("Enable stricter verification with -verify-scev is passed"));
164static cl::opt<bool>
165 VerifySCEVMap("verify-scev-maps", cl::Hidden,
166 cl::desc("Verify no dangling value in ScalarEvolution's "
167 "ExprValueMap (slow)"));
168
169static cl::opt<bool> VerifyIR(
170 "scev-verify-ir", cl::Hidden,
171 cl::desc("Verify IR correctness when making sensitive SCEV queries (slow)"),
172 cl::init(false));
173
174static cl::opt<unsigned> MulOpsInlineThreshold(
175 "scev-mulops-inline-threshold", cl::Hidden,
176 cl::desc("Threshold for inlining multiplication operands into a SCEV"),
177 cl::init(32));
178
179static cl::opt<unsigned> AddOpsInlineThreshold(
180 "scev-addops-inline-threshold", cl::Hidden,
181 cl::desc("Threshold for inlining addition operands into a SCEV"),
182 cl::init(500));
183
184static cl::opt<unsigned> MaxSCEVCompareDepth(
185 "scalar-evolution-max-scev-compare-depth", cl::Hidden,
186 cl::desc("Maximum depth of recursive SCEV complexity comparisons"),
187 cl::init(32));
188
189static cl::opt<unsigned> MaxSCEVOperationsImplicationDepth(
190 "scalar-evolution-max-scev-operations-implication-depth", cl::Hidden,
191 cl::desc("Maximum depth of recursive SCEV operations implication analysis"),
192 cl::init(2));
193
194static cl::opt<unsigned> MaxValueCompareDepth(
195 "scalar-evolution-max-value-compare-depth", cl::Hidden,
196 cl::desc("Maximum depth of recursive value complexity comparisons"),
197 cl::init(2));
198
199static cl::opt<unsigned>
200 MaxArithDepth("scalar-evolution-max-arith-depth", cl::Hidden,
201 cl::desc("Maximum depth of recursive arithmetics"),
202 cl::init(32));
203
204static cl::opt<unsigned> MaxConstantEvolvingDepth(
205 "scalar-evolution-max-constant-evolving-depth", cl::Hidden,
206 cl::desc("Maximum depth of recursive constant evolving"), cl::init(32));
207
208static cl::opt<unsigned>
209 MaxCastDepth("scalar-evolution-max-cast-depth", cl::Hidden,
210 cl::desc("Maximum depth of recursive SExt/ZExt/Trunc"),
211 cl::init(8));
212
213static cl::opt<unsigned>
214 MaxAddRecSize("scalar-evolution-max-add-rec-size", cl::Hidden,
215 cl::desc("Max coefficients in AddRec during evolving"),
216 cl::init(8));
217
218static cl::opt<unsigned>
219 HugeExprThreshold("scalar-evolution-huge-expr-threshold", cl::Hidden,
220 cl::desc("Size of the expression which is considered huge"),
221 cl::init(4096));
222
223static cl::opt<bool>
224ClassifyExpressions("scalar-evolution-classify-expressions",
225 cl::Hidden, cl::init(true),
226 cl::desc("When printing analysis, include information on every instruction"));
227
228static cl::opt<bool> UseExpensiveRangeSharpening(
229 "scalar-evolution-use-expensive-range-sharpening", cl::Hidden,
230 cl::init(false),
231 cl::desc("Use more powerful methods of sharpening expression ranges. May "
232 "be costly in terms of compile time"));
233
234//===----------------------------------------------------------------------===//
235// SCEV class definitions
236//===----------------------------------------------------------------------===//
237
238//===----------------------------------------------------------------------===//
239// Implementation of the SCEV class.
240//
241
242#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
243LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void SCEV::dump() const {
244 print(dbgs());
245 dbgs() << '\n';
246}
247#endif
248
249void SCEV::print(raw_ostream &OS) const {
250 switch (getSCEVType()) {
251 case scConstant:
252 cast<SCEVConstant>(this)->getValue()->printAsOperand(OS, false);
253 return;
254 case scPtrToInt: {
255 const SCEVPtrToIntExpr *PtrToInt = cast<SCEVPtrToIntExpr>(this);
256 const SCEV *Op = PtrToInt->getOperand();
257 OS << "(ptrtoint " << *Op->getType() << " " << *Op << " to "
258 << *PtrToInt->getType() << ")";
259 return;
260 }
261 case scTruncate: {
262 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(this);
263 const SCEV *Op = Trunc->getOperand();
264 OS << "(trunc " << *Op->getType() << " " << *Op << " to "
265 << *Trunc->getType() << ")";
266 return;
267 }
268 case scZeroExtend: {
269 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(this);
270 const SCEV *Op = ZExt->getOperand();
271 OS << "(zext " << *Op->getType() << " " << *Op << " to "
272 << *ZExt->getType() << ")";
273 return;
274 }
275 case scSignExtend: {
276 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(this);
277 const SCEV *Op = SExt->getOperand();
278 OS << "(sext " << *Op->getType() << " " << *Op << " to "
279 << *SExt->getType() << ")";
280 return;
281 }
282 case scAddRecExpr: {
283 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(this);
284 OS << "{" << *AR->getOperand(0);
285 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
286 OS << ",+," << *AR->getOperand(i);
287 OS << "}<";
288 if (AR->hasNoUnsignedWrap())
289 OS << "nuw><";
290 if (AR->hasNoSignedWrap())
291 OS << "nsw><";
292 if (AR->hasNoSelfWrap() &&
293 !AR->getNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW)))
294 OS << "nw><";
295 AR->getLoop()->getHeader()->printAsOperand(OS, /*PrintType=*/false);
296 OS << ">";
297 return;
298 }
299 case scAddExpr:
300 case scMulExpr:
301 case scUMaxExpr:
302 case scSMaxExpr:
303 case scUMinExpr:
304 case scSMinExpr:
305 case scSequentialUMinExpr: {
306 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(this);
307 const char *OpStr = nullptr;
308 switch (NAry->getSCEVType()) {
309 case scAddExpr: OpStr = " + "; break;
310 case scMulExpr: OpStr = " * "; break;
311 case scUMaxExpr: OpStr = " umax "; break;
312 case scSMaxExpr: OpStr = " smax "; break;
313 case scUMinExpr:
314 OpStr = " umin ";
315 break;
316 case scSMinExpr:
317 OpStr = " smin ";
318 break;
319 case scSequentialUMinExpr:
320 OpStr = " umin_seq ";
321 break;
322 default:
323 llvm_unreachable("There are no other nary expression types.")::llvm::llvm_unreachable_internal("There are no other nary expression types."
, "llvm/lib/Analysis/ScalarEvolution.cpp", 323)
;
324 }
325 OS << "(";
326 ListSeparator LS(OpStr);
327 for (const SCEV *Op : NAry->operands())
328 OS << LS << *Op;
329 OS << ")";
330 switch (NAry->getSCEVType()) {
331 case scAddExpr:
332 case scMulExpr:
333 if (NAry->hasNoUnsignedWrap())
334 OS << "<nuw>";
335 if (NAry->hasNoSignedWrap())
336 OS << "<nsw>";
337 break;
338 default:
339 // Nothing to print for other nary expressions.
340 break;
341 }
342 return;
343 }
344 case scUDivExpr: {
345 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(this);
346 OS << "(" << *UDiv->getLHS() << " /u " << *UDiv->getRHS() << ")";
347 return;
348 }
349 case scUnknown: {
350 const SCEVUnknown *U = cast<SCEVUnknown>(this);
351 Type *AllocTy;
352 if (U->isSizeOf(AllocTy)) {
353 OS << "sizeof(" << *AllocTy << ")";
354 return;
355 }
356 if (U->isAlignOf(AllocTy)) {
357 OS << "alignof(" << *AllocTy << ")";
358 return;
359 }
360
361 Type *CTy;
362 Constant *FieldNo;
363 if (U->isOffsetOf(CTy, FieldNo)) {
364 OS << "offsetof(" << *CTy << ", ";
365 FieldNo->printAsOperand(OS, false);
366 OS << ")";
367 return;
368 }
369
370 // Otherwise just print it normally.
371 U->getValue()->printAsOperand(OS, false);
372 return;
373 }
374 case scCouldNotCompute:
375 OS << "***COULDNOTCOMPUTE***";
376 return;
377 }
378 llvm_unreachable("Unknown SCEV kind!")::llvm::llvm_unreachable_internal("Unknown SCEV kind!", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 378)
;
379}
380
381Type *SCEV::getType() const {
382 switch (getSCEVType()) {
383 case scConstant:
384 return cast<SCEVConstant>(this)->getType();
385 case scPtrToInt:
386 case scTruncate:
387 case scZeroExtend:
388 case scSignExtend:
389 return cast<SCEVCastExpr>(this)->getType();
390 case scAddRecExpr:
391 return cast<SCEVAddRecExpr>(this)->getType();
392 case scMulExpr:
393 return cast<SCEVMulExpr>(this)->getType();
394 case scUMaxExpr:
395 case scSMaxExpr:
396 case scUMinExpr:
397 case scSMinExpr:
398 return cast<SCEVMinMaxExpr>(this)->getType();
399 case scSequentialUMinExpr:
400 return cast<SCEVSequentialMinMaxExpr>(this)->getType();
401 case scAddExpr:
402 return cast<SCEVAddExpr>(this)->getType();
403 case scUDivExpr:
404 return cast<SCEVUDivExpr>(this)->getType();
405 case scUnknown:
406 return cast<SCEVUnknown>(this)->getType();
407 case scCouldNotCompute:
408 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!")::llvm::llvm_unreachable_internal("Attempt to use a SCEVCouldNotCompute object!"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 408)
;
409 }
410 llvm_unreachable("Unknown SCEV kind!")::llvm::llvm_unreachable_internal("Unknown SCEV kind!", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 410)
;
411}
412
413bool SCEV::isZero() const {
414 if (const SCEVConstant *SC
42.1
'SC' is null
42.1
'SC' is null
42.1
'SC' is null
42.1
'SC' is null
= dyn_cast<SCEVConstant>(this))
42
Assuming the object is not a 'SCEVConstant'
43
Taking false branch
415 return SC->getValue()->isZero();
416 return false;
44
Returning zero, which participates in a condition later
417}
418
419bool SCEV::isOne() const {
420 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
421 return SC->getValue()->isOne();
422 return false;
423}
424
425bool SCEV::isAllOnesValue() const {
426 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
427 return SC->getValue()->isMinusOne();
428 return false;
429}
430
431bool SCEV::isNonConstantNegative() const {
432 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(this);
433 if (!Mul) return false;
434
435 // If there is a constant factor, it will be first.
436 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0));
437 if (!SC) return false;
438
439 // Return true if the value is negative, this matches things like (-42 * V).
440 return SC->getAPInt().isNegative();
441}
442
443SCEVCouldNotCompute::SCEVCouldNotCompute() :
444 SCEV(FoldingSetNodeIDRef(), scCouldNotCompute, 0) {}
445
446bool SCEVCouldNotCompute::classof(const SCEV *S) {
447 return S->getSCEVType() == scCouldNotCompute;
448}
449
450const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
451 FoldingSetNodeID ID;
452 ID.AddInteger(scConstant);
453 ID.AddPointer(V);
454 void *IP = nullptr;
455 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
456 SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V);
457 UniqueSCEVs.InsertNode(S, IP);
458 return S;
459}
460
461const SCEV *ScalarEvolution::getConstant(const APInt &Val) {
462 return getConstant(ConstantInt::get(getContext(), Val));
463}
464
465const SCEV *
466ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
467 IntegerType *ITy = cast<IntegerType>(getEffectiveSCEVType(Ty));
468 return getConstant(ConstantInt::get(ITy, V, isSigned));
469}
470
471SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
472 const SCEV *op, Type *ty)
473 : SCEV(ID, SCEVTy, computeExpressionSize(op)), Ty(ty) {
474 Operands[0] = op;
475}
476
477SCEVPtrToIntExpr::SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, const SCEV *Op,
478 Type *ITy)
479 : SCEVCastExpr(ID, scPtrToInt, Op, ITy) {
480 assert(getOperand()->getType()->isPointerTy() && Ty->isIntegerTy() &&(static_cast <bool> (getOperand()->getType()->isPointerTy
() && Ty->isIntegerTy() && "Must be a non-bit-width-changing pointer-to-integer cast!"
) ? void (0) : __assert_fail ("getOperand()->getType()->isPointerTy() && Ty->isIntegerTy() && \"Must be a non-bit-width-changing pointer-to-integer cast!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 481, __extension__
__PRETTY_FUNCTION__))
481 "Must be a non-bit-width-changing pointer-to-integer cast!")(static_cast <bool> (getOperand()->getType()->isPointerTy
() && Ty->isIntegerTy() && "Must be a non-bit-width-changing pointer-to-integer cast!"
) ? void (0) : __assert_fail ("getOperand()->getType()->isPointerTy() && Ty->isIntegerTy() && \"Must be a non-bit-width-changing pointer-to-integer cast!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 481, __extension__
__PRETTY_FUNCTION__))
;
482}
483
484SCEVIntegralCastExpr::SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID,
485 SCEVTypes SCEVTy, const SCEV *op,
486 Type *ty)
487 : SCEVCastExpr(ID, SCEVTy, op, ty) {}
488
489SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, const SCEV *op,
490 Type *ty)
491 : SCEVIntegralCastExpr(ID, scTruncate, op, ty) {
492 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&(static_cast <bool> (getOperand()->getType()->isIntOrPtrTy
() && Ty->isIntOrPtrTy() && "Cannot truncate non-integer value!"
) ? void (0) : __assert_fail ("getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot truncate non-integer value!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 493, __extension__
__PRETTY_FUNCTION__))
493 "Cannot truncate non-integer value!")(static_cast <bool> (getOperand()->getType()->isIntOrPtrTy
() && Ty->isIntOrPtrTy() && "Cannot truncate non-integer value!"
) ? void (0) : __assert_fail ("getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot truncate non-integer value!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 493, __extension__
__PRETTY_FUNCTION__))
;
494}
495
496SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID,
497 const SCEV *op, Type *ty)
498 : SCEVIntegralCastExpr(ID, scZeroExtend, op, ty) {
499 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&(static_cast <bool> (getOperand()->getType()->isIntOrPtrTy
() && Ty->isIntOrPtrTy() && "Cannot zero extend non-integer value!"
) ? void (0) : __assert_fail ("getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot zero extend non-integer value!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 500, __extension__
__PRETTY_FUNCTION__))
500 "Cannot zero extend non-integer value!")(static_cast <bool> (getOperand()->getType()->isIntOrPtrTy
() && Ty->isIntOrPtrTy() && "Cannot zero extend non-integer value!"
) ? void (0) : __assert_fail ("getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot zero extend non-integer value!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 500, __extension__
__PRETTY_FUNCTION__))
;
501}
502
503SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID,
504 const SCEV *op, Type *ty)
505 : SCEVIntegralCastExpr(ID, scSignExtend, op, ty) {
506 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&(static_cast <bool> (getOperand()->getType()->isIntOrPtrTy
() && Ty->isIntOrPtrTy() && "Cannot sign extend non-integer value!"
) ? void (0) : __assert_fail ("getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot sign extend non-integer value!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 507, __extension__
__PRETTY_FUNCTION__))
507 "Cannot sign extend non-integer value!")(static_cast <bool> (getOperand()->getType()->isIntOrPtrTy
() && Ty->isIntOrPtrTy() && "Cannot sign extend non-integer value!"
) ? void (0) : __assert_fail ("getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot sign extend non-integer value!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 507, __extension__
__PRETTY_FUNCTION__))
;
508}
509
510void SCEVUnknown::deleted() {
511 // Clear this SCEVUnknown from various maps.
512 SE->forgetMemoizedResults(this);
513
514 // Remove this SCEVUnknown from the uniquing map.
515 SE->UniqueSCEVs.RemoveNode(this);
516
517 // Release the value.
518 setValPtr(nullptr);
519}
520
521void SCEVUnknown::allUsesReplacedWith(Value *New) {
522 // Remove this SCEVUnknown from the uniquing map.
523 SE->UniqueSCEVs.RemoveNode(this);
524
525 // Update this SCEVUnknown to point to the new value. This is needed
526 // because there may still be outstanding SCEVs which still point to
527 // this SCEVUnknown.
528 setValPtr(New);
529}
530
531bool SCEVUnknown::isSizeOf(Type *&AllocTy) const {
532 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
533 if (VCE->getOpcode() == Instruction::PtrToInt)
534 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
535 if (CE->getOpcode() == Instruction::GetElementPtr &&
536 CE->getOperand(0)->isNullValue() &&
537 CE->getNumOperands() == 2)
538 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(1)))
539 if (CI->isOne()) {
540 AllocTy = cast<GEPOperator>(CE)->getSourceElementType();
541 return true;
542 }
543
544 return false;
545}
546
547bool SCEVUnknown::isAlignOf(Type *&AllocTy) const {
548 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
549 if (VCE->getOpcode() == Instruction::PtrToInt)
550 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
551 if (CE->getOpcode() == Instruction::GetElementPtr &&
552 CE->getOperand(0)->isNullValue()) {
553 Type *Ty = cast<GEPOperator>(CE)->getSourceElementType();
554 if (StructType *STy = dyn_cast<StructType>(Ty))
555 if (!STy->isPacked() &&
556 CE->getNumOperands() == 3 &&
557 CE->getOperand(1)->isNullValue()) {
558 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(2)))
559 if (CI->isOne() &&
560 STy->getNumElements() == 2 &&
561 STy->getElementType(0)->isIntegerTy(1)) {
562 AllocTy = STy->getElementType(1);
563 return true;
564 }
565 }
566 }
567
568 return false;
569}
570
571bool SCEVUnknown::isOffsetOf(Type *&CTy, Constant *&FieldNo) const {
572 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
573 if (VCE->getOpcode() == Instruction::PtrToInt)
574 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
575 if (CE->getOpcode() == Instruction::GetElementPtr &&
576 CE->getNumOperands() == 3 &&
577 CE->getOperand(0)->isNullValue() &&
578 CE->getOperand(1)->isNullValue()) {
579 Type *Ty = cast<GEPOperator>(CE)->getSourceElementType();
580 // Ignore vector types here so that ScalarEvolutionExpander doesn't
581 // emit getelementptrs that index into vectors.
582 if (Ty->isStructTy() || Ty->isArrayTy()) {
583 CTy = Ty;
584 FieldNo = CE->getOperand(2);
585 return true;
586 }
587 }
588
589 return false;
590}
591
592//===----------------------------------------------------------------------===//
593// SCEV Utilities
594//===----------------------------------------------------------------------===//
595
596/// Compare the two values \p LV and \p RV in terms of their "complexity" where
597/// "complexity" is a partial (and somewhat ad-hoc) relation used to order
598/// operands in SCEV expressions. \p EqCache is a set of pairs of values that
599/// have been previously deemed to be "equally complex" by this routine. It is
600/// intended to avoid exponential time complexity in cases like:
601///
602/// %a = f(%x, %y)
603/// %b = f(%a, %a)
604/// %c = f(%b, %b)
605///
606/// %d = f(%x, %y)
607/// %e = f(%d, %d)
608/// %f = f(%e, %e)
609///
610/// CompareValueComplexity(%f, %c)
611///
612/// Since we do not continue running this routine on expression trees once we
613/// have seen unequal values, there is no need to track them in the cache.
614static int
615CompareValueComplexity(EquivalenceClasses<const Value *> &EqCacheValue,
616 const LoopInfo *const LI, Value *LV, Value *RV,
617 unsigned Depth) {
618 if (Depth > MaxValueCompareDepth || EqCacheValue.isEquivalent(LV, RV))
619 return 0;
620
621 // Order pointer values after integer values. This helps SCEVExpander form
622 // GEPs.
623 bool LIsPointer = LV->getType()->isPointerTy(),
624 RIsPointer = RV->getType()->isPointerTy();
625 if (LIsPointer != RIsPointer)
626 return (int)LIsPointer - (int)RIsPointer;
627
628 // Compare getValueID values.
629 unsigned LID = LV->getValueID(), RID = RV->getValueID();
630 if (LID != RID)
631 return (int)LID - (int)RID;
632
633 // Sort arguments by their position.
634 if (const auto *LA = dyn_cast<Argument>(LV)) {
635 const auto *RA = cast<Argument>(RV);
636 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
637 return (int)LArgNo - (int)RArgNo;
638 }
639
640 if (const auto *LGV = dyn_cast<GlobalValue>(LV)) {
641 const auto *RGV = cast<GlobalValue>(RV);
642
643 const auto IsGVNameSemantic = [&](const GlobalValue *GV) {
644 auto LT = GV->getLinkage();
645 return !(GlobalValue::isPrivateLinkage(LT) ||
646 GlobalValue::isInternalLinkage(LT));
647 };
648
649 // Use the names to distinguish the two values, but only if the
650 // names are semantically important.
651 if (IsGVNameSemantic(LGV) && IsGVNameSemantic(RGV))
652 return LGV->getName().compare(RGV->getName());
653 }
654
655 // For instructions, compare their loop depth, and their operand count. This
656 // is pretty loose.
657 if (const auto *LInst = dyn_cast<Instruction>(LV)) {
658 const auto *RInst = cast<Instruction>(RV);
659
660 // Compare loop depths.
661 const BasicBlock *LParent = LInst->getParent(),
662 *RParent = RInst->getParent();
663 if (LParent != RParent) {
664 unsigned LDepth = LI->getLoopDepth(LParent),
665 RDepth = LI->getLoopDepth(RParent);
666 if (LDepth != RDepth)
667 return (int)LDepth - (int)RDepth;
668 }
669
670 // Compare the number of operands.
671 unsigned LNumOps = LInst->getNumOperands(),
672 RNumOps = RInst->getNumOperands();
673 if (LNumOps != RNumOps)
674 return (int)LNumOps - (int)RNumOps;
675
676 for (unsigned Idx : seq(0u, LNumOps)) {
677 int Result =
678 CompareValueComplexity(EqCacheValue, LI, LInst->getOperand(Idx),
679 RInst->getOperand(Idx), Depth + 1);
680 if (Result != 0)
681 return Result;
682 }
683 }
684
685 EqCacheValue.unionSets(LV, RV);
686 return 0;
687}
688
689// Return negative, zero, or positive, if LHS is less than, equal to, or greater
690// than RHS, respectively. A three-way result allows recursive comparisons to be
691// more efficient.
692// If the max analysis depth was reached, return None, assuming we do not know
693// if they are equivalent for sure.
694static Optional<int>
695CompareSCEVComplexity(EquivalenceClasses<const SCEV *> &EqCacheSCEV,
696 EquivalenceClasses<const Value *> &EqCacheValue,
697 const LoopInfo *const LI, const SCEV *LHS,
698 const SCEV *RHS, DominatorTree &DT, unsigned Depth = 0) {
699 // Fast-path: SCEVs are uniqued so we can do a quick equality check.
700 if (LHS == RHS)
701 return 0;
702
703 // Primarily, sort the SCEVs by their getSCEVType().
704 SCEVTypes LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
705 if (LType != RType)
706 return (int)LType - (int)RType;
707
708 if (EqCacheSCEV.isEquivalent(LHS, RHS))
709 return 0;
710
711 if (Depth > MaxSCEVCompareDepth)
712 return None;
713
714 // Aside from the getSCEVType() ordering, the particular ordering
715 // isn't very important except that it's beneficial to be consistent,
716 // so that (a + b) and (b + a) don't end up as different expressions.
717 switch (LType) {
718 case scUnknown: {
719 const SCEVUnknown *LU = cast<SCEVUnknown>(LHS);
720 const SCEVUnknown *RU = cast<SCEVUnknown>(RHS);
721
722 int X = CompareValueComplexity(EqCacheValue, LI, LU->getValue(),
723 RU->getValue(), Depth + 1);
724 if (X == 0)
725 EqCacheSCEV.unionSets(LHS, RHS);
726 return X;
727 }
728
729 case scConstant: {
730 const SCEVConstant *LC = cast<SCEVConstant>(LHS);
731 const SCEVConstant *RC = cast<SCEVConstant>(RHS);
732
733 // Compare constant values.
734 const APInt &LA = LC->getAPInt();
735 const APInt &RA = RC->getAPInt();
736 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
737 if (LBitWidth != RBitWidth)
738 return (int)LBitWidth - (int)RBitWidth;
739 return LA.ult(RA) ? -1 : 1;
740 }
741
742 case scAddRecExpr: {
743 const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(LHS);
744 const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(RHS);
745
746 // There is always a dominance between two recs that are used by one SCEV,
747 // so we can safely sort recs by loop header dominance. We require such
748 // order in getAddExpr.
749 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
750 if (LLoop != RLoop) {
751 const BasicBlock *LHead = LLoop->getHeader(), *RHead = RLoop->getHeader();
752 assert(LHead != RHead && "Two loops share the same header?")(static_cast <bool> (LHead != RHead && "Two loops share the same header?"
) ? void (0) : __assert_fail ("LHead != RHead && \"Two loops share the same header?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 752, __extension__
__PRETTY_FUNCTION__))
;
753 if (DT.dominates(LHead, RHead))
754 return 1;
755 else
756 assert(DT.dominates(RHead, LHead) &&(static_cast <bool> (DT.dominates(RHead, LHead) &&
"No dominance between recurrences used by one SCEV?") ? void
(0) : __assert_fail ("DT.dominates(RHead, LHead) && \"No dominance between recurrences used by one SCEV?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 757, __extension__
__PRETTY_FUNCTION__))
757 "No dominance between recurrences used by one SCEV?")(static_cast <bool> (DT.dominates(RHead, LHead) &&
"No dominance between recurrences used by one SCEV?") ? void
(0) : __assert_fail ("DT.dominates(RHead, LHead) && \"No dominance between recurrences used by one SCEV?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 757, __extension__
__PRETTY_FUNCTION__))
;
758 return -1;
759 }
760
761 // Addrec complexity grows with operand count.
762 unsigned LNumOps = LA->getNumOperands(), RNumOps = RA->getNumOperands();
763 if (LNumOps != RNumOps)
764 return (int)LNumOps - (int)RNumOps;
765
766 // Lexicographically compare.
767 for (unsigned i = 0; i != LNumOps; ++i) {
768 auto X = CompareSCEVComplexity(EqCacheSCEV, EqCacheValue, LI,
769 LA->getOperand(i), RA->getOperand(i), DT,
770 Depth + 1);
771 if (X != 0)
772 return X;
773 }
774 EqCacheSCEV.unionSets(LHS, RHS);
775 return 0;
776 }
777
778 case scAddExpr:
779 case scMulExpr:
780 case scSMaxExpr:
781 case scUMaxExpr:
782 case scSMinExpr:
783 case scUMinExpr:
784 case scSequentialUMinExpr: {
785 const SCEVNAryExpr *LC = cast<SCEVNAryExpr>(LHS);
786 const SCEVNAryExpr *RC = cast<SCEVNAryExpr>(RHS);
787
788 // Lexicographically compare n-ary expressions.
789 unsigned LNumOps = LC->getNumOperands(), RNumOps = RC->getNumOperands();
790 if (LNumOps != RNumOps)
791 return (int)LNumOps - (int)RNumOps;
792
793 for (unsigned i = 0; i != LNumOps; ++i) {
794 auto X = CompareSCEVComplexity(EqCacheSCEV, EqCacheValue, LI,
795 LC->getOperand(i), RC->getOperand(i), DT,
796 Depth + 1);
797 if (X != 0)
798 return X;
799 }
800 EqCacheSCEV.unionSets(LHS, RHS);
801 return 0;
802 }
803
804 case scUDivExpr: {
805 const SCEVUDivExpr *LC = cast<SCEVUDivExpr>(LHS);
806 const SCEVUDivExpr *RC = cast<SCEVUDivExpr>(RHS);
807
808 // Lexicographically compare udiv expressions.
809 auto X = CompareSCEVComplexity(EqCacheSCEV, EqCacheValue, LI, LC->getLHS(),
810 RC->getLHS(), DT, Depth + 1);
811 if (X != 0)
812 return X;
813 X = CompareSCEVComplexity(EqCacheSCEV, EqCacheValue, LI, LC->getRHS(),
814 RC->getRHS(), DT, Depth + 1);
815 if (X == 0)
816 EqCacheSCEV.unionSets(LHS, RHS);
817 return X;
818 }
819
820 case scPtrToInt:
821 case scTruncate:
822 case scZeroExtend:
823 case scSignExtend: {
824 const SCEVCastExpr *LC = cast<SCEVCastExpr>(LHS);
825 const SCEVCastExpr *RC = cast<SCEVCastExpr>(RHS);
826
827 // Compare cast expressions by operand.
828 auto X =
829 CompareSCEVComplexity(EqCacheSCEV, EqCacheValue, LI, LC->getOperand(),
830 RC->getOperand(), DT, Depth + 1);
831 if (X == 0)
832 EqCacheSCEV.unionSets(LHS, RHS);
833 return X;
834 }
835
836 case scCouldNotCompute:
837 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!")::llvm::llvm_unreachable_internal("Attempt to use a SCEVCouldNotCompute object!"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 837)
;
838 }
839 llvm_unreachable("Unknown SCEV kind!")::llvm::llvm_unreachable_internal("Unknown SCEV kind!", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 839)
;
840}
841
842/// Given a list of SCEV objects, order them by their complexity, and group
843/// objects of the same complexity together by value. When this routine is
844/// finished, we know that any duplicates in the vector are consecutive and that
845/// complexity is monotonically increasing.
846///
847/// Note that we go take special precautions to ensure that we get deterministic
848/// results from this routine. In other words, we don't want the results of
849/// this to depend on where the addresses of various SCEV objects happened to
850/// land in memory.
851static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops,
852 LoopInfo *LI, DominatorTree &DT) {
853 if (Ops.size() < 2) return; // Noop
854
855 EquivalenceClasses<const SCEV *> EqCacheSCEV;
856 EquivalenceClasses<const Value *> EqCacheValue;
857
858 // Whether LHS has provably less complexity than RHS.
859 auto IsLessComplex = [&](const SCEV *LHS, const SCEV *RHS) {
860 auto Complexity =
861 CompareSCEVComplexity(EqCacheSCEV, EqCacheValue, LI, LHS, RHS, DT);
862 return Complexity && *Complexity < 0;
863 };
864 if (Ops.size() == 2) {
865 // This is the common case, which also happens to be trivially simple.
866 // Special case it.
867 const SCEV *&LHS = Ops[0], *&RHS = Ops[1];
868 if (IsLessComplex(RHS, LHS))
869 std::swap(LHS, RHS);
870 return;
871 }
872
873 // Do the rough sort by complexity.
874 llvm::stable_sort(Ops, [&](const SCEV *LHS, const SCEV *RHS) {
875 return IsLessComplex(LHS, RHS);
876 });
877
878 // Now that we are sorted by complexity, group elements of the same
879 // complexity. Note that this is, at worst, N^2, but the vector is likely to
880 // be extremely short in practice. Note that we take this approach because we
881 // do not want to depend on the addresses of the objects we are grouping.
882 for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) {
883 const SCEV *S = Ops[i];
884 unsigned Complexity = S->getSCEVType();
885
886 // If there are any objects of the same complexity and same value as this
887 // one, group them.
888 for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
889 if (Ops[j] == S) { // Found a duplicate.
890 // Move it to immediately after i'th element.
891 std::swap(Ops[i+1], Ops[j]);
892 ++i; // no need to rescan it.
893 if (i == e-2) return; // Done!
894 }
895 }
896 }
897}
898
899/// Returns true if \p Ops contains a huge SCEV (the subtree of S contains at
900/// least HugeExprThreshold nodes).
901static bool hasHugeExpression(ArrayRef<const SCEV *> Ops) {
902 return any_of(Ops, [](const SCEV *S) {
903 return S->getExpressionSize() >= HugeExprThreshold;
904 });
905}
906
907//===----------------------------------------------------------------------===//
908// Simple SCEV method implementations
909//===----------------------------------------------------------------------===//
910
911/// Compute BC(It, K). The result has width W. Assume, K > 0.
912static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
913 ScalarEvolution &SE,
914 Type *ResultTy) {
915 // Handle the simplest case efficiently.
916 if (K == 1)
917 return SE.getTruncateOrZeroExtend(It, ResultTy);
918
919 // We are using the following formula for BC(It, K):
920 //
921 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
922 //
923 // Suppose, W is the bitwidth of the return value. We must be prepared for
924 // overflow. Hence, we must assure that the result of our computation is
925 // equal to the accurate one modulo 2^W. Unfortunately, division isn't
926 // safe in modular arithmetic.
927 //
928 // However, this code doesn't use exactly that formula; the formula it uses
929 // is something like the following, where T is the number of factors of 2 in
930 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
931 // exponentiation:
932 //
933 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
934 //
935 // This formula is trivially equivalent to the previous formula. However,
936 // this formula can be implemented much more efficiently. The trick is that
937 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
938 // arithmetic. To do exact division in modular arithmetic, all we have
939 // to do is multiply by the inverse. Therefore, this step can be done at
940 // width W.
941 //
942 // The next issue is how to safely do the division by 2^T. The way this
943 // is done is by doing the multiplication step at a width of at least W + T
944 // bits. This way, the bottom W+T bits of the product are accurate. Then,
945 // when we perform the division by 2^T (which is equivalent to a right shift
946 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get
947 // truncated out after the division by 2^T.
948 //
949 // In comparison to just directly using the first formula, this technique
950 // is much more efficient; using the first formula requires W * K bits,
951 // but this formula less than W + K bits. Also, the first formula requires
952 // a division step, whereas this formula only requires multiplies and shifts.
953 //
954 // It doesn't matter whether the subtraction step is done in the calculation
955 // width or the input iteration count's width; if the subtraction overflows,
956 // the result must be zero anyway. We prefer here to do it in the width of
957 // the induction variable because it helps a lot for certain cases; CodeGen
958 // isn't smart enough to ignore the overflow, which leads to much less
959 // efficient code if the width of the subtraction is wider than the native
960 // register width.
961 //
962 // (It's possible to not widen at all by pulling out factors of 2 before
963 // the multiplication; for example, K=2 can be calculated as
964 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
965 // extra arithmetic, so it's not an obvious win, and it gets
966 // much more complicated for K > 3.)
967
968 // Protection from insane SCEVs; this bound is conservative,
969 // but it probably doesn't matter.
970 if (K > 1000)
971 return SE.getCouldNotCompute();
972
973 unsigned W = SE.getTypeSizeInBits(ResultTy);
974
975 // Calculate K! / 2^T and T; we divide out the factors of two before
976 // multiplying for calculating K! / 2^T to avoid overflow.
977 // Other overflow doesn't matter because we only care about the bottom
978 // W bits of the result.
979 APInt OddFactorial(W, 1);
980 unsigned T = 1;
981 for (unsigned i = 3; i <= K; ++i) {
982 APInt Mult(W, i);
983 unsigned TwoFactors = Mult.countTrailingZeros();
984 T += TwoFactors;
985 Mult.lshrInPlace(TwoFactors);
986 OddFactorial *= Mult;
987 }
988
989 // We need at least W + T bits for the multiplication step
990 unsigned CalculationBits = W + T;
991
992 // Calculate 2^T, at width T+W.
993 APInt DivFactor = APInt::getOneBitSet(CalculationBits, T);
994
995 // Calculate the multiplicative inverse of K! / 2^T;
996 // this multiplication factor will perform the exact division by
997 // K! / 2^T.
998 APInt Mod = APInt::getSignedMinValue(W+1);
999 APInt MultiplyFactor = OddFactorial.zext(W+1);
1000 MultiplyFactor = MultiplyFactor.multiplicativeInverse(Mod);
1001 MultiplyFactor = MultiplyFactor.trunc(W);
1002
1003 // Calculate the product, at width T+W
1004 IntegerType *CalculationTy = IntegerType::get(SE.getContext(),
1005 CalculationBits);
1006 const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy);
1007 for (unsigned i = 1; i != K; ++i) {
1008 const SCEV *S = SE.getMinusSCEV(It, SE.getConstant(It->getType(), i));
1009 Dividend = SE.getMulExpr(Dividend,
1010 SE.getTruncateOrZeroExtend(S, CalculationTy));
1011 }
1012
1013 // Divide by 2^T
1014 const SCEV *DivResult = SE.getUDivExpr(Dividend, SE.getConstant(DivFactor));
1015
1016 // Truncate the result, and divide by K! / 2^T.
1017
1018 return SE.getMulExpr(SE.getConstant(MultiplyFactor),
1019 SE.getTruncateOrZeroExtend(DivResult, ResultTy));
1020}
1021
1022/// Return the value of this chain of recurrences at the specified iteration
1023/// number. We can evaluate this recurrence by multiplying each element in the
1024/// chain by the binomial coefficient corresponding to it. In other words, we
1025/// can evaluate {A,+,B,+,C,+,D} as:
1026///
1027/// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
1028///
1029/// where BC(It, k) stands for binomial coefficient.
1030const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
1031 ScalarEvolution &SE) const {
1032 return evaluateAtIteration(makeArrayRef(op_begin(), op_end()), It, SE);
1033}
1034
1035const SCEV *
1036SCEVAddRecExpr::evaluateAtIteration(ArrayRef<const SCEV *> Operands,
1037 const SCEV *It, ScalarEvolution &SE) {
1038 assert(Operands.size() > 0)(static_cast <bool> (Operands.size() > 0) ? void (0)
: __assert_fail ("Operands.size() > 0", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 1038, __extension__ __PRETTY_FUNCTION__))
;
1039 const SCEV *Result = Operands[0];
1040 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1041 // The computation is correct in the face of overflow provided that the
1042 // multiplication is performed _after_ the evaluation of the binomial
1043 // coefficient.
1044 const SCEV *Coeff = BinomialCoefficient(It, i, SE, Result->getType());
1045 if (isa<SCEVCouldNotCompute>(Coeff))
1046 return Coeff;
1047
1048 Result = SE.getAddExpr(Result, SE.getMulExpr(Operands[i], Coeff));
1049 }
1050 return Result;
1051}
1052
1053//===----------------------------------------------------------------------===//
1054// SCEV Expression folder implementations
1055//===----------------------------------------------------------------------===//
1056
1057const SCEV *ScalarEvolution::getLosslessPtrToIntExpr(const SCEV *Op,
1058 unsigned Depth) {
1059 assert(Depth <= 1 &&(static_cast <bool> (Depth <= 1 && "getLosslessPtrToIntExpr() should self-recurse at most once."
) ? void (0) : __assert_fail ("Depth <= 1 && \"getLosslessPtrToIntExpr() should self-recurse at most once.\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1060, __extension__
__PRETTY_FUNCTION__))
1060 "getLosslessPtrToIntExpr() should self-recurse at most once.")(static_cast <bool> (Depth <= 1 && "getLosslessPtrToIntExpr() should self-recurse at most once."
) ? void (0) : __assert_fail ("Depth <= 1 && \"getLosslessPtrToIntExpr() should self-recurse at most once.\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1060, __extension__
__PRETTY_FUNCTION__))
;
1061
1062 // We could be called with an integer-typed operands during SCEV rewrites.
1063 // Since the operand is an integer already, just perform zext/trunc/self cast.
1064 if (!Op->getType()->isPointerTy())
1065 return Op;
1066
1067 // What would be an ID for such a SCEV cast expression?
1068 FoldingSetNodeID ID;
1069 ID.AddInteger(scPtrToInt);
1070 ID.AddPointer(Op);
1071
1072 void *IP = nullptr;
1073
1074 // Is there already an expression for such a cast?
1075 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
1076 return S;
1077
1078 // It isn't legal for optimizations to construct new ptrtoint expressions
1079 // for non-integral pointers.
1080 if (getDataLayout().isNonIntegralPointerType(Op->getType()))
1081 return getCouldNotCompute();
1082
1083 Type *IntPtrTy = getDataLayout().getIntPtrType(Op->getType());
1084
1085 // We can only trivially model ptrtoint if SCEV's effective (integer) type
1086 // is sufficiently wide to represent all possible pointer values.
1087 // We could theoretically teach SCEV to truncate wider pointers, but
1088 // that isn't implemented for now.
1089 if (getDataLayout().getTypeSizeInBits(getEffectiveSCEVType(Op->getType())) !=
1090 getDataLayout().getTypeSizeInBits(IntPtrTy))
1091 return getCouldNotCompute();
1092
1093 // If not, is this expression something we can't reduce any further?
1094 if (auto *U = dyn_cast<SCEVUnknown>(Op)) {
1095 // Perform some basic constant folding. If the operand of the ptr2int cast
1096 // is a null pointer, don't create a ptr2int SCEV expression (that will be
1097 // left as-is), but produce a zero constant.
1098 // NOTE: We could handle a more general case, but lack motivational cases.
1099 if (isa<ConstantPointerNull>(U->getValue()))
1100 return getZero(IntPtrTy);
1101
1102 // Create an explicit cast node.
1103 // We can reuse the existing insert position since if we get here,
1104 // we won't have made any changes which would invalidate it.
1105 SCEV *S = new (SCEVAllocator)
1106 SCEVPtrToIntExpr(ID.Intern(SCEVAllocator), Op, IntPtrTy);
1107 UniqueSCEVs.InsertNode(S, IP);
1108 registerUser(S, Op);
1109 return S;
1110 }
1111
1112 assert(Depth == 0 && "getLosslessPtrToIntExpr() should not self-recurse for "(static_cast <bool> (Depth == 0 && "getLosslessPtrToIntExpr() should not self-recurse for "
"non-SCEVUnknown's.") ? void (0) : __assert_fail ("Depth == 0 && \"getLosslessPtrToIntExpr() should not self-recurse for \" \"non-SCEVUnknown's.\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1113, __extension__
__PRETTY_FUNCTION__))
1113 "non-SCEVUnknown's.")(static_cast <bool> (Depth == 0 && "getLosslessPtrToIntExpr() should not self-recurse for "
"non-SCEVUnknown's.") ? void (0) : __assert_fail ("Depth == 0 && \"getLosslessPtrToIntExpr() should not self-recurse for \" \"non-SCEVUnknown's.\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1113, __extension__
__PRETTY_FUNCTION__))
;
1114
1115 // Otherwise, we've got some expression that is more complex than just a
1116 // single SCEVUnknown. But we don't want to have a SCEVPtrToIntExpr of an
1117 // arbitrary expression, we want to have SCEVPtrToIntExpr of an SCEVUnknown
1118 // only, and the expressions must otherwise be integer-typed.
1119 // So sink the cast down to the SCEVUnknown's.
1120
1121 /// The SCEVPtrToIntSinkingRewriter takes a scalar evolution expression,
1122 /// which computes a pointer-typed value, and rewrites the whole expression
1123 /// tree so that *all* the computations are done on integers, and the only
1124 /// pointer-typed operands in the expression are SCEVUnknown.
1125 class SCEVPtrToIntSinkingRewriter
1126 : public SCEVRewriteVisitor<SCEVPtrToIntSinkingRewriter> {
1127 using Base = SCEVRewriteVisitor<SCEVPtrToIntSinkingRewriter>;
1128
1129 public:
1130 SCEVPtrToIntSinkingRewriter(ScalarEvolution &SE) : SCEVRewriteVisitor(SE) {}
1131
1132 static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE) {
1133 SCEVPtrToIntSinkingRewriter Rewriter(SE);
1134 return Rewriter.visit(Scev);
1135 }
1136
1137 const SCEV *visit(const SCEV *S) {
1138 Type *STy = S->getType();
1139 // If the expression is not pointer-typed, just keep it as-is.
1140 if (!STy->isPointerTy())
1141 return S;
1142 // Else, recursively sink the cast down into it.
1143 return Base::visit(S);
1144 }
1145
1146 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
1147 SmallVector<const SCEV *, 2> Operands;
1148 bool Changed = false;
1149 for (auto *Op : Expr->operands()) {
1150 Operands.push_back(visit(Op));
1151 Changed |= Op != Operands.back();
1152 }
1153 return !Changed ? Expr : SE.getAddExpr(Operands, Expr->getNoWrapFlags());
1154 }
1155
1156 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
1157 SmallVector<const SCEV *, 2> Operands;
1158 bool Changed = false;
1159 for (auto *Op : Expr->operands()) {
1160 Operands.push_back(visit(Op));
1161 Changed |= Op != Operands.back();
1162 }
1163 return !Changed ? Expr : SE.getMulExpr(Operands, Expr->getNoWrapFlags());
1164 }
1165
1166 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
1167 assert(Expr->getType()->isPointerTy() &&(static_cast <bool> (Expr->getType()->isPointerTy
() && "Should only reach pointer-typed SCEVUnknown's."
) ? void (0) : __assert_fail ("Expr->getType()->isPointerTy() && \"Should only reach pointer-typed SCEVUnknown's.\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1168, __extension__
__PRETTY_FUNCTION__))
1168 "Should only reach pointer-typed SCEVUnknown's.")(static_cast <bool> (Expr->getType()->isPointerTy
() && "Should only reach pointer-typed SCEVUnknown's."
) ? void (0) : __assert_fail ("Expr->getType()->isPointerTy() && \"Should only reach pointer-typed SCEVUnknown's.\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1168, __extension__
__PRETTY_FUNCTION__))
;
1169 return SE.getLosslessPtrToIntExpr(Expr, /*Depth=*/1);
1170 }
1171 };
1172
1173 // And actually perform the cast sinking.
1174 const SCEV *IntOp = SCEVPtrToIntSinkingRewriter::rewrite(Op, *this);
1175 assert(IntOp->getType()->isIntegerTy() &&(static_cast <bool> (IntOp->getType()->isIntegerTy
() && "We must have succeeded in sinking the cast, " "and ending up with an integer-typed expression!"
) ? void (0) : __assert_fail ("IntOp->getType()->isIntegerTy() && \"We must have succeeded in sinking the cast, \" \"and ending up with an integer-typed expression!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1177, __extension__
__PRETTY_FUNCTION__))
1176 "We must have succeeded in sinking the cast, "(static_cast <bool> (IntOp->getType()->isIntegerTy
() && "We must have succeeded in sinking the cast, " "and ending up with an integer-typed expression!"
) ? void (0) : __assert_fail ("IntOp->getType()->isIntegerTy() && \"We must have succeeded in sinking the cast, \" \"and ending up with an integer-typed expression!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1177, __extension__
__PRETTY_FUNCTION__))
1177 "and ending up with an integer-typed expression!")(static_cast <bool> (IntOp->getType()->isIntegerTy
() && "We must have succeeded in sinking the cast, " "and ending up with an integer-typed expression!"
) ? void (0) : __assert_fail ("IntOp->getType()->isIntegerTy() && \"We must have succeeded in sinking the cast, \" \"and ending up with an integer-typed expression!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1177, __extension__
__PRETTY_FUNCTION__))
;
1178 return IntOp;
1179}
1180
1181const SCEV *ScalarEvolution::getPtrToIntExpr(const SCEV *Op, Type *Ty) {
1182 assert(Ty->isIntegerTy() && "Target type must be an integer type!")(static_cast <bool> (Ty->isIntegerTy() && "Target type must be an integer type!"
) ? void (0) : __assert_fail ("Ty->isIntegerTy() && \"Target type must be an integer type!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1182, __extension__
__PRETTY_FUNCTION__))
;
1183
1184 const SCEV *IntOp = getLosslessPtrToIntExpr(Op);
1185 if (isa<SCEVCouldNotCompute>(IntOp))
1186 return IntOp;
1187
1188 return getTruncateOrZeroExtend(IntOp, Ty);
1189}
1190
1191const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, Type *Ty,
1192 unsigned Depth) {
1193 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&(static_cast <bool> (getTypeSizeInBits(Op->getType()
) > getTypeSizeInBits(Ty) && "This is not a truncating conversion!"
) ? void (0) : __assert_fail ("getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) && \"This is not a truncating conversion!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1194, __extension__
__PRETTY_FUNCTION__))
1194 "This is not a truncating conversion!")(static_cast <bool> (getTypeSizeInBits(Op->getType()
) > getTypeSizeInBits(Ty) && "This is not a truncating conversion!"
) ? void (0) : __assert_fail ("getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) && \"This is not a truncating conversion!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1194, __extension__
__PRETTY_FUNCTION__))
;
1195 assert(isSCEVable(Ty) &&(static_cast <bool> (isSCEVable(Ty) && "This is not a conversion to a SCEVable type!"
) ? void (0) : __assert_fail ("isSCEVable(Ty) && \"This is not a conversion to a SCEVable type!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1196, __extension__
__PRETTY_FUNCTION__))
1196 "This is not a conversion to a SCEVable type!")(static_cast <bool> (isSCEVable(Ty) && "This is not a conversion to a SCEVable type!"
) ? void (0) : __assert_fail ("isSCEVable(Ty) && \"This is not a conversion to a SCEVable type!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1196, __extension__
__PRETTY_FUNCTION__))
;
1197 assert(!Op->getType()->isPointerTy() && "Can't truncate pointer!")(static_cast <bool> (!Op->getType()->isPointerTy(
) && "Can't truncate pointer!") ? void (0) : __assert_fail
("!Op->getType()->isPointerTy() && \"Can't truncate pointer!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1197, __extension__
__PRETTY_FUNCTION__))
;
1198 Ty = getEffectiveSCEVType(Ty);
1199
1200 FoldingSetNodeID ID;
1201 ID.AddInteger(scTruncate);
1202 ID.AddPointer(Op);
1203 ID.AddPointer(Ty);
1204 void *IP = nullptr;
1205 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
1206
1207 // Fold if the operand is constant.
1208 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
1209 return getConstant(
1210 cast<ConstantInt>(ConstantExpr::getTrunc(SC->getValue(), Ty)));
1211
1212 // trunc(trunc(x)) --> trunc(x)
1213 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op))
1214 return getTruncateExpr(ST->getOperand(), Ty, Depth + 1);
1215
1216 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
1217 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op))
1218 return getTruncateOrSignExtend(SS->getOperand(), Ty, Depth + 1);
1219
1220 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
1221 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
1222 return getTruncateOrZeroExtend(SZ->getOperand(), Ty, Depth + 1);
1223
1224 if (Depth > MaxCastDepth) {
1225 SCEV *S =
1226 new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), Op, Ty);
1227 UniqueSCEVs.InsertNode(S, IP);
1228 registerUser(S, Op);
1229 return S;
1230 }
1231
1232 // trunc(x1 + ... + xN) --> trunc(x1) + ... + trunc(xN) and
1233 // trunc(x1 * ... * xN) --> trunc(x1) * ... * trunc(xN),
1234 // if after transforming we have at most one truncate, not counting truncates
1235 // that replace other casts.
1236 if (isa<SCEVAddExpr>(Op) || isa<SCEVMulExpr>(Op)) {
1237 auto *CommOp = cast<SCEVCommutativeExpr>(Op);
1238 SmallVector<const SCEV *, 4> Operands;
1239 unsigned numTruncs = 0;
1240 for (unsigned i = 0, e = CommOp->getNumOperands(); i != e && numTruncs < 2;
1241 ++i) {
1242 const SCEV *S = getTruncateExpr(CommOp->getOperand(i), Ty, Depth + 1);
1243 if (!isa<SCEVIntegralCastExpr>(CommOp->getOperand(i)) &&
1244 isa<SCEVTruncateExpr>(S))
1245 numTruncs++;
1246 Operands.push_back(S);
1247 }
1248 if (numTruncs < 2) {
1249 if (isa<SCEVAddExpr>(Op))
1250 return getAddExpr(Operands);
1251 else if (isa<SCEVMulExpr>(Op))
1252 return getMulExpr(Operands);
1253 else
1254 llvm_unreachable("Unexpected SCEV type for Op.")::llvm::llvm_unreachable_internal("Unexpected SCEV type for Op."
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1254)
;
1255 }
1256 // Although we checked in the beginning that ID is not in the cache, it is
1257 // possible that during recursion and different modification ID was inserted
1258 // into the cache. So if we find it, just return it.
1259 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
1260 return S;
1261 }
1262
1263 // If the input value is a chrec scev, truncate the chrec's operands.
1264 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) {
1265 SmallVector<const SCEV *, 4> Operands;
1266 for (const SCEV *Op : AddRec->operands())
1267 Operands.push_back(getTruncateExpr(Op, Ty, Depth + 1));
1268 return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagAnyWrap);
1269 }
1270
1271 // Return zero if truncating to known zeros.
1272 uint32_t MinTrailingZeros = GetMinTrailingZeros(Op);
1273 if (MinTrailingZeros >= getTypeSizeInBits(Ty))
1274 return getZero(Ty);
1275
1276 // The cast wasn't folded; create an explicit cast node. We can reuse
1277 // the existing insert position since if we get here, we won't have
1278 // made any changes which would invalidate it.
1279 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator),
1280 Op, Ty);
1281 UniqueSCEVs.InsertNode(S, IP);
1282 registerUser(S, Op);
1283 return S;
1284}
1285
1286// Get the limit of a recurrence such that incrementing by Step cannot cause
1287// signed overflow as long as the value of the recurrence within the
1288// loop does not exceed this limit before incrementing.
1289static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step,
1290 ICmpInst::Predicate *Pred,
1291 ScalarEvolution *SE) {
1292 unsigned BitWidth = SE->getTypeSizeInBits(Step->getType());
1293 if (SE->isKnownPositive(Step)) {
1294 *Pred = ICmpInst::ICMP_SLT;
1295 return SE->getConstant(APInt::getSignedMinValue(BitWidth) -
1296 SE->getSignedRangeMax(Step));
1297 }
1298 if (SE->isKnownNegative(Step)) {
1299 *Pred = ICmpInst::ICMP_SGT;
1300 return SE->getConstant(APInt::getSignedMaxValue(BitWidth) -
1301 SE->getSignedRangeMin(Step));
1302 }
1303 return nullptr;
1304}
1305
1306// Get the limit of a recurrence such that incrementing by Step cannot cause
1307// unsigned overflow as long as the value of the recurrence within the loop does
1308// not exceed this limit before incrementing.
1309static const SCEV *getUnsignedOverflowLimitForStep(const SCEV *Step,
1310 ICmpInst::Predicate *Pred,
1311 ScalarEvolution *SE) {
1312 unsigned BitWidth = SE->getTypeSizeInBits(Step->getType());
1313 *Pred = ICmpInst::ICMP_ULT;
1314
1315 return SE->getConstant(APInt::getMinValue(BitWidth) -
1316 SE->getUnsignedRangeMax(Step));
1317}
1318
1319namespace {
1320
1321struct ExtendOpTraitsBase {
1322 typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(const SCEV *, Type *,
1323 unsigned);
1324};
1325
1326// Used to make code generic over signed and unsigned overflow.
1327template <typename ExtendOp> struct ExtendOpTraits {
1328 // Members present:
1329 //
1330 // static const SCEV::NoWrapFlags WrapType;
1331 //
1332 // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr;
1333 //
1334 // static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1335 // ICmpInst::Predicate *Pred,
1336 // ScalarEvolution *SE);
1337};
1338
1339template <>
1340struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase {
1341 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNSW;
1342
1343 static const GetExtendExprTy GetExtendExpr;
1344
1345 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1346 ICmpInst::Predicate *Pred,
1347 ScalarEvolution *SE) {
1348 return getSignedOverflowLimitForStep(Step, Pred, SE);
1349 }
1350};
1351
1352const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1353 SCEVSignExtendExpr>::GetExtendExpr = &ScalarEvolution::getSignExtendExpr;
1354
1355template <>
1356struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase {
1357 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNUW;
1358
1359 static const GetExtendExprTy GetExtendExpr;
1360
1361 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1362 ICmpInst::Predicate *Pred,
1363 ScalarEvolution *SE) {
1364 return getUnsignedOverflowLimitForStep(Step, Pred, SE);
1365 }
1366};
1367
1368const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1369 SCEVZeroExtendExpr>::GetExtendExpr = &ScalarEvolution::getZeroExtendExpr;
1370
1371} // end anonymous namespace
1372
1373// The recurrence AR has been shown to have no signed/unsigned wrap or something
1374// close to it. Typically, if we can prove NSW/NUW for AR, then we can just as
1375// easily prove NSW/NUW for its preincrement or postincrement sibling. This
1376// allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step +
1377// Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the
1378// expression "Step + sext/zext(PreIncAR)" is congruent with
1379// "sext/zext(PostIncAR)"
1380template <typename ExtendOpTy>
1381static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR, Type *Ty,
1382 ScalarEvolution *SE, unsigned Depth) {
1383 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1384 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1385
1386 const Loop *L = AR->getLoop();
1387 const SCEV *Start = AR->getStart();
1388 const SCEV *Step = AR->getStepRecurrence(*SE);
1389
1390 // Check for a simple looking step prior to loop entry.
1391 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Start);
1392 if (!SA)
1393 return nullptr;
1394
1395 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
1396 // subtraction is expensive. For this purpose, perform a quick and dirty
1397 // difference, by checking for Step in the operand list.
1398 SmallVector<const SCEV *, 4> DiffOps;
1399 for (const SCEV *Op : SA->operands())
1400 if (Op != Step)
1401 DiffOps.push_back(Op);
1402
1403 if (DiffOps.size() == SA->getNumOperands())
1404 return nullptr;
1405
1406 // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` +
1407 // `Step`:
1408
1409 // 1. NSW/NUW flags on the step increment.
1410 auto PreStartFlags =
1411 ScalarEvolution::maskFlags(SA->getNoWrapFlags(), SCEV::FlagNUW);
1412 const SCEV *PreStart = SE->getAddExpr(DiffOps, PreStartFlags);
1413 const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>(
1414 SE->getAddRecExpr(PreStart, Step, L, SCEV::FlagAnyWrap));
1415
1416 // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies
1417 // "S+X does not sign/unsign-overflow".
1418 //
1419
1420 const SCEV *BECount = SE->getBackedgeTakenCount(L);
1421 if (PreAR && PreAR->getNoWrapFlags(WrapType) &&
1422 !isa<SCEVCouldNotCompute>(BECount) && SE->isKnownPositive(BECount))
1423 return PreStart;
1424
1425 // 2. Direct overflow check on the step operation's expression.
1426 unsigned BitWidth = SE->getTypeSizeInBits(AR->getType());
1427 Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2);
1428 const SCEV *OperandExtendedStart =
1429 SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy, Depth),
1430 (SE->*GetExtendExpr)(Step, WideTy, Depth));
1431 if ((SE->*GetExtendExpr)(Start, WideTy, Depth) == OperandExtendedStart) {
1432 if (PreAR && AR->getNoWrapFlags(WrapType)) {
1433 // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW
1434 // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then
1435 // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`. Cache this fact.
1436 SE->setNoWrapFlags(const_cast<SCEVAddRecExpr *>(PreAR), WrapType);
1437 }
1438 return PreStart;
1439 }
1440
1441 // 3. Loop precondition.
1442 ICmpInst::Predicate Pred;
1443 const SCEV *OverflowLimit =
1444 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
1445
1446 if (OverflowLimit &&
1447 SE->isLoopEntryGuardedByCond(L, Pred, PreStart, OverflowLimit))
1448 return PreStart;
1449
1450 return nullptr;
1451}
1452
1453// Get the normalized zero or sign extended expression for this AddRec's Start.
1454template <typename ExtendOpTy>
1455static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty,
1456 ScalarEvolution *SE,
1457 unsigned Depth) {
1458 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1459
1460 const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, Ty, SE, Depth);
1461 if (!PreStart)
1462 return (SE->*GetExtendExpr)(AR->getStart(), Ty, Depth);
1463
1464 return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(*SE), Ty,
1465 Depth),
1466 (SE->*GetExtendExpr)(PreStart, Ty, Depth));
1467}
1468
1469// Try to prove away overflow by looking at "nearby" add recurrences. A
1470// motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it
1471// does not itself wrap then we can conclude that `{1,+,4}` is `nuw`.
1472//
1473// Formally:
1474//
1475// {S,+,X} == {S-T,+,X} + T
1476// => Ext({S,+,X}) == Ext({S-T,+,X} + T)
1477//
1478// If ({S-T,+,X} + T) does not overflow ... (1)
1479//
1480// RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T)
1481//
1482// If {S-T,+,X} does not overflow ... (2)
1483//
1484// RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T)
1485// == {Ext(S-T)+Ext(T),+,Ext(X)}
1486//
1487// If (S-T)+T does not overflow ... (3)
1488//
1489// RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)}
1490// == {Ext(S),+,Ext(X)} == LHS
1491//
1492// Thus, if (1), (2) and (3) are true for some T, then
1493// Ext({S,+,X}) == {Ext(S),+,Ext(X)}
1494//
1495// (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T)
1496// does not overflow" restricted to the 0th iteration. Therefore we only need
1497// to check for (1) and (2).
1498//
1499// In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T
1500// is `Delta` (defined below).
1501template <typename ExtendOpTy>
1502bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
1503 const SCEV *Step,
1504 const Loop *L) {
1505 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1506
1507 // We restrict `Start` to a constant to prevent SCEV from spending too much
1508 // time here. It is correct (but more expensive) to continue with a
1509 // non-constant `Start` and do a general SCEV subtraction to compute
1510 // `PreStart` below.
1511 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start);
1512 if (!StartC)
1513 return false;
1514
1515 APInt StartAI = StartC->getAPInt();
1516
1517 for (unsigned Delta : {-2, -1, 1, 2}) {
1518 const SCEV *PreStart = getConstant(StartAI - Delta);
1519
1520 FoldingSetNodeID ID;
1521 ID.AddInteger(scAddRecExpr);
1522 ID.AddPointer(PreStart);
1523 ID.AddPointer(Step);
1524 ID.AddPointer(L);
1525 void *IP = nullptr;
1526 const auto *PreAR =
1527 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
1528
1529 // Give up if we don't already have the add recurrence we need because
1530 // actually constructing an add recurrence is relatively expensive.
1531 if (PreAR && PreAR->getNoWrapFlags(WrapType)) { // proves (2)
1532 const SCEV *DeltaS = getConstant(StartC->getType(), Delta);
1533 ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
1534 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
1535 DeltaS, &Pred, this);
1536 if (Limit && isKnownPredicate(Pred, PreAR, Limit)) // proves (1)
1537 return true;
1538 }
1539 }
1540
1541 return false;
1542}
1543
1544// Finds an integer D for an expression (C + x + y + ...) such that the top
1545// level addition in (D + (C - D + x + y + ...)) would not wrap (signed or
1546// unsigned) and the number of trailing zeros of (C - D + x + y + ...) is
1547// maximized, where C is the \p ConstantTerm, x, y, ... are arbitrary SCEVs, and
1548// the (C + x + y + ...) expression is \p WholeAddExpr.
1549static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1550 const SCEVConstant *ConstantTerm,
1551 const SCEVAddExpr *WholeAddExpr) {
1552 const APInt &C = ConstantTerm->getAPInt();
1553 const unsigned BitWidth = C.getBitWidth();
1554 // Find number of trailing zeros of (x + y + ...) w/o the C first:
1555 uint32_t TZ = BitWidth;
1556 for (unsigned I = 1, E = WholeAddExpr->getNumOperands(); I < E && TZ; ++I)
1557 TZ = std::min(TZ, SE.GetMinTrailingZeros(WholeAddExpr->getOperand(I)));
1558 if (TZ) {
1559 // Set D to be as many least significant bits of C as possible while still
1560 // guaranteeing that adding D to (C - D + x + y + ...) won't cause a wrap:
1561 return TZ < BitWidth ? C.trunc(TZ).zext(BitWidth) : C;
1562 }
1563 return APInt(BitWidth, 0);
1564}
1565
1566// Finds an integer D for an affine AddRec expression {C,+,x} such that the top
1567// level addition in (D + {C-D,+,x}) would not wrap (signed or unsigned) and the
1568// number of trailing zeros of (C - D + x * n) is maximized, where C is the \p
1569// ConstantStart, x is an arbitrary \p Step, and n is the loop trip count.
1570static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1571 const APInt &ConstantStart,
1572 const SCEV *Step) {
1573 const unsigned BitWidth = ConstantStart.getBitWidth();
1574 const uint32_t TZ = SE.GetMinTrailingZeros(Step);
1575 if (TZ)
1576 return TZ < BitWidth ? ConstantStart.trunc(TZ).zext(BitWidth)
1577 : ConstantStart;
1578 return APInt(BitWidth, 0);
1579}
1580
1581const SCEV *
1582ScalarEvolution::getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) {
1583 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&(static_cast <bool> (getTypeSizeInBits(Op->getType()
) < getTypeSizeInBits(Ty) && "This is not an extending conversion!"
) ? void (0) : __assert_fail ("getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && \"This is not an extending conversion!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1584, __extension__
__PRETTY_FUNCTION__))
1584 "This is not an extending conversion!")(static_cast <bool> (getTypeSizeInBits(Op->getType()
) < getTypeSizeInBits(Ty) && "This is not an extending conversion!"
) ? void (0) : __assert_fail ("getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && \"This is not an extending conversion!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1584, __extension__
__PRETTY_FUNCTION__))
;
1585 assert(isSCEVable(Ty) &&(static_cast <bool> (isSCEVable(Ty) && "This is not a conversion to a SCEVable type!"
) ? void (0) : __assert_fail ("isSCEVable(Ty) && \"This is not a conversion to a SCEVable type!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1586, __extension__
__PRETTY_FUNCTION__))
1586 "This is not a conversion to a SCEVable type!")(static_cast <bool> (isSCEVable(Ty) && "This is not a conversion to a SCEVable type!"
) ? void (0) : __assert_fail ("isSCEVable(Ty) && \"This is not a conversion to a SCEVable type!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1586, __extension__
__PRETTY_FUNCTION__))
;
1587 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!")(static_cast <bool> (!Op->getType()->isPointerTy(
) && "Can't extend pointer!") ? void (0) : __assert_fail
("!Op->getType()->isPointerTy() && \"Can't extend pointer!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1587, __extension__
__PRETTY_FUNCTION__))
;
1588 Ty = getEffectiveSCEVType(Ty);
1589
1590 // Fold if the operand is constant.
1591 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
1592 return getConstant(
1593 cast<ConstantInt>(ConstantExpr::getZExt(SC->getValue(), Ty)));
1594
1595 // zext(zext(x)) --> zext(x)
1596 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
1597 return getZeroExtendExpr(SZ->getOperand(), Ty, Depth + 1);
1598
1599 // Before doing any expensive analysis, check to see if we've already
1600 // computed a SCEV for this Op and Ty.
1601 FoldingSetNodeID ID;
1602 ID.AddInteger(scZeroExtend);
1603 ID.AddPointer(Op);
1604 ID.AddPointer(Ty);
1605 void *IP = nullptr;
1606 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
1607 if (Depth > MaxCastDepth) {
1608 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
1609 Op, Ty);
1610 UniqueSCEVs.InsertNode(S, IP);
1611 registerUser(S, Op);
1612 return S;
1613 }
1614
1615 // zext(trunc(x)) --> zext(x) or x or trunc(x)
1616 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) {
1617 // It's possible the bits taken off by the truncate were all zero bits. If
1618 // so, we should be able to simplify this further.
1619 const SCEV *X = ST->getOperand();
1620 ConstantRange CR = getUnsignedRange(X);
1621 unsigned TruncBits = getTypeSizeInBits(ST->getType());
1622 unsigned NewBits = getTypeSizeInBits(Ty);
1623 if (CR.truncate(TruncBits).zeroExtend(NewBits).contains(
1624 CR.zextOrTrunc(NewBits)))
1625 return getTruncateOrZeroExtend(X, Ty, Depth);
1626 }
1627
1628 // If the input value is a chrec scev, and we can prove that the value
1629 // did not overflow the old, smaller, value, we can zero extend all of the
1630 // operands (often constants). This allows analysis of something like
1631 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
1632 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op))
1633 if (AR->isAffine()) {
1634 const SCEV *Start = AR->getStart();
1635 const SCEV *Step = AR->getStepRecurrence(*this);
1636 unsigned BitWidth = getTypeSizeInBits(AR->getType());
1637 const Loop *L = AR->getLoop();
1638
1639 if (!AR->hasNoUnsignedWrap()) {
1640 auto NewFlags = proveNoWrapViaConstantRanges(AR);
1641 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), NewFlags);
1642 }
1643
1644 // If we have special knowledge that this addrec won't overflow,
1645 // we don't need to do any further analysis.
1646 if (AR->hasNoUnsignedWrap())
1647 return getAddRecExpr(
1648 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1),
1649 getZeroExtendExpr(Step, Ty, Depth + 1), L, AR->getNoWrapFlags());
1650
1651 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1652 // Note that this serves two purposes: It filters out loops that are
1653 // simply not analyzable, and it covers the case where this code is
1654 // being called from within backedge-taken count analysis, such that
1655 // attempting to ask for the backedge-taken count would likely result
1656 // in infinite recursion. In the later case, the analysis code will
1657 // cope with a conservative value, and it will take care to purge
1658 // that value once it has finished.
1659 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
1660 if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
1661 // Manually compute the final value for AR, checking for overflow.
1662
1663 // Check whether the backedge-taken count can be losslessly casted to
1664 // the addrec's type. The count is always unsigned.
1665 const SCEV *CastedMaxBECount =
1666 getTruncateOrZeroExtend(MaxBECount, Start->getType(), Depth);
1667 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
1668 CastedMaxBECount, MaxBECount->getType(), Depth);
1669 if (MaxBECount == RecastedMaxBECount) {
1670 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
1671 // Check whether Start+Step*MaxBECount has no unsigned overflow.
1672 const SCEV *ZMul = getMulExpr(CastedMaxBECount, Step,
1673 SCEV::FlagAnyWrap, Depth + 1);
1674 const SCEV *ZAdd = getZeroExtendExpr(getAddExpr(Start, ZMul,
1675 SCEV::FlagAnyWrap,
1676 Depth + 1),
1677 WideTy, Depth + 1);
1678 const SCEV *WideStart = getZeroExtendExpr(Start, WideTy, Depth + 1);
1679 const SCEV *WideMaxBECount =
1680 getZeroExtendExpr(CastedMaxBECount, WideTy, Depth + 1);
1681 const SCEV *OperandExtendedAdd =
1682 getAddExpr(WideStart,
1683 getMulExpr(WideMaxBECount,
1684 getZeroExtendExpr(Step, WideTy, Depth + 1),
1685 SCEV::FlagAnyWrap, Depth + 1),
1686 SCEV::FlagAnyWrap, Depth + 1);
1687 if (ZAdd == OperandExtendedAdd) {
1688 // Cache knowledge of AR NUW, which is propagated to this AddRec.
1689 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNUW);
1690 // Return the expression with the addrec on the outside.
1691 return getAddRecExpr(
1692 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this,
1693 Depth + 1),
1694 getZeroExtendExpr(Step, Ty, Depth + 1), L,
1695 AR->getNoWrapFlags());
1696 }
1697 // Similar to above, only this time treat the step value as signed.
1698 // This covers loops that count down.
1699 OperandExtendedAdd =
1700 getAddExpr(WideStart,
1701 getMulExpr(WideMaxBECount,
1702 getSignExtendExpr(Step, WideTy, Depth + 1),
1703 SCEV::FlagAnyWrap, Depth + 1),
1704 SCEV::FlagAnyWrap, Depth + 1);
1705 if (ZAdd == OperandExtendedAdd) {
1706 // Cache knowledge of AR NW, which is propagated to this AddRec.
1707 // Negative step causes unsigned wrap, but it still can't self-wrap.
1708 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW);
1709 // Return the expression with the addrec on the outside.
1710 return getAddRecExpr(
1711 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this,
1712 Depth + 1),
1713 getSignExtendExpr(Step, Ty, Depth + 1), L,
1714 AR->getNoWrapFlags());
1715 }
1716 }
1717 }
1718
1719 // Normally, in the cases we can prove no-overflow via a
1720 // backedge guarding condition, we can also compute a backedge
1721 // taken count for the loop. The exceptions are assumptions and
1722 // guards present in the loop -- SCEV is not great at exploiting
1723 // these to compute max backedge taken counts, but can still use
1724 // these to prove lack of overflow. Use this fact to avoid
1725 // doing extra work that may not pay off.
1726 if (!isa<SCEVCouldNotCompute>(MaxBECount) || HasGuards ||
1727 !AC.assumptions().empty()) {
1728
1729 auto NewFlags = proveNoUnsignedWrapViaInduction(AR);
1730 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), NewFlags);
1731 if (AR->hasNoUnsignedWrap()) {
1732 // Same as nuw case above - duplicated here to avoid a compile time
1733 // issue. It's not clear that the order of checks does matter, but
1734 // it's one of two issue possible causes for a change which was
1735 // reverted. Be conservative for the moment.
1736 return getAddRecExpr(
1737 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this,
1738 Depth + 1),
1739 getZeroExtendExpr(Step, Ty, Depth + 1), L,
1740 AR->getNoWrapFlags());
1741 }
1742
1743 // For a negative step, we can extend the operands iff doing so only
1744 // traverses values in the range zext([0,UINT_MAX]).
1745 if (isKnownNegative(Step)) {
1746 const SCEV *N = getConstant(APInt::getMaxValue(BitWidth) -
1747 getSignedRangeMin(Step));
1748 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, AR, N) ||
1749 isKnownOnEveryIteration(ICmpInst::ICMP_UGT, AR, N)) {
1750 // Cache knowledge of AR NW, which is propagated to this
1751 // AddRec. Negative step causes unsigned wrap, but it
1752 // still can't self-wrap.
1753 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW);
1754 // Return the expression with the addrec on the outside.
1755 return getAddRecExpr(
1756 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this,
1757 Depth + 1),
1758 getSignExtendExpr(Step, Ty, Depth + 1), L,
1759 AR->getNoWrapFlags());
1760 }
1761 }
1762 }
1763
1764 // zext({C,+,Step}) --> (zext(D) + zext({C-D,+,Step}))<nuw><nsw>
1765 // if D + (C - D + Step * n) could be proven to not unsigned wrap
1766 // where D maximizes the number of trailing zeros of (C - D + Step * n)
1767 if (const auto *SC = dyn_cast<SCEVConstant>(Start)) {
1768 const APInt &C = SC->getAPInt();
1769 const APInt &D = extractConstantWithoutWrapping(*this, C, Step);
1770 if (D != 0) {
1771 const SCEV *SZExtD = getZeroExtendExpr(getConstant(D), Ty, Depth);
1772 const SCEV *SResidual =
1773 getAddRecExpr(getConstant(C - D), Step, L, AR->getNoWrapFlags());
1774 const SCEV *SZExtR = getZeroExtendExpr(SResidual, Ty, Depth + 1);
1775 return getAddExpr(SZExtD, SZExtR,
1776 (SCEV::NoWrapFlags)(SCEV::FlagNSW | SCEV::FlagNUW),
1777 Depth + 1);
1778 }
1779 }
1780
1781 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
1782 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNUW);
1783 return getAddRecExpr(
1784 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1),
1785 getZeroExtendExpr(Step, Ty, Depth + 1), L, AR->getNoWrapFlags());
1786 }
1787 }
1788
1789 // zext(A % B) --> zext(A) % zext(B)
1790 {
1791 const SCEV *LHS;
1792 const SCEV *RHS;
1793 if (matchURem(Op, LHS, RHS))
1794 return getURemExpr(getZeroExtendExpr(LHS, Ty, Depth + 1),
1795 getZeroExtendExpr(RHS, Ty, Depth + 1));
1796 }
1797
1798 // zext(A / B) --> zext(A) / zext(B).
1799 if (auto *Div = dyn_cast<SCEVUDivExpr>(Op))
1800 return getUDivExpr(getZeroExtendExpr(Div->getLHS(), Ty, Depth + 1),
1801 getZeroExtendExpr(Div->getRHS(), Ty, Depth + 1));
1802
1803 if (auto *SA = dyn_cast<SCEVAddExpr>(Op)) {
1804 // zext((A + B + ...)<nuw>) --> (zext(A) + zext(B) + ...)<nuw>
1805 if (SA->hasNoUnsignedWrap()) {
1806 // If the addition does not unsign overflow then we can, by definition,
1807 // commute the zero extension with the addition operation.
1808 SmallVector<const SCEV *, 4> Ops;
1809 for (const auto *Op : SA->operands())
1810 Ops.push_back(getZeroExtendExpr(Op, Ty, Depth + 1));
1811 return getAddExpr(Ops, SCEV::FlagNUW, Depth + 1);
1812 }
1813
1814 // zext(C + x + y + ...) --> (zext(D) + zext((C - D) + x + y + ...))
1815 // if D + (C - D + x + y + ...) could be proven to not unsigned wrap
1816 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1817 //
1818 // Often address arithmetics contain expressions like
1819 // (zext (add (shl X, C1), C2)), for instance, (zext (5 + (4 * X))).
1820 // This transformation is useful while proving that such expressions are
1821 // equal or differ by a small constant amount, see LoadStoreVectorizer pass.
1822 if (const auto *SC = dyn_cast<SCEVConstant>(SA->getOperand(0))) {
1823 const APInt &D = extractConstantWithoutWrapping(*this, SC, SA);
1824 if (D != 0) {
1825 const SCEV *SZExtD = getZeroExtendExpr(getConstant(D), Ty, Depth);
1826 const SCEV *SResidual =
1827 getAddExpr(getConstant(-D), SA, SCEV::FlagAnyWrap, Depth);
1828 const SCEV *SZExtR = getZeroExtendExpr(SResidual, Ty, Depth + 1);
1829 return getAddExpr(SZExtD, SZExtR,
1830 (SCEV::NoWrapFlags)(SCEV::FlagNSW | SCEV::FlagNUW),
1831 Depth + 1);
1832 }
1833 }
1834 }
1835
1836 if (auto *SM = dyn_cast<SCEVMulExpr>(Op)) {
1837 // zext((A * B * ...)<nuw>) --> (zext(A) * zext(B) * ...)<nuw>
1838 if (SM->hasNoUnsignedWrap()) {
1839 // If the multiply does not unsign overflow then we can, by definition,
1840 // commute the zero extension with the multiply operation.
1841 SmallVector<const SCEV *, 4> Ops;
1842 for (const auto *Op : SM->operands())
1843 Ops.push_back(getZeroExtendExpr(Op, Ty, Depth + 1));
1844 return getMulExpr(Ops, SCEV::FlagNUW, Depth + 1);
1845 }
1846
1847 // zext(2^K * (trunc X to iN)) to iM ->
1848 // 2^K * (zext(trunc X to i{N-K}) to iM)<nuw>
1849 //
1850 // Proof:
1851 //
1852 // zext(2^K * (trunc X to iN)) to iM
1853 // = zext((trunc X to iN) << K) to iM
1854 // = zext((trunc X to i{N-K}) << K)<nuw> to iM
1855 // (because shl removes the top K bits)
1856 // = zext((2^K * (trunc X to i{N-K}))<nuw>) to iM
1857 // = (2^K * (zext(trunc X to i{N-K}) to iM))<nuw>.
1858 //
1859 if (SM->getNumOperands() == 2)
1860 if (auto *MulLHS = dyn_cast<SCEVConstant>(SM->getOperand(0)))
1861 if (MulLHS->getAPInt().isPowerOf2())
1862 if (auto *TruncRHS = dyn_cast<SCEVTruncateExpr>(SM->getOperand(1))) {
1863 int NewTruncBits = getTypeSizeInBits(TruncRHS->getType()) -
1864 MulLHS->getAPInt().logBase2();
1865 Type *NewTruncTy = IntegerType::get(getContext(), NewTruncBits);
1866 return getMulExpr(
1867 getZeroExtendExpr(MulLHS, Ty),
1868 getZeroExtendExpr(
1869 getTruncateExpr(TruncRHS->getOperand(), NewTruncTy), Ty),
1870 SCEV::FlagNUW, Depth + 1);
1871 }
1872 }
1873
1874 // The cast wasn't folded; create an explicit cast node.
1875 // Recompute the insert position, as it may have been invalidated.
1876 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
1877 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
1878 Op, Ty);
1879 UniqueSCEVs.InsertNode(S, IP);
1880 registerUser(S, Op);
1881 return S;
1882}
1883
1884const SCEV *
1885ScalarEvolution::getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) {
1886 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&(static_cast <bool> (getTypeSizeInBits(Op->getType()
) < getTypeSizeInBits(Ty) && "This is not an extending conversion!"
) ? void (0) : __assert_fail ("getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && \"This is not an extending conversion!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1887, __extension__
__PRETTY_FUNCTION__))
1887 "This is not an extending conversion!")(static_cast <bool> (getTypeSizeInBits(Op->getType()
) < getTypeSizeInBits(Ty) && "This is not an extending conversion!"
) ? void (0) : __assert_fail ("getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && \"This is not an extending conversion!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1887, __extension__
__PRETTY_FUNCTION__))
;
1888 assert(isSCEVable(Ty) &&(static_cast <bool> (isSCEVable(Ty) && "This is not a conversion to a SCEVable type!"
) ? void (0) : __assert_fail ("isSCEVable(Ty) && \"This is not a conversion to a SCEVable type!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1889, __extension__
__PRETTY_FUNCTION__))
1889 "This is not a conversion to a SCEVable type!")(static_cast <bool> (isSCEVable(Ty) && "This is not a conversion to a SCEVable type!"
) ? void (0) : __assert_fail ("isSCEVable(Ty) && \"This is not a conversion to a SCEVable type!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1889, __extension__
__PRETTY_FUNCTION__))
;
1890 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!")(static_cast <bool> (!Op->getType()->isPointerTy(
) && "Can't extend pointer!") ? void (0) : __assert_fail
("!Op->getType()->isPointerTy() && \"Can't extend pointer!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 1890, __extension__
__PRETTY_FUNCTION__))
;
1891 Ty = getEffectiveSCEVType(Ty);
1892
1893 // Fold if the operand is constant.
1894 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
1895 return getConstant(
1896 cast<ConstantInt>(ConstantExpr::getSExt(SC->getValue(), Ty)));
1897
1898 // sext(sext(x)) --> sext(x)
1899 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op))
1900 return getSignExtendExpr(SS->getOperand(), Ty, Depth + 1);
1901
1902 // sext(zext(x)) --> zext(x)
1903 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
1904 return getZeroExtendExpr(SZ->getOperand(), Ty, Depth + 1);
1905
1906 // Before doing any expensive analysis, check to see if we've already
1907 // computed a SCEV for this Op and Ty.
1908 FoldingSetNodeID ID;
1909 ID.AddInteger(scSignExtend);
1910 ID.AddPointer(Op);
1911 ID.AddPointer(Ty);
1912 void *IP = nullptr;
1913 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
1914 // Limit recursion depth.
1915 if (Depth > MaxCastDepth) {
1916 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
1917 Op, Ty);
1918 UniqueSCEVs.InsertNode(S, IP);
1919 registerUser(S, Op);
1920 return S;
1921 }
1922
1923 // sext(trunc(x)) --> sext(x) or x or trunc(x)
1924 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) {
1925 // It's possible the bits taken off by the truncate were all sign bits. If
1926 // so, we should be able to simplify this further.
1927 const SCEV *X = ST->getOperand();
1928 ConstantRange CR = getSignedRange(X);
1929 unsigned TruncBits = getTypeSizeInBits(ST->getType());
1930 unsigned NewBits = getTypeSizeInBits(Ty);
1931 if (CR.truncate(TruncBits).signExtend(NewBits).contains(
1932 CR.sextOrTrunc(NewBits)))
1933 return getTruncateOrSignExtend(X, Ty, Depth);
1934 }
1935
1936 if (auto *SA = dyn_cast<SCEVAddExpr>(Op)) {
1937 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
1938 if (SA->hasNoSignedWrap()) {
1939 // If the addition does not sign overflow then we can, by definition,
1940 // commute the sign extension with the addition operation.
1941 SmallVector<const SCEV *, 4> Ops;
1942 for (const auto *Op : SA->operands())
1943 Ops.push_back(getSignExtendExpr(Op, Ty, Depth + 1));
1944 return getAddExpr(Ops, SCEV::FlagNSW, Depth + 1);
1945 }
1946
1947 // sext(C + x + y + ...) --> (sext(D) + sext((C - D) + x + y + ...))
1948 // if D + (C - D + x + y + ...) could be proven to not signed wrap
1949 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1950 //
1951 // For instance, this will bring two seemingly different expressions:
1952 // 1 + sext(5 + 20 * %x + 24 * %y) and
1953 // sext(6 + 20 * %x + 24 * %y)
1954 // to the same form:
1955 // 2 + sext(4 + 20 * %x + 24 * %y)
1956 if (const auto *SC = dyn_cast<SCEVConstant>(SA->getOperand(0))) {
1957 const APInt &D = extractConstantWithoutWrapping(*this, SC, SA);
1958 if (D != 0) {
1959 const SCEV *SSExtD = getSignExtendExpr(getConstant(D), Ty, Depth);
1960 const SCEV *SResidual =
1961 getAddExpr(getConstant(-D), SA, SCEV::FlagAnyWrap, Depth);
1962 const SCEV *SSExtR = getSignExtendExpr(SResidual, Ty, Depth + 1);
1963 return getAddExpr(SSExtD, SSExtR,
1964 (SCEV::NoWrapFlags)(SCEV::FlagNSW | SCEV::FlagNUW),
1965 Depth + 1);
1966 }
1967 }
1968 }
1969 // If the input value is a chrec scev, and we can prove that the value
1970 // did not overflow the old, smaller, value, we can sign extend all of the
1971 // operands (often constants). This allows analysis of something like
1972 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; }
1973 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op))
1974 if (AR->isAffine()) {
1975 const SCEV *Start = AR->getStart();
1976 const SCEV *Step = AR->getStepRecurrence(*this);
1977 unsigned BitWidth = getTypeSizeInBits(AR->getType());
1978 const Loop *L = AR->getLoop();
1979
1980 if (!AR->hasNoSignedWrap()) {
1981 auto NewFlags = proveNoWrapViaConstantRanges(AR);
1982 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), NewFlags);
1983 }
1984
1985 // If we have special knowledge that this addrec won't overflow,
1986 // we don't need to do any further analysis.
1987 if (AR->hasNoSignedWrap())
1988 return getAddRecExpr(
1989 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1),
1990 getSignExtendExpr(Step, Ty, Depth + 1), L, SCEV::FlagNSW);
1991
1992 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1993 // Note that this serves two purposes: It filters out loops that are
1994 // simply not analyzable, and it covers the case where this code is
1995 // being called from within backedge-taken count analysis, such that
1996 // attempting to ask for the backedge-taken count would likely result
1997 // in infinite recursion. In the later case, the analysis code will
1998 // cope with a conservative value, and it will take care to purge
1999 // that value once it has finished.
2000 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
2001 if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
2002 // Manually compute the final value for AR, checking for
2003 // overflow.
2004
2005 // Check whether the backedge-taken count can be losslessly casted to
2006 // the addrec's type. The count is always unsigned.
2007 const SCEV *CastedMaxBECount =
2008 getTruncateOrZeroExtend(MaxBECount, Start->getType(), Depth);
2009 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
2010 CastedMaxBECount, MaxBECount->getType(), Depth);
2011 if (MaxBECount == RecastedMaxBECount) {
2012 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
2013 // Check whether Start+Step*MaxBECount has no signed overflow.
2014 const SCEV *SMul = getMulExpr(CastedMaxBECount, Step,
2015 SCEV::FlagAnyWrap, Depth + 1);
2016 const SCEV *SAdd = getSignExtendExpr(getAddExpr(Start, SMul,
2017 SCEV::FlagAnyWrap,
2018 Depth + 1),
2019 WideTy, Depth + 1);
2020 const SCEV *WideStart = getSignExtendExpr(Start, WideTy, Depth + 1);
2021 const SCEV *WideMaxBECount =
2022 getZeroExtendExpr(CastedMaxBECount, WideTy, Depth + 1);
2023 const SCEV *OperandExtendedAdd =
2024 getAddExpr(WideStart,
2025 getMulExpr(WideMaxBECount,
2026 getSignExtendExpr(Step, WideTy, Depth + 1),
2027 SCEV::FlagAnyWrap, Depth + 1),
2028 SCEV::FlagAnyWrap, Depth + 1);
2029 if (SAdd == OperandExtendedAdd) {
2030 // Cache knowledge of AR NSW, which is propagated to this AddRec.
2031 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNSW);
2032 // Return the expression with the addrec on the outside.
2033 return getAddRecExpr(
2034 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this,
2035 Depth + 1),
2036 getSignExtendExpr(Step, Ty, Depth + 1), L,
2037 AR->getNoWrapFlags());
2038 }
2039 // Similar to above, only this time treat the step value as unsigned.
2040 // This covers loops that count up with an unsigned step.
2041 OperandExtendedAdd =
2042 getAddExpr(WideStart,
2043 getMulExpr(WideMaxBECount,
2044 getZeroExtendExpr(Step, WideTy, Depth + 1),
2045 SCEV::FlagAnyWrap, Depth + 1),
2046 SCEV::FlagAnyWrap, Depth + 1);
2047 if (SAdd == OperandExtendedAdd) {
2048 // If AR wraps around then
2049 //
2050 // abs(Step) * MaxBECount > unsigned-max(AR->getType())
2051 // => SAdd != OperandExtendedAdd
2052 //
2053 // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=>
2054 // (SAdd == OperandExtendedAdd => AR is NW)
2055
2056 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW);
2057
2058 // Return the expression with the addrec on the outside.
2059 return getAddRecExpr(
2060 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this,
2061 Depth + 1),
2062 getZeroExtendExpr(Step, Ty, Depth + 1), L,
2063 AR->getNoWrapFlags());
2064 }
2065 }
2066 }
2067
2068 auto NewFlags = proveNoSignedWrapViaInduction(AR);
2069 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), NewFlags);
2070 if (AR->hasNoSignedWrap()) {
2071 // Same as nsw case above - duplicated here to avoid a compile time
2072 // issue. It's not clear that the order of checks does matter, but
2073 // it's one of two issue possible causes for a change which was
2074 // reverted. Be conservative for the moment.
2075 return getAddRecExpr(
2076 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1),
2077 getSignExtendExpr(Step, Ty, Depth + 1), L, AR->getNoWrapFlags());
2078 }
2079
2080 // sext({C,+,Step}) --> (sext(D) + sext({C-D,+,Step}))<nuw><nsw>
2081 // if D + (C - D + Step * n) could be proven to not signed wrap
2082 // where D maximizes the number of trailing zeros of (C - D + Step * n)
2083 if (const auto *SC = dyn_cast<SCEVConstant>(Start)) {
2084 const APInt &C = SC->getAPInt();
2085 const APInt &D = extractConstantWithoutWrapping(*this, C, Step);
2086 if (D != 0) {
2087 const SCEV *SSExtD = getSignExtendExpr(getConstant(D), Ty, Depth);
2088 const SCEV *SResidual =
2089 getAddRecExpr(getConstant(C - D), Step, L, AR->getNoWrapFlags());
2090 const SCEV *SSExtR = getSignExtendExpr(SResidual, Ty, Depth + 1);
2091 return getAddExpr(SSExtD, SSExtR,
2092 (SCEV::NoWrapFlags)(SCEV::FlagNSW | SCEV::FlagNUW),
2093 Depth + 1);
2094 }
2095 }
2096
2097 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
2098 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNSW);
2099 return getAddRecExpr(
2100 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1),
2101 getSignExtendExpr(Step, Ty, Depth + 1), L, AR->getNoWrapFlags());
2102 }
2103 }
2104
2105 // If the input value is provably positive and we could not simplify
2106 // away the sext build a zext instead.
2107 if (isKnownNonNegative(Op))
2108 return getZeroExtendExpr(Op, Ty, Depth + 1);
2109
2110 // The cast wasn't folded; create an explicit cast node.
2111 // Recompute the insert position, as it may have been invalidated.
2112 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
2113 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
2114 Op, Ty);
2115 UniqueSCEVs.InsertNode(S, IP);
2116 registerUser(S, { Op });
2117 return S;
2118}
2119
2120const SCEV *ScalarEvolution::getCastExpr(SCEVTypes Kind, const SCEV *Op,
2121 Type *Ty) {
2122 switch (Kind) {
2123 case scTruncate:
2124 return getTruncateExpr(Op, Ty);
2125 case scZeroExtend:
2126 return getZeroExtendExpr(Op, Ty);
2127 case scSignExtend:
2128 return getSignExtendExpr(Op, Ty);
2129 case scPtrToInt:
2130 return getPtrToIntExpr(Op, Ty);
2131 default:
2132 llvm_unreachable("Not a SCEV cast expression!")::llvm::llvm_unreachable_internal("Not a SCEV cast expression!"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2132)
;
2133 }
2134}
2135
2136/// getAnyExtendExpr - Return a SCEV for the given operand extended with
2137/// unspecified bits out to the given type.
2138const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op,
2139 Type *Ty) {
2140 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&(static_cast <bool> (getTypeSizeInBits(Op->getType()
) < getTypeSizeInBits(Ty) && "This is not an extending conversion!"
) ? void (0) : __assert_fail ("getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && \"This is not an extending conversion!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2141, __extension__
__PRETTY_FUNCTION__))
2141 "This is not an extending conversion!")(static_cast <bool> (getTypeSizeInBits(Op->getType()
) < getTypeSizeInBits(Ty) && "This is not an extending conversion!"
) ? void (0) : __assert_fail ("getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && \"This is not an extending conversion!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2141, __extension__
__PRETTY_FUNCTION__))
;
2142 assert(isSCEVable(Ty) &&(static_cast <bool> (isSCEVable(Ty) && "This is not a conversion to a SCEVable type!"
) ? void (0) : __assert_fail ("isSCEVable(Ty) && \"This is not a conversion to a SCEVable type!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2143, __extension__
__PRETTY_FUNCTION__))
2143 "This is not a conversion to a SCEVable type!")(static_cast <bool> (isSCEVable(Ty) && "This is not a conversion to a SCEVable type!"
) ? void (0) : __assert_fail ("isSCEVable(Ty) && \"This is not a conversion to a SCEVable type!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2143, __extension__
__PRETTY_FUNCTION__))
;
2144 Ty = getEffectiveSCEVType(Ty);
2145
2146 // Sign-extend negative constants.
2147 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
2148 if (SC->getAPInt().isNegative())
2149 return getSignExtendExpr(Op, Ty);
2150
2151 // Peel off a truncate cast.
2152 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Op)) {
2153 const SCEV *NewOp = T->getOperand();
2154 if (getTypeSizeInBits(NewOp->getType()) < getTypeSizeInBits(Ty))
2155 return getAnyExtendExpr(NewOp, Ty);
2156 return getTruncateOrNoop(NewOp, Ty);
2157 }
2158
2159 // Next try a zext cast. If the cast is folded, use it.
2160 const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
2161 if (!isa<SCEVZeroExtendExpr>(ZExt))
2162 return ZExt;
2163
2164 // Next try a sext cast. If the cast is folded, use it.
2165 const SCEV *SExt = getSignExtendExpr(Op, Ty);
2166 if (!isa<SCEVSignExtendExpr>(SExt))
2167 return SExt;
2168
2169 // Force the cast to be folded into the operands of an addrec.
2170 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) {
2171 SmallVector<const SCEV *, 4> Ops;
2172 for (const SCEV *Op : AR->operands())
2173 Ops.push_back(getAnyExtendExpr(Op, Ty));
2174 return getAddRecExpr(Ops, AR->getLoop(), SCEV::FlagNW);
2175 }
2176
2177 // If the expression is obviously signed, use the sext cast value.
2178 if (isa<SCEVSMaxExpr>(Op))
2179 return SExt;
2180
2181 // Absent any other information, use the zext cast value.
2182 return ZExt;
2183}
2184
2185/// Process the given Ops list, which is a list of operands to be added under
2186/// the given scale, update the given map. This is a helper function for
2187/// getAddRecExpr. As an example of what it does, given a sequence of operands
2188/// that would form an add expression like this:
2189///
2190/// m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
2191///
2192/// where A and B are constants, update the map with these values:
2193///
2194/// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
2195///
2196/// and add 13 + A*B*29 to AccumulatedConstant.
2197/// This will allow getAddRecExpr to produce this:
2198///
2199/// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
2200///
2201/// This form often exposes folding opportunities that are hidden in
2202/// the original operand list.
2203///
2204/// Return true iff it appears that any interesting folding opportunities
2205/// may be exposed. This helps getAddRecExpr short-circuit extra work in
2206/// the common case where no interesting opportunities are present, and
2207/// is also used as a check to avoid infinite recursion.
2208static bool
2209CollectAddOperandsWithScales(DenseMap<const SCEV *, APInt> &M,
2210 SmallVectorImpl<const SCEV *> &NewOps,
2211 APInt &AccumulatedConstant,
2212 const SCEV *const *Ops, size_t NumOperands,
2213 const APInt &Scale,
2214 ScalarEvolution &SE) {
2215 bool Interesting = false;
2216
2217 // Iterate over the add operands. They are sorted, with constants first.
2218 unsigned i = 0;
2219 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) {
2220 ++i;
2221 // Pull a buried constant out to the outside.
2222 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
2223 Interesting = true;
2224 AccumulatedConstant += Scale * C->getAPInt();
2225 }
2226
2227 // Next comes everything else. We're especially interested in multiplies
2228 // here, but they're in the middle, so just visit the rest with one loop.
2229 for (; i != NumOperands; ++i) {
2230 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[i]);
2231 if (Mul && isa<SCEVConstant>(Mul->getOperand(0))) {
2232 APInt NewScale =
2233 Scale * cast<SCEVConstant>(Mul->getOperand(0))->getAPInt();
2234 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) {
2235 // A multiplication of a constant with another add; recurse.
2236 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1));
2237 Interesting |=
2238 CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
2239 Add->op_begin(), Add->getNumOperands(),
2240 NewScale, SE);
2241 } else {
2242 // A multiplication of a constant with some other value. Update
2243 // the map.
2244 SmallVector<const SCEV *, 4> MulOps(drop_begin(Mul->operands()));
2245 const SCEV *Key = SE.getMulExpr(MulOps);
2246 auto Pair = M.insert({Key, NewScale});
2247 if (Pair.second) {
2248 NewOps.push_back(Pair.first->first);
2249 } else {
2250 Pair.first->second += NewScale;
2251 // The map already had an entry for this value, which may indicate
2252 // a folding opportunity.
2253 Interesting = true;
2254 }
2255 }
2256 } else {
2257 // An ordinary operand. Update the map.
2258 std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair =
2259 M.insert({Ops[i], Scale});
2260 if (Pair.second) {
2261 NewOps.push_back(Pair.first->first);
2262 } else {
2263 Pair.first->second += Scale;
2264 // The map already had an entry for this value, which may indicate
2265 // a folding opportunity.
2266 Interesting = true;
2267 }
2268 }
2269 }
2270
2271 return Interesting;
2272}
2273
2274bool ScalarEvolution::willNotOverflow(Instruction::BinaryOps BinOp, bool Signed,
2275 const SCEV *LHS, const SCEV *RHS) {
2276 const SCEV *(ScalarEvolution::*Operation)(const SCEV *, const SCEV *,
2277 SCEV::NoWrapFlags, unsigned);
2278 switch (BinOp) {
2279 default:
2280 llvm_unreachable("Unsupported binary op")::llvm::llvm_unreachable_internal("Unsupported binary op", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 2280)
;
2281 case Instruction::Add:
2282 Operation = &ScalarEvolution::getAddExpr;
2283 break;
2284 case Instruction::Sub:
2285 Operation = &ScalarEvolution::getMinusSCEV;
2286 break;
2287 case Instruction::Mul:
2288 Operation = &ScalarEvolution::getMulExpr;
2289 break;
2290 }
2291
2292 const SCEV *(ScalarEvolution::*Extension)(const SCEV *, Type *, unsigned) =
2293 Signed ? &ScalarEvolution::getSignExtendExpr
2294 : &ScalarEvolution::getZeroExtendExpr;
2295
2296 // Check ext(LHS op RHS) == ext(LHS) op ext(RHS)
2297 auto *NarrowTy = cast<IntegerType>(LHS->getType());
2298 auto *WideTy =
2299 IntegerType::get(NarrowTy->getContext(), NarrowTy->getBitWidth() * 2);
2300
2301 const SCEV *A = (this->*Extension)(
2302 (this->*Operation)(LHS, RHS, SCEV::FlagAnyWrap, 0), WideTy, 0);
2303 const SCEV *B = (this->*Operation)((this->*Extension)(LHS, WideTy, 0),
2304 (this->*Extension)(RHS, WideTy, 0),
2305 SCEV::FlagAnyWrap, 0);
2306 return A == B;
2307}
2308
2309std::pair<SCEV::NoWrapFlags, bool /*Deduced*/>
2310ScalarEvolution::getStrengthenedNoWrapFlagsFromBinOp(
2311 const OverflowingBinaryOperator *OBO) {
2312 SCEV::NoWrapFlags Flags = SCEV::NoWrapFlags::FlagAnyWrap;
2313
2314 if (OBO->hasNoUnsignedWrap())
2315 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
2316 if (OBO->hasNoSignedWrap())
2317 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW);
2318
2319 bool Deduced = false;
2320
2321 if (OBO->hasNoUnsignedWrap() && OBO->hasNoSignedWrap())
2322 return {Flags, Deduced};
2323
2324 if (OBO->getOpcode() != Instruction::Add &&
2325 OBO->getOpcode() != Instruction::Sub &&
2326 OBO->getOpcode() != Instruction::Mul)
2327 return {Flags, Deduced};
2328
2329 const SCEV *LHS = getSCEV(OBO->getOperand(0));
2330 const SCEV *RHS = getSCEV(OBO->getOperand(1));
2331
2332 if (!OBO->hasNoUnsignedWrap() &&
2333 willNotOverflow((Instruction::BinaryOps)OBO->getOpcode(),
2334 /* Signed */ false, LHS, RHS)) {
2335 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
2336 Deduced = true;
2337 }
2338
2339 if (!OBO->hasNoSignedWrap() &&
2340 willNotOverflow((Instruction::BinaryOps)OBO->getOpcode(),
2341 /* Signed */ true, LHS, RHS)) {
2342 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW);
2343 Deduced = true;
2344 }
2345
2346 return {Flags, Deduced};
2347}
2348
2349// We're trying to construct a SCEV of type `Type' with `Ops' as operands and
2350// `OldFlags' as can't-wrap behavior. Infer a more aggressive set of
2351// can't-overflow flags for the operation if possible.
2352static SCEV::NoWrapFlags
2353StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type,
2354 const ArrayRef<const SCEV *> Ops,
2355 SCEV::NoWrapFlags Flags) {
2356 using namespace std::placeholders;
2357
2358 using OBO = OverflowingBinaryOperator;
2359
2360 bool CanAnalyze =
2361 Type == scAddExpr || Type == scAddRecExpr || Type == scMulExpr;
2362 (void)CanAnalyze;
2363 assert(CanAnalyze && "don't call from other places!")(static_cast <bool> (CanAnalyze && "don't call from other places!"
) ? void (0) : __assert_fail ("CanAnalyze && \"don't call from other places!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2363, __extension__
__PRETTY_FUNCTION__))
;
2364
2365 int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
2366 SCEV::NoWrapFlags SignOrUnsignWrap =
2367 ScalarEvolution::maskFlags(Flags, SignOrUnsignMask);
2368
2369 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
2370 auto IsKnownNonNegative = [&](const SCEV *S) {
2371 return SE->isKnownNonNegative(S);
2372 };
2373
2374 if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Ops, IsKnownNonNegative))
2375 Flags =
2376 ScalarEvolution::setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask);
2377
2378 SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, SignOrUnsignMask);
2379
2380 if (SignOrUnsignWrap != SignOrUnsignMask &&
2381 (Type == scAddExpr || Type == scMulExpr) && Ops.size() == 2 &&
2382 isa<SCEVConstant>(Ops[0])) {
2383
2384 auto Opcode = [&] {
2385 switch (Type) {
2386 case scAddExpr:
2387 return Instruction::Add;
2388 case scMulExpr:
2389 return Instruction::Mul;
2390 default:
2391 llvm_unreachable("Unexpected SCEV op.")::llvm::llvm_unreachable_internal("Unexpected SCEV op.", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 2391)
;
2392 }
2393 }();
2394
2395 const APInt &C = cast<SCEVConstant>(Ops[0])->getAPInt();
2396
2397 // (A <opcode> C) --> (A <opcode> C)<nsw> if the op doesn't sign overflow.
2398 if (!(SignOrUnsignWrap & SCEV::FlagNSW)) {
2399 auto NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
2400 Opcode, C, OBO::NoSignedWrap);
2401 if (NSWRegion.contains(SE->getSignedRange(Ops[1])))
2402 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW);
2403 }
2404
2405 // (A <opcode> C) --> (A <opcode> C)<nuw> if the op doesn't unsign overflow.
2406 if (!(SignOrUnsignWrap & SCEV::FlagNUW)) {
2407 auto NUWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
2408 Opcode, C, OBO::NoUnsignedWrap);
2409 if (NUWRegion.contains(SE->getUnsignedRange(Ops[1])))
2410 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
2411 }
2412 }
2413
2414 // <0,+,nonnegative><nw> is also nuw
2415 // TODO: Add corresponding nsw case
2416 if (Type == scAddRecExpr && ScalarEvolution::hasFlags(Flags, SCEV::FlagNW) &&
2417 !ScalarEvolution::hasFlags(Flags, SCEV::FlagNUW) && Ops.size() == 2 &&
2418 Ops[0]->isZero() && IsKnownNonNegative(Ops[1]))
2419 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
2420
2421 // both (udiv X, Y) * Y and Y * (udiv X, Y) are always NUW
2422 if (Type == scMulExpr && !ScalarEvolution::hasFlags(Flags, SCEV::FlagNUW) &&
2423 Ops.size() == 2) {
2424 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Ops[0]))
2425 if (UDiv->getOperand(1) == Ops[1])
2426 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
2427 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Ops[1]))
2428 if (UDiv->getOperand(1) == Ops[0])
2429 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
2430 }
2431
2432 return Flags;
2433}
2434
2435bool ScalarEvolution::isAvailableAtLoopEntry(const SCEV *S, const Loop *L) {
2436 return isLoopInvariant(S, L) && properlyDominates(S, L->getHeader());
2437}
2438
2439/// Get a canonical add expression, or something simpler if possible.
2440const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
2441 SCEV::NoWrapFlags OrigFlags,
2442 unsigned Depth) {
2443 assert(!(OrigFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&(static_cast <bool> (!(OrigFlags & ~(SCEV::FlagNUW |
SCEV::FlagNSW)) && "only nuw or nsw allowed") ? void
(0) : __assert_fail ("!(OrigFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) && \"only nuw or nsw allowed\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2444, __extension__
__PRETTY_FUNCTION__))
2444 "only nuw or nsw allowed")(static_cast <bool> (!(OrigFlags & ~(SCEV::FlagNUW |
SCEV::FlagNSW)) && "only nuw or nsw allowed") ? void
(0) : __assert_fail ("!(OrigFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) && \"only nuw or nsw allowed\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2444, __extension__
__PRETTY_FUNCTION__))
;
2445 assert(!Ops.empty() && "Cannot get empty add!")(static_cast <bool> (!Ops.empty() && "Cannot get empty add!"
) ? void (0) : __assert_fail ("!Ops.empty() && \"Cannot get empty add!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2445, __extension__
__PRETTY_FUNCTION__))
;
2446 if (Ops.size() == 1) return Ops[0];
2447#ifndef NDEBUG
2448 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
2449 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
2450 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&(static_cast <bool> (getEffectiveSCEVType(Ops[i]->getType
()) == ETy && "SCEVAddExpr operand types don't match!"
) ? void (0) : __assert_fail ("getEffectiveSCEVType(Ops[i]->getType()) == ETy && \"SCEVAddExpr operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2451, __extension__
__PRETTY_FUNCTION__))
2451 "SCEVAddExpr operand types don't match!")(static_cast <bool> (getEffectiveSCEVType(Ops[i]->getType
()) == ETy && "SCEVAddExpr operand types don't match!"
) ? void (0) : __assert_fail ("getEffectiveSCEVType(Ops[i]->getType()) == ETy && \"SCEVAddExpr operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2451, __extension__
__PRETTY_FUNCTION__))
;
2452 unsigned NumPtrs = count_if(
2453 Ops, [](const SCEV *Op) { return Op->getType()->isPointerTy(); });
2454 assert(NumPtrs <= 1 && "add has at most one pointer operand")(static_cast <bool> (NumPtrs <= 1 && "add has at most one pointer operand"
) ? void (0) : __assert_fail ("NumPtrs <= 1 && \"add has at most one pointer operand\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2454, __extension__
__PRETTY_FUNCTION__))
;
2455#endif
2456
2457 // Sort by complexity, this groups all similar expression types together.
2458 GroupByComplexity(Ops, &LI, DT);
2459
2460 // If there are any constants, fold them together.
2461 unsigned Idx = 0;
2462 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
2463 ++Idx;
2464 assert(Idx < Ops.size())(static_cast <bool> (Idx < Ops.size()) ? void (0) : __assert_fail
("Idx < Ops.size()", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 2464, __extension__ __PRETTY_FUNCTION__))
;
2465 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
2466 // We found two constants, fold them together!
2467 Ops[0] = getConstant(LHSC->getAPInt() + RHSC->getAPInt());
2468 if (Ops.size() == 2) return Ops[0];
2469 Ops.erase(Ops.begin()+1); // Erase the folded element
2470 LHSC = cast<SCEVConstant>(Ops[0]);
2471 }
2472
2473 // If we are left with a constant zero being added, strip it off.
2474 if (LHSC->getValue()->isZero()) {
2475 Ops.erase(Ops.begin());
2476 --Idx;
2477 }
2478
2479 if (Ops.size() == 1) return Ops[0];
2480 }
2481
2482 // Delay expensive flag strengthening until necessary.
2483 auto ComputeFlags = [this, OrigFlags](const ArrayRef<const SCEV *> Ops) {
2484 return StrengthenNoWrapFlags(this, scAddExpr, Ops, OrigFlags);
2485 };
2486
2487 // Limit recursion calls depth.
2488 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
2489 return getOrCreateAddExpr(Ops, ComputeFlags(Ops));
2490
2491 if (SCEV *S = findExistingSCEVInCache(scAddExpr, Ops)) {
2492 // Don't strengthen flags if we have no new information.
2493 SCEVAddExpr *Add = static_cast<SCEVAddExpr *>(S);
2494 if (Add->getNoWrapFlags(OrigFlags) != OrigFlags)
2495 Add->setNoWrapFlags(ComputeFlags(Ops));
2496 return S;
2497 }
2498
2499 // Okay, check to see if the same value occurs in the operand list more than
2500 // once. If so, merge them together into an multiply expression. Since we
2501 // sorted the list, these values are required to be adjacent.
2502 Type *Ty = Ops[0]->getType();
2503 bool FoundMatch = false;
2504 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
2505 if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2
2506 // Scan ahead to count how many equal operands there are.
2507 unsigned Count = 2;
2508 while (i+Count != e && Ops[i+Count] == Ops[i])
2509 ++Count;
2510 // Merge the values into a multiply.
2511 const SCEV *Scale = getConstant(Ty, Count);
2512 const SCEV *Mul = getMulExpr(Scale, Ops[i], SCEV::FlagAnyWrap, Depth + 1);
2513 if (Ops.size() == Count)
2514 return Mul;
2515 Ops[i] = Mul;
2516 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+Count);
2517 --i; e -= Count - 1;
2518 FoundMatch = true;
2519 }
2520 if (FoundMatch)
2521 return getAddExpr(Ops, OrigFlags, Depth + 1);
2522
2523 // Check for truncates. If all the operands are truncated from the same
2524 // type, see if factoring out the truncate would permit the result to be
2525 // folded. eg., n*trunc(x) + m*trunc(y) --> trunc(trunc(m)*x + trunc(n)*y)
2526 // if the contents of the resulting outer trunc fold to something simple.
2527 auto FindTruncSrcType = [&]() -> Type * {
2528 // We're ultimately looking to fold an addrec of truncs and muls of only
2529 // constants and truncs, so if we find any other types of SCEV
2530 // as operands of the addrec then we bail and return nullptr here.
2531 // Otherwise, we return the type of the operand of a trunc that we find.
2532 if (auto *T = dyn_cast<SCEVTruncateExpr>(Ops[Idx]))
2533 return T->getOperand()->getType();
2534 if (const auto *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
2535 const auto *LastOp = Mul->getOperand(Mul->getNumOperands() - 1);
2536 if (const auto *T = dyn_cast<SCEVTruncateExpr>(LastOp))
2537 return T->getOperand()->getType();
2538 }
2539 return nullptr;
2540 };
2541 if (auto *SrcType = FindTruncSrcType()) {
2542 SmallVector<const SCEV *, 8> LargeOps;
2543 bool Ok = true;
2544 // Check all the operands to see if they can be represented in the
2545 // source type of the truncate.
2546 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
2547 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Ops[i])) {
2548 if (T->getOperand()->getType() != SrcType) {
2549 Ok = false;
2550 break;
2551 }
2552 LargeOps.push_back(T->getOperand());
2553 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) {
2554 LargeOps.push_back(getAnyExtendExpr(C, SrcType));
2555 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Ops[i])) {
2556 SmallVector<const SCEV *, 8> LargeMulOps;
2557 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
2558 if (const SCEVTruncateExpr *T =
2559 dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) {
2560 if (T->getOperand()->getType() != SrcType) {
2561 Ok = false;
2562 break;
2563 }
2564 LargeMulOps.push_back(T->getOperand());
2565 } else if (const auto *C = dyn_cast<SCEVConstant>(M->getOperand(j))) {
2566 LargeMulOps.push_back(getAnyExtendExpr(C, SrcType));
2567 } else {
2568 Ok = false;
2569 break;
2570 }
2571 }
2572 if (Ok)
2573 LargeOps.push_back(getMulExpr(LargeMulOps, SCEV::FlagAnyWrap, Depth + 1));
2574 } else {
2575 Ok = false;
2576 break;
2577 }
2578 }
2579 if (Ok) {
2580 // Evaluate the expression in the larger type.
2581 const SCEV *Fold = getAddExpr(LargeOps, SCEV::FlagAnyWrap, Depth + 1);
2582 // If it folds to something simple, use it. Otherwise, don't.
2583 if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold))
2584 return getTruncateExpr(Fold, Ty);
2585 }
2586 }
2587
2588 if (Ops.size() == 2) {
2589 // Check if we have an expression of the form ((X + C1) - C2), where C1 and
2590 // C2 can be folded in a way that allows retaining wrapping flags of (X +
2591 // C1).
2592 const SCEV *A = Ops[0];
2593 const SCEV *B = Ops[1];
2594 auto *AddExpr = dyn_cast<SCEVAddExpr>(B);
2595 auto *C = dyn_cast<SCEVConstant>(A);
2596 if (AddExpr && C && isa<SCEVConstant>(AddExpr->getOperand(0))) {
2597 auto C1 = cast<SCEVConstant>(AddExpr->getOperand(0))->getAPInt();
2598 auto C2 = C->getAPInt();
2599 SCEV::NoWrapFlags PreservedFlags = SCEV::FlagAnyWrap;
2600
2601 APInt ConstAdd = C1 + C2;
2602 auto AddFlags = AddExpr->getNoWrapFlags();
2603 // Adding a smaller constant is NUW if the original AddExpr was NUW.
2604 if (ScalarEvolution::hasFlags(AddFlags, SCEV::FlagNUW) &&
2605 ConstAdd.ule(C1)) {
2606 PreservedFlags =
2607 ScalarEvolution::setFlags(PreservedFlags, SCEV::FlagNUW);
2608 }
2609
2610 // Adding a constant with the same sign and small magnitude is NSW, if the
2611 // original AddExpr was NSW.
2612 if (ScalarEvolution::hasFlags(AddFlags, SCEV::FlagNSW) &&
2613 C1.isSignBitSet() == ConstAdd.isSignBitSet() &&
2614 ConstAdd.abs().ule(C1.abs())) {
2615 PreservedFlags =
2616 ScalarEvolution::setFlags(PreservedFlags, SCEV::FlagNSW);
2617 }
2618
2619 if (PreservedFlags != SCEV::FlagAnyWrap) {
2620 SmallVector<const SCEV *, 4> NewOps(AddExpr->operands());
2621 NewOps[0] = getConstant(ConstAdd);
2622 return getAddExpr(NewOps, PreservedFlags);
2623 }
2624 }
2625 }
2626
2627 // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y)
2628 if (Ops.size() == 2) {
2629 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[0]);
2630 if (Mul && Mul->getNumOperands() == 2 &&
2631 Mul->getOperand(0)->isAllOnesValue()) {
2632 const SCEV *X;
2633 const SCEV *Y;
2634 if (matchURem(Mul->getOperand(1), X, Y) && X == Ops[1]) {
2635 return getMulExpr(Y, getUDivExpr(X, Y));
2636 }
2637 }
2638 }
2639
2640 // Skip past any other cast SCEVs.
2641 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
2642 ++Idx;
2643
2644 // If there are add operands they would be next.
2645 if (Idx < Ops.size()) {
2646 bool DeletedAdd = false;
2647 // If the original flags and all inlined SCEVAddExprs are NUW, use the
2648 // common NUW flag for expression after inlining. Other flags cannot be
2649 // preserved, because they may depend on the original order of operations.
2650 SCEV::NoWrapFlags CommonFlags = maskFlags(OrigFlags, SCEV::FlagNUW);
2651 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) {
2652 if (Ops.size() > AddOpsInlineThreshold ||
2653 Add->getNumOperands() > AddOpsInlineThreshold)
2654 break;
2655 // If we have an add, expand the add operands onto the end of the operands
2656 // list.
2657 Ops.erase(Ops.begin()+Idx);
2658 Ops.append(Add->op_begin(), Add->op_end());
2659 DeletedAdd = true;
2660 CommonFlags = maskFlags(CommonFlags, Add->getNoWrapFlags());
2661 }
2662
2663 // If we deleted at least one add, we added operands to the end of the list,
2664 // and they are not necessarily sorted. Recurse to resort and resimplify
2665 // any operands we just acquired.
2666 if (DeletedAdd)
2667 return getAddExpr(Ops, CommonFlags, Depth + 1);
2668 }
2669
2670 // Skip over the add expression until we get to a multiply.
2671 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
2672 ++Idx;
2673
2674 // Check to see if there are any folding opportunities present with
2675 // operands multiplied by constant values.
2676 if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) {
2677 uint64_t BitWidth = getTypeSizeInBits(Ty);
2678 DenseMap<const SCEV *, APInt> M;
2679 SmallVector<const SCEV *, 8> NewOps;
2680 APInt AccumulatedConstant(BitWidth, 0);
2681 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
2682 Ops.data(), Ops.size(),
2683 APInt(BitWidth, 1), *this)) {
2684 struct APIntCompare {
2685 bool operator()(const APInt &LHS, const APInt &RHS) const {
2686 return LHS.ult(RHS);
2687 }
2688 };
2689
2690 // Some interesting folding opportunity is present, so its worthwhile to
2691 // re-generate the operands list. Group the operands by constant scale,
2692 // to avoid multiplying by the same constant scale multiple times.
2693 std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists;
2694 for (const SCEV *NewOp : NewOps)
2695 MulOpLists[M.find(NewOp)->second].push_back(NewOp);
2696 // Re-generate the operands list.
2697 Ops.clear();
2698 if (AccumulatedConstant != 0)
2699 Ops.push_back(getConstant(AccumulatedConstant));
2700 for (auto &MulOp : MulOpLists) {
2701 if (MulOp.first == 1) {
2702 Ops.push_back(getAddExpr(MulOp.second, SCEV::FlagAnyWrap, Depth + 1));
2703 } else if (MulOp.first != 0) {
2704 Ops.push_back(getMulExpr(
2705 getConstant(MulOp.first),
2706 getAddExpr(MulOp.second, SCEV::FlagAnyWrap, Depth + 1),
2707 SCEV::FlagAnyWrap, Depth + 1));
2708 }
2709 }
2710 if (Ops.empty())
2711 return getZero(Ty);
2712 if (Ops.size() == 1)
2713 return Ops[0];
2714 return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
2715 }
2716 }
2717
2718 // If we are adding something to a multiply expression, make sure the
2719 // something is not already an operand of the multiply. If so, merge it into
2720 // the multiply.
2721 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) {
2722 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]);
2723 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
2724 const SCEV *MulOpSCEV = Mul->getOperand(MulOp);
2725 if (isa<SCEVConstant>(MulOpSCEV))
2726 continue;
2727 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp)
2728 if (MulOpSCEV == Ops[AddOp]) {
2729 // Fold W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1))
2730 const SCEV *InnerMul = Mul->getOperand(MulOp == 0);
2731 if (Mul->getNumOperands() != 2) {
2732 // If the multiply has more than two operands, we must get the
2733 // Y*Z term.
2734 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(),
2735 Mul->op_begin()+MulOp);
2736 MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end());
2737 InnerMul = getMulExpr(MulOps, SCEV::FlagAnyWrap, Depth + 1);
2738 }
2739 SmallVector<const SCEV *, 2> TwoOps = {getOne(Ty), InnerMul};
2740 const SCEV *AddOne = getAddExpr(TwoOps, SCEV::FlagAnyWrap, Depth + 1);
2741 const SCEV *OuterMul = getMulExpr(AddOne, MulOpSCEV,
2742 SCEV::FlagAnyWrap, Depth + 1);
2743 if (Ops.size() == 2) return OuterMul;
2744 if (AddOp < Idx) {
2745 Ops.erase(Ops.begin()+AddOp);
2746 Ops.erase(Ops.begin()+Idx-1);
2747 } else {
2748 Ops.erase(Ops.begin()+Idx);
2749 Ops.erase(Ops.begin()+AddOp-1);
2750 }
2751 Ops.push_back(OuterMul);
2752 return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
2753 }
2754
2755 // Check this multiply against other multiplies being added together.
2756 for (unsigned OtherMulIdx = Idx+1;
2757 OtherMulIdx < Ops.size() && isa<SCEVMulExpr>(Ops[OtherMulIdx]);
2758 ++OtherMulIdx) {
2759 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[OtherMulIdx]);
2760 // If MulOp occurs in OtherMul, we can fold the two multiplies
2761 // together.
2762 for (unsigned OMulOp = 0, e = OtherMul->getNumOperands();
2763 OMulOp != e; ++OMulOp)
2764 if (OtherMul->getOperand(OMulOp) == MulOpSCEV) {
2765 // Fold X + (A*B*C) + (A*D*E) --> X + (A*(B*C+D*E))
2766 const SCEV *InnerMul1 = Mul->getOperand(MulOp == 0);
2767 if (Mul->getNumOperands() != 2) {
2768 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(),
2769 Mul->op_begin()+MulOp);
2770 MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end());
2771 InnerMul1 = getMulExpr(MulOps, SCEV::FlagAnyWrap, Depth + 1);
2772 }
2773 const SCEV *InnerMul2 = OtherMul->getOperand(OMulOp == 0);
2774 if (OtherMul->getNumOperands() != 2) {
2775 SmallVector<const SCEV *, 4> MulOps(OtherMul->op_begin(),
2776 OtherMul->op_begin()+OMulOp);
2777 MulOps.append(OtherMul->op_begin()+OMulOp+1, OtherMul->op_end());
2778 InnerMul2 = getMulExpr(MulOps, SCEV::FlagAnyWrap, Depth + 1);
2779 }
2780 SmallVector<const SCEV *, 2> TwoOps = {InnerMul1, InnerMul2};
2781 const SCEV *InnerMulSum =
2782 getAddExpr(TwoOps, SCEV::FlagAnyWrap, Depth + 1);
2783 const SCEV *OuterMul = getMulExpr(MulOpSCEV, InnerMulSum,
2784 SCEV::FlagAnyWrap, Depth + 1);
2785 if (Ops.size() == 2) return OuterMul;
2786 Ops.erase(Ops.begin()+Idx);
2787 Ops.erase(Ops.begin()+OtherMulIdx-1);
2788 Ops.push_back(OuterMul);
2789 return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
2790 }
2791 }
2792 }
2793 }
2794
2795 // If there are any add recurrences in the operands list, see if any other
2796 // added values are loop invariant. If so, we can fold them into the
2797 // recurrence.
2798 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
2799 ++Idx;
2800
2801 // Scan over all recurrences, trying to fold loop invariants into them.
2802 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
2803 // Scan all of the other operands to this add and add them to the vector if
2804 // they are loop invariant w.r.t. the recurrence.
2805 SmallVector<const SCEV *, 8> LIOps;
2806 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
2807 const Loop *AddRecLoop = AddRec->getLoop();
2808 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
2809 if (isAvailableAtLoopEntry(Ops[i], AddRecLoop)) {
2810 LIOps.push_back(Ops[i]);
2811 Ops.erase(Ops.begin()+i);
2812 --i; --e;
2813 }
2814
2815 // If we found some loop invariants, fold them into the recurrence.
2816 if (!LIOps.empty()) {
2817 // Compute nowrap flags for the addition of the loop-invariant ops and
2818 // the addrec. Temporarily push it as an operand for that purpose. These
2819 // flags are valid in the scope of the addrec only.
2820 LIOps.push_back(AddRec);
2821 SCEV::NoWrapFlags Flags = ComputeFlags(LIOps);
2822 LIOps.pop_back();
2823
2824 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step}
2825 LIOps.push_back(AddRec->getStart());
2826
2827 SmallVector<const SCEV *, 4> AddRecOps(AddRec->operands());
2828
2829 // It is not in general safe to propagate flags valid on an add within
2830 // the addrec scope to one outside it. We must prove that the inner
2831 // scope is guaranteed to execute if the outer one does to be able to
2832 // safely propagate. We know the program is undefined if poison is
2833 // produced on the inner scoped addrec. We also know that *for this use*
2834 // the outer scoped add can't overflow (because of the flags we just
2835 // computed for the inner scoped add) without the program being undefined.
2836 // Proving that entry to the outer scope neccesitates entry to the inner
2837 // scope, thus proves the program undefined if the flags would be violated
2838 // in the outer scope.
2839 SCEV::NoWrapFlags AddFlags = Flags;
2840 if (AddFlags != SCEV::FlagAnyWrap) {
2841 auto *DefI = getDefiningScopeBound(LIOps);
2842 auto *ReachI = &*AddRecLoop->getHeader()->begin();
2843 if (!isGuaranteedToTransferExecutionTo(DefI, ReachI))
2844 AddFlags = SCEV::FlagAnyWrap;
2845 }
2846 AddRecOps[0] = getAddExpr(LIOps, AddFlags, Depth + 1);
2847
2848 // Build the new addrec. Propagate the NUW and NSW flags if both the
2849 // outer add and the inner addrec are guaranteed to have no overflow.
2850 // Always propagate NW.
2851 Flags = AddRec->getNoWrapFlags(setFlags(Flags, SCEV::FlagNW));
2852 const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, Flags);
2853
2854 // If all of the other operands were loop invariant, we are done.
2855 if (Ops.size() == 1) return NewRec;
2856
2857 // Otherwise, add the folded AddRec by the non-invariant parts.
2858 for (unsigned i = 0;; ++i)
2859 if (Ops[i] == AddRec) {
2860 Ops[i] = NewRec;
2861 break;
2862 }
2863 return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
2864 }
2865
2866 // Okay, if there weren't any loop invariants to be folded, check to see if
2867 // there are multiple AddRec's with the same loop induction variable being
2868 // added together. If so, we can fold them.
2869 for (unsigned OtherIdx = Idx+1;
2870 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
2871 ++OtherIdx) {
2872 // We expect the AddRecExpr's to be sorted in reverse dominance order,
2873 // so that the 1st found AddRecExpr is dominated by all others.
2874 assert(DT.dominates((static_cast <bool> (DT.dominates( cast<SCEVAddRecExpr
>(Ops[OtherIdx])->getLoop()->getHeader(), AddRec->
getLoop()->getHeader()) && "AddRecExprs are not sorted in reverse dominance order?"
) ? void (0) : __assert_fail ("DT.dominates( cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(), AddRec->getLoop()->getHeader()) && \"AddRecExprs are not sorted in reverse dominance order?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2877, __extension__
__PRETTY_FUNCTION__))
2875 cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(),(static_cast <bool> (DT.dominates( cast<SCEVAddRecExpr
>(Ops[OtherIdx])->getLoop()->getHeader(), AddRec->
getLoop()->getHeader()) && "AddRecExprs are not sorted in reverse dominance order?"
) ? void (0) : __assert_fail ("DT.dominates( cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(), AddRec->getLoop()->getHeader()) && \"AddRecExprs are not sorted in reverse dominance order?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2877, __extension__
__PRETTY_FUNCTION__))
2876 AddRec->getLoop()->getHeader()) &&(static_cast <bool> (DT.dominates( cast<SCEVAddRecExpr
>(Ops[OtherIdx])->getLoop()->getHeader(), AddRec->
getLoop()->getHeader()) && "AddRecExprs are not sorted in reverse dominance order?"
) ? void (0) : __assert_fail ("DT.dominates( cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(), AddRec->getLoop()->getHeader()) && \"AddRecExprs are not sorted in reverse dominance order?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2877, __extension__
__PRETTY_FUNCTION__))
2877 "AddRecExprs are not sorted in reverse dominance order?")(static_cast <bool> (DT.dominates( cast<SCEVAddRecExpr
>(Ops[OtherIdx])->getLoop()->getHeader(), AddRec->
getLoop()->getHeader()) && "AddRecExprs are not sorted in reverse dominance order?"
) ? void (0) : __assert_fail ("DT.dominates( cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(), AddRec->getLoop()->getHeader()) && \"AddRecExprs are not sorted in reverse dominance order?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 2877, __extension__
__PRETTY_FUNCTION__))
;
2878 if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) {
2879 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L>
2880 SmallVector<const SCEV *, 4> AddRecOps(AddRec->operands());
2881 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
2882 ++OtherIdx) {
2883 const auto *OtherAddRec = cast<SCEVAddRecExpr>(Ops[OtherIdx]);
2884 if (OtherAddRec->getLoop() == AddRecLoop) {
2885 for (unsigned i = 0, e = OtherAddRec->getNumOperands();
2886 i != e; ++i) {
2887 if (i >= AddRecOps.size()) {
2888 AddRecOps.append(OtherAddRec->op_begin()+i,
2889 OtherAddRec->op_end());
2890 break;
2891 }
2892 SmallVector<const SCEV *, 2> TwoOps = {
2893 AddRecOps[i], OtherAddRec->getOperand(i)};
2894 AddRecOps[i] = getAddExpr(TwoOps, SCEV::FlagAnyWrap, Depth + 1);
2895 }
2896 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
2897 }
2898 }
2899 // Step size has changed, so we cannot guarantee no self-wraparound.
2900 Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagAnyWrap);
2901 return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
2902 }
2903 }
2904
2905 // Otherwise couldn't fold anything into this recurrence. Move onto the
2906 // next one.
2907 }
2908
2909 // Okay, it looks like we really DO need an add expr. Check to see if we
2910 // already have one, otherwise create a new one.
2911 return getOrCreateAddExpr(Ops, ComputeFlags(Ops));
2912}
2913
2914const SCEV *
2915ScalarEvolution::getOrCreateAddExpr(ArrayRef<const SCEV *> Ops,
2916 SCEV::NoWrapFlags Flags) {
2917 FoldingSetNodeID ID;
2918 ID.AddInteger(scAddExpr);
2919 for (const SCEV *Op : Ops)
2920 ID.AddPointer(Op);
2921 void *IP = nullptr;
2922 SCEVAddExpr *S =
2923 static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
2924 if (!S) {
2925 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
2926 std::uninitialized_copy(Ops.begin(), Ops.end(), O);
2927 S = new (SCEVAllocator)
2928 SCEVAddExpr(ID.Intern(SCEVAllocator), O, Ops.size());
2929 UniqueSCEVs.InsertNode(S, IP);
2930 registerUser(S, Ops);
2931 }
2932 S->setNoWrapFlags(Flags);
2933 return S;
2934}
2935
2936const SCEV *
2937ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<const SCEV *> Ops,
2938 const Loop *L, SCEV::NoWrapFlags Flags) {
2939 FoldingSetNodeID ID;
2940 ID.AddInteger(scAddRecExpr);
2941 for (const SCEV *Op : Ops)
2942 ID.AddPointer(Op);
2943 ID.AddPointer(L);
2944 void *IP = nullptr;
2945 SCEVAddRecExpr *S =
2946 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
2947 if (!S) {
2948 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
2949 std::uninitialized_copy(Ops.begin(), Ops.end(), O);
2950 S = new (SCEVAllocator)
2951 SCEVAddRecExpr(ID.Intern(SCEVAllocator), O, Ops.size(), L);
2952 UniqueSCEVs.InsertNode(S, IP);
2953 LoopUsers[L].push_back(S);
2954 registerUser(S, Ops);
2955 }
2956 setNoWrapFlags(S, Flags);
2957 return S;
2958}
2959
2960const SCEV *
2961ScalarEvolution::getOrCreateMulExpr(ArrayRef<const SCEV *> Ops,
2962 SCEV::NoWrapFlags Flags) {
2963 FoldingSetNodeID ID;
2964 ID.AddInteger(scMulExpr);
2965 for (const SCEV *Op : Ops)
2966 ID.AddPointer(Op);
2967 void *IP = nullptr;
2968 SCEVMulExpr *S =
2969 static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
2970 if (!S) {
2971 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
2972 std::uninitialized_copy(Ops.begin(), Ops.end(), O);
2973 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator),
2974 O, Ops.size());
2975 UniqueSCEVs.InsertNode(S, IP);
2976 registerUser(S, Ops);
2977 }
2978 S->setNoWrapFlags(Flags);
2979 return S;
2980}
2981
2982static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
2983 uint64_t k = i*j;
2984 if (j > 1 && k / j != i) Overflow = true;
2985 return k;
2986}
2987
2988/// Compute the result of "n choose k", the binomial coefficient. If an
2989/// intermediate computation overflows, Overflow will be set and the return will
2990/// be garbage. Overflow is not cleared on absence of overflow.
2991static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
2992 // We use the multiplicative formula:
2993 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
2994 // At each iteration, we take the n-th term of the numeral and divide by the
2995 // (k-n)th term of the denominator. This division will always produce an
2996 // integral result, and helps reduce the chance of overflow in the
2997 // intermediate computations. However, we can still overflow even when the
2998 // final result would fit.
2999
3000 if (n == 0 || n == k) return 1;
3001 if (k > n) return 0;
3002
3003 if (k > n/2)
3004 k = n-k;
3005
3006 uint64_t r = 1;
3007 for (uint64_t i = 1; i <= k; ++i) {
3008 r = umul_ov(r, n-(i-1), Overflow);
3009 r /= i;
3010 }
3011 return r;
3012}
3013
3014/// Determine if any of the operands in this SCEV are a constant or if
3015/// any of the add or multiply expressions in this SCEV contain a constant.
3016static bool containsConstantInAddMulChain(const SCEV *StartExpr) {
3017 struct FindConstantInAddMulChain {
3018 bool FoundConstant = false;
3019
3020 bool follow(const SCEV *S) {
3021 FoundConstant |= isa<SCEVConstant>(S);
3022 return isa<SCEVAddExpr>(S) || isa<SCEVMulExpr>(S);
3023 }
3024
3025 bool isDone() const {
3026 return FoundConstant;
3027 }
3028 };
3029
3030 FindConstantInAddMulChain F;
3031 SCEVTraversal<FindConstantInAddMulChain> ST(F);
3032 ST.visitAll(StartExpr);
3033 return F.FoundConstant;
3034}
3035
3036/// Get a canonical multiply expression, or something simpler if possible.
3037const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
3038 SCEV::NoWrapFlags OrigFlags,
3039 unsigned Depth) {
3040 assert(OrigFlags == maskFlags(OrigFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&(static_cast <bool> (OrigFlags == maskFlags(OrigFlags, SCEV
::FlagNUW | SCEV::FlagNSW) && "only nuw or nsw allowed"
) ? void (0) : __assert_fail ("OrigFlags == maskFlags(OrigFlags, SCEV::FlagNUW | SCEV::FlagNSW) && \"only nuw or nsw allowed\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3041, __extension__
__PRETTY_FUNCTION__))
3041 "only nuw or nsw allowed")(static_cast <bool> (OrigFlags == maskFlags(OrigFlags, SCEV
::FlagNUW | SCEV::FlagNSW) && "only nuw or nsw allowed"
) ? void (0) : __assert_fail ("OrigFlags == maskFlags(OrigFlags, SCEV::FlagNUW | SCEV::FlagNSW) && \"only nuw or nsw allowed\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3041, __extension__
__PRETTY_FUNCTION__))
;
3042 assert(!Ops.empty() && "Cannot get empty mul!")(static_cast <bool> (!Ops.empty() && "Cannot get empty mul!"
) ? void (0) : __assert_fail ("!Ops.empty() && \"Cannot get empty mul!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3042, __extension__
__PRETTY_FUNCTION__))
;
3043 if (Ops.size() == 1) return Ops[0];
3044#ifndef NDEBUG
3045 Type *ETy = Ops[0]->getType();
3046 assert(!ETy->isPointerTy())(static_cast <bool> (!ETy->isPointerTy()) ? void (0)
: __assert_fail ("!ETy->isPointerTy()", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 3046, __extension__ __PRETTY_FUNCTION__))
;
3047 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
3048 assert(Ops[i]->getType() == ETy &&(static_cast <bool> (Ops[i]->getType() == ETy &&
"SCEVMulExpr operand types don't match!") ? void (0) : __assert_fail
("Ops[i]->getType() == ETy && \"SCEVMulExpr operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3049, __extension__
__PRETTY_FUNCTION__))
3049 "SCEVMulExpr operand types don't match!")(static_cast <bool> (Ops[i]->getType() == ETy &&
"SCEVMulExpr operand types don't match!") ? void (0) : __assert_fail
("Ops[i]->getType() == ETy && \"SCEVMulExpr operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3049, __extension__
__PRETTY_FUNCTION__))
;
3050#endif
3051
3052 // Sort by complexity, this groups all similar expression types together.
3053 GroupByComplexity(Ops, &LI, DT);
3054
3055 // If there are any constants, fold them together.
3056 unsigned Idx = 0;
3057 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
3058 ++Idx;
3059 assert(Idx < Ops.size())(static_cast <bool> (Idx < Ops.size()) ? void (0) : __assert_fail
("Idx < Ops.size()", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 3059, __extension__ __PRETTY_FUNCTION__))
;
3060 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
3061 // We found two constants, fold them together!
3062 Ops[0] = getConstant(LHSC->getAPInt() * RHSC->getAPInt());
3063 if (Ops.size() == 2) return Ops[0];
3064 Ops.erase(Ops.begin()+1); // Erase the folded element
3065 LHSC = cast<SCEVConstant>(Ops[0]);
3066 }
3067
3068 // If we have a multiply of zero, it will always be zero.
3069 if (LHSC->getValue()->isZero())
3070 return LHSC;
3071
3072 // If we are left with a constant one being multiplied, strip it off.
3073 if (LHSC->getValue()->isOne()) {
3074 Ops.erase(Ops.begin());
3075 --Idx;
3076 }
3077
3078 if (Ops.size() == 1)
3079 return Ops[0];
3080 }
3081
3082 // Delay expensive flag strengthening until necessary.
3083 auto ComputeFlags = [this, OrigFlags](const ArrayRef<const SCEV *> Ops) {
3084 return StrengthenNoWrapFlags(this, scMulExpr, Ops, OrigFlags);
3085 };
3086
3087 // Limit recursion calls depth.
3088 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
3089 return getOrCreateMulExpr(Ops, ComputeFlags(Ops));
3090
3091 if (SCEV *S = findExistingSCEVInCache(scMulExpr, Ops)) {
3092 // Don't strengthen flags if we have no new information.
3093 SCEVMulExpr *Mul = static_cast<SCEVMulExpr *>(S);
3094 if (Mul->getNoWrapFlags(OrigFlags) != OrigFlags)
3095 Mul->setNoWrapFlags(ComputeFlags(Ops));
3096 return S;
3097 }
3098
3099 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
3100 if (Ops.size() == 2) {
3101 // C1*(C2+V) -> C1*C2 + C1*V
3102 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1]))
3103 // If any of Add's ops are Adds or Muls with a constant, apply this
3104 // transformation as well.
3105 //
3106 // TODO: There are some cases where this transformation is not
3107 // profitable; for example, Add = (C0 + X) * Y + Z. Maybe the scope of
3108 // this transformation should be narrowed down.
3109 if (Add->getNumOperands() == 2 && containsConstantInAddMulChain(Add))
3110 return getAddExpr(getMulExpr(LHSC, Add->getOperand(0),
3111 SCEV::FlagAnyWrap, Depth + 1),
3112 getMulExpr(LHSC, Add->getOperand(1),
3113 SCEV::FlagAnyWrap, Depth + 1),
3114 SCEV::FlagAnyWrap, Depth + 1);
3115
3116 if (Ops[0]->isAllOnesValue()) {
3117 // If we have a mul by -1 of an add, try distributing the -1 among the
3118 // add operands.
3119 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) {
3120 SmallVector<const SCEV *, 4> NewOps;
3121 bool AnyFolded = false;
3122 for (const SCEV *AddOp : Add->operands()) {
3123 const SCEV *Mul = getMulExpr(Ops[0], AddOp, SCEV::FlagAnyWrap,
3124 Depth + 1);
3125 if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true;
3126 NewOps.push_back(Mul);
3127 }
3128 if (AnyFolded)
3129 return getAddExpr(NewOps, SCEV::FlagAnyWrap, Depth + 1);
3130 } else if (const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Ops[1])) {
3131 // Negation preserves a recurrence's no self-wrap property.
3132 SmallVector<const SCEV *, 4> Operands;
3133 for (const SCEV *AddRecOp : AddRec->operands())
3134 Operands.push_back(getMulExpr(Ops[0], AddRecOp, SCEV::FlagAnyWrap,
3135 Depth + 1));
3136
3137 return getAddRecExpr(Operands, AddRec->getLoop(),
3138 AddRec->getNoWrapFlags(SCEV::FlagNW));
3139 }
3140 }
3141 }
3142 }
3143
3144 // Skip over the add expression until we get to a multiply.
3145 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
3146 ++Idx;
3147
3148 // If there are mul operands inline them all into this expression.
3149 if (Idx < Ops.size()) {
3150 bool DeletedMul = false;
3151 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
3152 if (Ops.size() > MulOpsInlineThreshold)
3153 break;
3154 // If we have an mul, expand the mul operands onto the end of the
3155 // operands list.
3156 Ops.erase(Ops.begin()+Idx);
3157 Ops.append(Mul->op_begin(), Mul->op_end());
3158 DeletedMul = true;
3159 }
3160
3161 // If we deleted at least one mul, we added operands to the end of the
3162 // list, and they are not necessarily sorted. Recurse to resort and
3163 // resimplify any operands we just acquired.
3164 if (DeletedMul)
3165 return getMulExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
3166 }
3167
3168 // If there are any add recurrences in the operands list, see if any other
3169 // added values are loop invariant. If so, we can fold them into the
3170 // recurrence.
3171 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
3172 ++Idx;
3173
3174 // Scan over all recurrences, trying to fold loop invariants into them.
3175 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
3176 // Scan all of the other operands to this mul and add them to the vector
3177 // if they are loop invariant w.r.t. the recurrence.
3178 SmallVector<const SCEV *, 8> LIOps;
3179 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
3180 const Loop *AddRecLoop = AddRec->getLoop();
3181 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
3182 if (isAvailableAtLoopEntry(Ops[i], AddRecLoop)) {
3183 LIOps.push_back(Ops[i]);
3184 Ops.erase(Ops.begin()+i);
3185 --i; --e;
3186 }
3187
3188 // If we found some loop invariants, fold them into the recurrence.
3189 if (!LIOps.empty()) {
3190 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step}
3191 SmallVector<const SCEV *, 4> NewOps;
3192 NewOps.reserve(AddRec->getNumOperands());
3193 const SCEV *Scale = getMulExpr(LIOps, SCEV::FlagAnyWrap, Depth + 1);
3194 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i)
3195 NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i),
3196 SCEV::FlagAnyWrap, Depth + 1));
3197
3198 // Build the new addrec. Propagate the NUW and NSW flags if both the
3199 // outer mul and the inner addrec are guaranteed to have no overflow.
3200 //
3201 // No self-wrap cannot be guaranteed after changing the step size, but
3202 // will be inferred if either NUW or NSW is true.
3203 SCEV::NoWrapFlags Flags = ComputeFlags({Scale, AddRec});
3204 const SCEV *NewRec = getAddRecExpr(
3205 NewOps, AddRecLoop, AddRec->getNoWrapFlags(Flags));
3206
3207 // If all of the other operands were loop invariant, we are done.
3208 if (Ops.size() == 1) return NewRec;
3209
3210 // Otherwise, multiply the folded AddRec by the non-invariant parts.
3211 for (unsigned i = 0;; ++i)
3212 if (Ops[i] == AddRec) {
3213 Ops[i] = NewRec;
3214 break;
3215 }
3216 return getMulExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
3217 }
3218
3219 // Okay, if there weren't any loop invariants to be folded, check to see
3220 // if there are multiple AddRec's with the same loop induction variable
3221 // being multiplied together. If so, we can fold them.
3222
3223 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
3224 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
3225 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
3226 // ]]],+,...up to x=2n}.
3227 // Note that the arguments to choose() are always integers with values
3228 // known at compile time, never SCEV objects.
3229 //
3230 // The implementation avoids pointless extra computations when the two
3231 // addrec's are of different length (mathematically, it's equivalent to
3232 // an infinite stream of zeros on the right).
3233 bool OpsModified = false;
3234 for (unsigned OtherIdx = Idx+1;
3235 OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
3236 ++OtherIdx) {
3237 const SCEVAddRecExpr *OtherAddRec =
3238 dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]);
3239 if (!OtherAddRec || OtherAddRec->getLoop() != AddRecLoop)
3240 continue;
3241
3242 // Limit max number of arguments to avoid creation of unreasonably big
3243 // SCEVAddRecs with very complex operands.
3244 if (AddRec->getNumOperands() + OtherAddRec->getNumOperands() - 1 >
3245 MaxAddRecSize || hasHugeExpression({AddRec, OtherAddRec}))
3246 continue;
3247
3248 bool Overflow = false;
3249 Type *Ty = AddRec->getType();
3250 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
3251 SmallVector<const SCEV*, 7> AddRecOps;
3252 for (int x = 0, xe = AddRec->getNumOperands() +
3253 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
3254 SmallVector <const SCEV *, 7> SumOps;
3255 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
3256 uint64_t Coeff1 = Choose(x, 2*x - y, Overflow);
3257 for (int z = std::max(y-x, y-(int)AddRec->getNumOperands()+1),
3258 ze = std::min(x+1, (int)OtherAddRec->getNumOperands());
3259 z < ze && !Overflow; ++z) {
3260 uint64_t Coeff2 = Choose(2*x - y, x-z, Overflow);
3261 uint64_t Coeff;
3262 if (LargerThan64Bits)
3263 Coeff = umul_ov(Coeff1, Coeff2, Overflow);
3264 else
3265 Coeff = Coeff1*Coeff2;
3266 const SCEV *CoeffTerm = getConstant(Ty, Coeff);
3267 const SCEV *Term1 = AddRec->getOperand(y-z);
3268 const SCEV *Term2 = OtherAddRec->getOperand(z);
3269 SumOps.push_back(getMulExpr(CoeffTerm, Term1, Term2,
3270 SCEV::FlagAnyWrap, Depth + 1));
3271 }
3272 }
3273 if (SumOps.empty())
3274 SumOps.push_back(getZero(Ty));
3275 AddRecOps.push_back(getAddExpr(SumOps, SCEV::FlagAnyWrap, Depth + 1));
3276 }
3277 if (!Overflow) {
3278 const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRecLoop,
3279 SCEV::FlagAnyWrap);
3280 if (Ops.size() == 2) return NewAddRec;
3281 Ops[Idx] = NewAddRec;
3282 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
3283 OpsModified = true;
3284 AddRec = dyn_cast<SCEVAddRecExpr>(NewAddRec);
3285 if (!AddRec)
3286 break;
3287 }
3288 }
3289 if (OpsModified)
3290 return getMulExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
3291
3292 // Otherwise couldn't fold anything into this recurrence. Move onto the
3293 // next one.
3294 }
3295
3296 // Okay, it looks like we really DO need an mul expr. Check to see if we
3297 // already have one, otherwise create a new one.
3298 return getOrCreateMulExpr(Ops, ComputeFlags(Ops));
3299}
3300
3301/// Represents an unsigned remainder expression based on unsigned division.
3302const SCEV *ScalarEvolution::getURemExpr(const SCEV *LHS,
3303 const SCEV *RHS) {
3304 assert(getEffectiveSCEVType(LHS->getType()) ==(static_cast <bool> (getEffectiveSCEVType(LHS->getType
()) == getEffectiveSCEVType(RHS->getType()) && "SCEVURemExpr operand types don't match!"
) ? void (0) : __assert_fail ("getEffectiveSCEVType(LHS->getType()) == getEffectiveSCEVType(RHS->getType()) && \"SCEVURemExpr operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3306, __extension__
__PRETTY_FUNCTION__))
3305 getEffectiveSCEVType(RHS->getType()) &&(static_cast <bool> (getEffectiveSCEVType(LHS->getType
()) == getEffectiveSCEVType(RHS->getType()) && "SCEVURemExpr operand types don't match!"
) ? void (0) : __assert_fail ("getEffectiveSCEVType(LHS->getType()) == getEffectiveSCEVType(RHS->getType()) && \"SCEVURemExpr operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3306, __extension__
__PRETTY_FUNCTION__))
3306 "SCEVURemExpr operand types don't match!")(static_cast <bool> (getEffectiveSCEVType(LHS->getType
()) == getEffectiveSCEVType(RHS->getType()) && "SCEVURemExpr operand types don't match!"
) ? void (0) : __assert_fail ("getEffectiveSCEVType(LHS->getType()) == getEffectiveSCEVType(RHS->getType()) && \"SCEVURemExpr operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3306, __extension__
__PRETTY_FUNCTION__))
;
3307
3308 // Short-circuit easy cases
3309 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
3310 // If constant is one, the result is trivial
3311 if (RHSC->getValue()->isOne())
3312 return getZero(LHS->getType()); // X urem 1 --> 0
3313
3314 // If constant is a power of two, fold into a zext(trunc(LHS)).
3315 if (RHSC->getAPInt().isPowerOf2()) {
3316 Type *FullTy = LHS->getType();
3317 Type *TruncTy =
3318 IntegerType::get(getContext(), RHSC->getAPInt().logBase2());
3319 return getZeroExtendExpr(getTruncateExpr(LHS, TruncTy), FullTy);
3320 }
3321 }
3322
3323 // Fallback to %a == %x urem %y == %x -<nuw> ((%x udiv %y) *<nuw> %y)
3324 const SCEV *UDiv = getUDivExpr(LHS, RHS);
3325 const SCEV *Mult = getMulExpr(UDiv, RHS, SCEV::FlagNUW);
3326 return getMinusSCEV(LHS, Mult, SCEV::FlagNUW);
3327}
3328
3329/// Get a canonical unsigned division expression, or something simpler if
3330/// possible.
3331const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS,
3332 const SCEV *RHS) {
3333 assert(!LHS->getType()->isPointerTy() &&(static_cast <bool> (!LHS->getType()->isPointerTy
() && "SCEVUDivExpr operand can't be pointer!") ? void
(0) : __assert_fail ("!LHS->getType()->isPointerTy() && \"SCEVUDivExpr operand can't be pointer!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3334, __extension__
__PRETTY_FUNCTION__))
3334 "SCEVUDivExpr operand can't be pointer!")(static_cast <bool> (!LHS->getType()->isPointerTy
() && "SCEVUDivExpr operand can't be pointer!") ? void
(0) : __assert_fail ("!LHS->getType()->isPointerTy() && \"SCEVUDivExpr operand can't be pointer!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3334, __extension__
__PRETTY_FUNCTION__))
;
3335 assert(LHS->getType() == RHS->getType() &&(static_cast <bool> (LHS->getType() == RHS->getType
() && "SCEVUDivExpr operand types don't match!") ? void
(0) : __assert_fail ("LHS->getType() == RHS->getType() && \"SCEVUDivExpr operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3336, __extension__
__PRETTY_FUNCTION__))
3336 "SCEVUDivExpr operand types don't match!")(static_cast <bool> (LHS->getType() == RHS->getType
() && "SCEVUDivExpr operand types don't match!") ? void
(0) : __assert_fail ("LHS->getType() == RHS->getType() && \"SCEVUDivExpr operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3336, __extension__
__PRETTY_FUNCTION__))
;
3337
3338 FoldingSetNodeID ID;
3339 ID.AddInteger(scUDivExpr);
3340 ID.AddPointer(LHS);
3341 ID.AddPointer(RHS);
3342 void *IP = nullptr;
3343 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
3344 return S;
3345
3346 // 0 udiv Y == 0
3347 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS))
3348 if (LHSC->getValue()->isZero())
3349 return LHS;
3350
3351 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
3352 if (RHSC->getValue()->isOne())
3353 return LHS; // X udiv 1 --> x
3354 // If the denominator is zero, the result of the udiv is undefined. Don't
3355 // try to analyze it, because the resolution chosen here may differ from
3356 // the resolution chosen in other parts of the compiler.
3357 if (!RHSC->getValue()->isZero()) {
3358 // Determine if the division can be folded into the operands of
3359 // its operands.
3360 // TODO: Generalize this to non-constants by using known-bits information.
3361 Type *Ty = LHS->getType();
3362 unsigned LZ = RHSC->getAPInt().countLeadingZeros();
3363 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
3364 // For non-power-of-two values, effectively round the value up to the
3365 // nearest power of two.
3366 if (!RHSC->getAPInt().isPowerOf2())
3367 ++MaxShiftAmt;
3368 IntegerType *ExtTy =
3369 IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt);
3370 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS))
3371 if (const SCEVConstant *Step =
3372 dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) {
3373 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
3374 const APInt &StepInt = Step->getAPInt();
3375 const APInt &DivInt = RHSC->getAPInt();
3376 if (!StepInt.urem(DivInt) &&
3377 getZeroExtendExpr(AR, ExtTy) ==
3378 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
3379 getZeroExtendExpr(Step, ExtTy),
3380 AR->getLoop(), SCEV::FlagAnyWrap)) {
3381 SmallVector<const SCEV *, 4> Operands;
3382 for (const SCEV *Op : AR->operands())
3383 Operands.push_back(getUDivExpr(Op, RHS));
3384 return getAddRecExpr(Operands, AR->getLoop(), SCEV::FlagNW);
3385 }
3386 /// Get a canonical UDivExpr for a recurrence.
3387 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
3388 // We can currently only fold X%N if X is constant.
3389 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(AR->getStart());
3390 if (StartC && !DivInt.urem(StepInt) &&
3391 getZeroExtendExpr(AR, ExtTy) ==
3392 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
3393 getZeroExtendExpr(Step, ExtTy),
3394 AR->getLoop(), SCEV::FlagAnyWrap)) {
3395 const APInt &StartInt = StartC->getAPInt();
3396 const APInt &StartRem = StartInt.urem(StepInt);
3397 if (StartRem != 0) {
3398 const SCEV *NewLHS =
3399 getAddRecExpr(getConstant(StartInt - StartRem), Step,
3400 AR->getLoop(), SCEV::FlagNW);
3401 if (LHS != NewLHS) {
3402 LHS = NewLHS;
3403
3404 // Reset the ID to include the new LHS, and check if it is
3405 // already cached.
3406 ID.clear();
3407 ID.AddInteger(scUDivExpr);
3408 ID.AddPointer(LHS);
3409 ID.AddPointer(RHS);
3410 IP = nullptr;
3411 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
3412 return S;
3413 }
3414 }
3415 }
3416 }
3417 // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
3418 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) {
3419 SmallVector<const SCEV *, 4> Operands;
3420 for (const SCEV *Op : M->operands())
3421 Operands.push_back(getZeroExtendExpr(Op, ExtTy));
3422 if (getZeroExtendExpr(M, ExtTy) == getMulExpr(Operands))
3423 // Find an operand that's safely divisible.
3424 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
3425 const SCEV *Op = M->getOperand(i);
3426 const SCEV *Div = getUDivExpr(Op, RHSC);
3427 if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) {
3428 Operands = SmallVector<const SCEV *, 4>(M->operands());
3429 Operands[i] = Div;
3430 return getMulExpr(Operands);
3431 }
3432 }
3433 }
3434
3435 // (A/B)/C --> A/(B*C) if safe and B*C can be folded.
3436 if (const SCEVUDivExpr *OtherDiv = dyn_cast<SCEVUDivExpr>(LHS)) {
3437 if (auto *DivisorConstant =
3438 dyn_cast<SCEVConstant>(OtherDiv->getRHS())) {
3439 bool Overflow = false;
3440 APInt NewRHS =
3441 DivisorConstant->getAPInt().umul_ov(RHSC->getAPInt(), Overflow);
3442 if (Overflow) {
3443 return getConstant(RHSC->getType(), 0, false);
3444 }
3445 return getUDivExpr(OtherDiv->getLHS(), getConstant(NewRHS));
3446 }
3447 }
3448
3449 // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded.
3450 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) {
3451 SmallVector<const SCEV *, 4> Operands;
3452 for (const SCEV *Op : A->operands())
3453 Operands.push_back(getZeroExtendExpr(Op, ExtTy));
3454 if (getZeroExtendExpr(A, ExtTy) == getAddExpr(Operands)) {
3455 Operands.clear();
3456 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
3457 const SCEV *Op = getUDivExpr(A->getOperand(i), RHS);
3458 if (isa<SCEVUDivExpr>(Op) ||
3459 getMulExpr(Op, RHS) != A->getOperand(i))
3460 break;
3461 Operands.push_back(Op);
3462 }
3463 if (Operands.size() == A->getNumOperands())
3464 return getAddExpr(Operands);
3465 }
3466 }
3467
3468 // Fold if both operands are constant.
3469 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
3470 Constant *LHSCV = LHSC->getValue();
3471 Constant *RHSCV = RHSC->getValue();
3472 return getConstant(cast<ConstantInt>(ConstantExpr::getUDiv(LHSCV,
3473 RHSCV)));
3474 }
3475 }
3476 }
3477
3478 // The Insertion Point (IP) might be invalid by now (due to UniqueSCEVs
3479 // changes). Make sure we get a new one.
3480 IP = nullptr;
3481 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
3482 SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator),
3483 LHS, RHS);
3484 UniqueSCEVs.InsertNode(S, IP);
3485 registerUser(S, {LHS, RHS});
3486 return S;
3487}
3488
3489static const APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2) {
3490 APInt A = C1->getAPInt().abs();
3491 APInt B = C2->getAPInt().abs();
3492 uint32_t ABW = A.getBitWidth();
3493 uint32_t BBW = B.getBitWidth();
3494
3495 if (ABW > BBW)
3496 B = B.zext(ABW);
3497 else if (ABW < BBW)
3498 A = A.zext(BBW);
3499
3500 return APIntOps::GreatestCommonDivisor(std::move(A), std::move(B));
3501}
3502
3503/// Get a canonical unsigned division expression, or something simpler if
3504/// possible. There is no representation for an exact udiv in SCEV IR, but we
3505/// can attempt to remove factors from the LHS and RHS. We can't do this when
3506/// it's not exact because the udiv may be clearing bits.
3507const SCEV *ScalarEvolution::getUDivExactExpr(const SCEV *LHS,
3508 const SCEV *RHS) {
3509 // TODO: we could try to find factors in all sorts of things, but for now we
3510 // just deal with u/exact (multiply, constant). See SCEVDivision towards the
3511 // end of this file for inspiration.
3512
3513 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS);
3514 if (!Mul || !Mul->hasNoUnsignedWrap())
3515 return getUDivExpr(LHS, RHS);
3516
3517 if (const SCEVConstant *RHSCst = dyn_cast<SCEVConstant>(RHS)) {
3518 // If the mulexpr multiplies by a constant, then that constant must be the
3519 // first element of the mulexpr.
3520 if (const auto *LHSCst = dyn_cast<SCEVConstant>(Mul->getOperand(0))) {
3521 if (LHSCst == RHSCst) {
3522 SmallVector<const SCEV *, 2> Operands(drop_begin(Mul->operands()));
3523 return getMulExpr(Operands);
3524 }
3525
3526 // We can't just assume that LHSCst divides RHSCst cleanly, it could be
3527 // that there's a factor provided by one of the other terms. We need to
3528 // check.
3529 APInt Factor = gcd(LHSCst, RHSCst);
3530 if (!Factor.isIntN(1)) {
3531 LHSCst =
3532 cast<SCEVConstant>(getConstant(LHSCst->getAPInt().udiv(Factor)));
3533 RHSCst =
3534 cast<SCEVConstant>(getConstant(RHSCst->getAPInt().udiv(Factor)));
3535 SmallVector<const SCEV *, 2> Operands;
3536 Operands.push_back(LHSCst);
3537 Operands.append(Mul->op_begin() + 1, Mul->op_end());
3538 LHS = getMulExpr(Operands);
3539 RHS = RHSCst;
3540 Mul = dyn_cast<SCEVMulExpr>(LHS);
3541 if (!Mul)
3542 return getUDivExactExpr(LHS, RHS);
3543 }
3544 }
3545 }
3546
3547 for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
3548 if (Mul->getOperand(i) == RHS) {
3549 SmallVector<const SCEV *, 2> Operands;
3550 Operands.append(Mul->op_begin(), Mul->op_begin() + i);
3551 Operands.append(Mul->op_begin() + i + 1, Mul->op_end());
3552 return getMulExpr(Operands);
3553 }
3554 }
3555
3556 return getUDivExpr(LHS, RHS);
3557}
3558
3559/// Get an add recurrence expression for the specified loop. Simplify the
3560/// expression as much as possible.
3561const SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start, const SCEV *Step,
3562 const Loop *L,
3563 SCEV::NoWrapFlags Flags) {
3564 SmallVector<const SCEV *, 4> Operands;
3565 Operands.push_back(Start);
3566 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step))
3567 if (StepChrec->getLoop() == L) {
3568 Operands.append(StepChrec->op_begin(), StepChrec->op_end());
3569 return getAddRecExpr(Operands, L, maskFlags(Flags, SCEV::FlagNW));
3570 }
3571
3572 Operands.push_back(Step);
3573 return getAddRecExpr(Operands, L, Flags);
3574}
3575
3576/// Get an add recurrence expression for the specified loop. Simplify the
3577/// expression as much as possible.
3578const SCEV *
3579ScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands,
3580 const Loop *L, SCEV::NoWrapFlags Flags) {
3581 if (Operands.size() == 1) return Operands[0];
3582#ifndef NDEBUG
3583 Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
3584 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3585 assert(getEffectiveSCEVType(Operands[i]->getType()) == ETy &&(static_cast <bool> (getEffectiveSCEVType(Operands[i]->
getType()) == ETy && "SCEVAddRecExpr operand types don't match!"
) ? void (0) : __assert_fail ("getEffectiveSCEVType(Operands[i]->getType()) == ETy && \"SCEVAddRecExpr operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3586, __extension__
__PRETTY_FUNCTION__))
3586 "SCEVAddRecExpr operand types don't match!")(static_cast <bool> (getEffectiveSCEVType(Operands[i]->
getType()) == ETy && "SCEVAddRecExpr operand types don't match!"
) ? void (0) : __assert_fail ("getEffectiveSCEVType(Operands[i]->getType()) == ETy && \"SCEVAddRecExpr operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3586, __extension__
__PRETTY_FUNCTION__))
;
3587 assert(!Operands[i]->getType()->isPointerTy() && "Step must be integer")(static_cast <bool> (!Operands[i]->getType()->isPointerTy
() && "Step must be integer") ? void (0) : __assert_fail
("!Operands[i]->getType()->isPointerTy() && \"Step must be integer\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3587, __extension__
__PRETTY_FUNCTION__))
;
3588 }
3589 for (unsigned i = 0, e = Operands.size(); i != e; ++i)
3590 assert(isLoopInvariant(Operands[i], L) &&(static_cast <bool> (isLoopInvariant(Operands[i], L) &&
"SCEVAddRecExpr operand is not loop-invariant!") ? void (0) :
__assert_fail ("isLoopInvariant(Operands[i], L) && \"SCEVAddRecExpr operand is not loop-invariant!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3591, __extension__
__PRETTY_FUNCTION__))
3591 "SCEVAddRecExpr operand is not loop-invariant!")(static_cast <bool> (isLoopInvariant(Operands[i], L) &&
"SCEVAddRecExpr operand is not loop-invariant!") ? void (0) :
__assert_fail ("isLoopInvariant(Operands[i], L) && \"SCEVAddRecExpr operand is not loop-invariant!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3591, __extension__
__PRETTY_FUNCTION__))
;
3592#endif
3593
3594 if (Operands.back()->isZero()) {
3595 Operands.pop_back();
3596 return getAddRecExpr(Operands, L, SCEV::FlagAnyWrap); // {X,+,0} --> X
3597 }
3598
3599 // It's tempting to want to call getConstantMaxBackedgeTakenCount count here and
3600 // use that information to infer NUW and NSW flags. However, computing a
3601 // BE count requires calling getAddRecExpr, so we may not yet have a
3602 // meaningful BE count at this point (and if we don't, we'd be stuck
3603 // with a SCEVCouldNotCompute as the cached BE count).
3604
3605 Flags = StrengthenNoWrapFlags(this, scAddRecExpr, Operands, Flags);
3606
3607 // Canonicalize nested AddRecs in by nesting them in order of loop depth.
3608 if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Operands[0])) {
3609 const Loop *NestedLoop = NestedAR->getLoop();
3610 if (L->contains(NestedLoop)
3611 ? (L->getLoopDepth() < NestedLoop->getLoopDepth())
3612 : (!NestedLoop->contains(L) &&
3613 DT.dominates(L->getHeader(), NestedLoop->getHeader()))) {
3614 SmallVector<const SCEV *, 4> NestedOperands(NestedAR->operands());
3615 Operands[0] = NestedAR->getStart();
3616 // AddRecs require their operands be loop-invariant with respect to their
3617 // loops. Don't perform this transformation if it would break this
3618 // requirement.
3619 bool AllInvariant = all_of(
3620 Operands, [&](const SCEV *Op) { return isLoopInvariant(Op, L); });
3621
3622 if (AllInvariant) {
3623 // Create a recurrence for the outer loop with the same step size.
3624 //
3625 // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the
3626 // inner recurrence has the same property.
3627 SCEV::NoWrapFlags OuterFlags =
3628 maskFlags(Flags, SCEV::FlagNW | NestedAR->getNoWrapFlags());
3629
3630 NestedOperands[0] = getAddRecExpr(Operands, L, OuterFlags);
3631 AllInvariant = all_of(NestedOperands, [&](const SCEV *Op) {
3632 return isLoopInvariant(Op, NestedLoop);
3633 });
3634
3635 if (AllInvariant) {
3636 // Ok, both add recurrences are valid after the transformation.
3637 //
3638 // The inner recurrence keeps its NW flag but only keeps NUW/NSW if
3639 // the outer recurrence has the same property.
3640 SCEV::NoWrapFlags InnerFlags =
3641 maskFlags(NestedAR->getNoWrapFlags(), SCEV::FlagNW | Flags);
3642 return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags);
3643 }
3644 }
3645 // Reset Operands to its original state.
3646 Operands[0] = NestedAR;
3647 }
3648 }
3649
3650 // Okay, it looks like we really DO need an addrec expr. Check to see if we
3651 // already have one, otherwise create a new one.
3652 return getOrCreateAddRecExpr(Operands, L, Flags);
3653}
3654
3655const SCEV *
3656ScalarEvolution::getGEPExpr(GEPOperator *GEP,
3657 const SmallVectorImpl<const SCEV *> &IndexExprs) {
3658 const SCEV *BaseExpr = getSCEV(GEP->getPointerOperand());
3659 // getSCEV(Base)->getType() has the same address space as Base->getType()
3660 // because SCEV::getType() preserves the address space.
3661 Type *IntIdxTy = getEffectiveSCEVType(BaseExpr->getType());
3662 const bool AssumeInBoundsFlags = [&]() {
3663 if (!GEP->isInBounds())
3664 return false;
3665
3666 // We'd like to propagate flags from the IR to the corresponding SCEV nodes,
3667 // but to do that, we have to ensure that said flag is valid in the entire
3668 // defined scope of the SCEV.
3669 auto *GEPI = dyn_cast<Instruction>(GEP);
3670 // TODO: non-instructions have global scope. We might be able to prove
3671 // some global scope cases
3672 return GEPI && isSCEVExprNeverPoison(GEPI);
3673 }();
3674
3675 SCEV::NoWrapFlags OffsetWrap =
3676 AssumeInBoundsFlags ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
3677
3678 Type *CurTy = GEP->getType();
3679 bool FirstIter = true;
3680 SmallVector<const SCEV *, 4> Offsets;
3681 for (const SCEV *IndexExpr : IndexExprs) {
3682 // Compute the (potentially symbolic) offset in bytes for this index.
3683 if (StructType *STy = dyn_cast<StructType>(CurTy)) {
3684 // For a struct, add the member offset.
3685 ConstantInt *Index = cast<SCEVConstant>(IndexExpr)->getValue();
3686 unsigned FieldNo = Index->getZExtValue();
3687 const SCEV *FieldOffset = getOffsetOfExpr(IntIdxTy, STy, FieldNo);
3688 Offsets.push_back(FieldOffset);
3689
3690 // Update CurTy to the type of the field at Index.
3691 CurTy = STy->getTypeAtIndex(Index);
3692 } else {
3693 // Update CurTy to its element type.
3694 if (FirstIter) {
3695 assert(isa<PointerType>(CurTy) &&(static_cast <bool> (isa<PointerType>(CurTy) &&
"The first index of a GEP indexes a pointer") ? void (0) : __assert_fail
("isa<PointerType>(CurTy) && \"The first index of a GEP indexes a pointer\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3696, __extension__
__PRETTY_FUNCTION__))
3696 "The first index of a GEP indexes a pointer")(static_cast <bool> (isa<PointerType>(CurTy) &&
"The first index of a GEP indexes a pointer") ? void (0) : __assert_fail
("isa<PointerType>(CurTy) && \"The first index of a GEP indexes a pointer\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3696, __extension__
__PRETTY_FUNCTION__))
;
3697 CurTy = GEP->getSourceElementType();
3698 FirstIter = false;
3699 } else {
3700 CurTy = GetElementPtrInst::getTypeAtIndex(CurTy, (uint64_t)0);
3701 }
3702 // For an array, add the element offset, explicitly scaled.
3703 const SCEV *ElementSize = getSizeOfExpr(IntIdxTy, CurTy);
3704 // Getelementptr indices are signed.
3705 IndexExpr = getTruncateOrSignExtend(IndexExpr, IntIdxTy);
3706
3707 // Multiply the index by the element size to compute the element offset.
3708 const SCEV *LocalOffset = getMulExpr(IndexExpr, ElementSize, OffsetWrap);
3709 Offsets.push_back(LocalOffset);
3710 }
3711 }
3712
3713 // Handle degenerate case of GEP without offsets.
3714 if (Offsets.empty())
3715 return BaseExpr;
3716
3717 // Add the offsets together, assuming nsw if inbounds.
3718 const SCEV *Offset = getAddExpr(Offsets, OffsetWrap);
3719 // Add the base address and the offset. We cannot use the nsw flag, as the
3720 // base address is unsigned. However, if we know that the offset is
3721 // non-negative, we can use nuw.
3722 SCEV::NoWrapFlags BaseWrap = AssumeInBoundsFlags && isKnownNonNegative(Offset)
3723 ? SCEV::FlagNUW : SCEV::FlagAnyWrap;
3724 auto *GEPExpr = getAddExpr(BaseExpr, Offset, BaseWrap);
3725 assert(BaseExpr->getType() == GEPExpr->getType() &&(static_cast <bool> (BaseExpr->getType() == GEPExpr->
getType() && "GEP should not change type mid-flight."
) ? void (0) : __assert_fail ("BaseExpr->getType() == GEPExpr->getType() && \"GEP should not change type mid-flight.\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3726, __extension__
__PRETTY_FUNCTION__))
3726 "GEP should not change type mid-flight.")(static_cast <bool> (BaseExpr->getType() == GEPExpr->
getType() && "GEP should not change type mid-flight."
) ? void (0) : __assert_fail ("BaseExpr->getType() == GEPExpr->getType() && \"GEP should not change type mid-flight.\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3726, __extension__
__PRETTY_FUNCTION__))
;
3727 return GEPExpr;
3728}
3729
3730SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
3731 ArrayRef<const SCEV *> Ops) {
3732 FoldingSetNodeID ID;
3733 ID.AddInteger(SCEVType);
3734 for (const SCEV *Op : Ops)
3735 ID.AddPointer(Op);
3736 void *IP = nullptr;
3737 return UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
3738}
3739
3740const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) {
3741 SCEV::NoWrapFlags Flags = IsNSW ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
3742 return getSMaxExpr(Op, getNegativeSCEV(Op, Flags));
3743}
3744
3745const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
3746 SmallVectorImpl<const SCEV *> &Ops) {
3747 assert(SCEVMinMaxExpr::isMinMaxType(Kind) && "Not a SCEVMinMaxExpr!")(static_cast <bool> (SCEVMinMaxExpr::isMinMaxType(Kind)
&& "Not a SCEVMinMaxExpr!") ? void (0) : __assert_fail
("SCEVMinMaxExpr::isMinMaxType(Kind) && \"Not a SCEVMinMaxExpr!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3747, __extension__
__PRETTY_FUNCTION__))
;
3748 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!")(static_cast <bool> (!Ops.empty() && "Cannot get empty (u|s)(min|max)!"
) ? void (0) : __assert_fail ("!Ops.empty() && \"Cannot get empty (u|s)(min|max)!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3748, __extension__
__PRETTY_FUNCTION__))
;
3749 if (Ops.size() == 1) return Ops[0];
3750#ifndef NDEBUG
3751 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
3752 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
3753 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&(static_cast <bool> (getEffectiveSCEVType(Ops[i]->getType
()) == ETy && "Operand types don't match!") ? void (0
) : __assert_fail ("getEffectiveSCEVType(Ops[i]->getType()) == ETy && \"Operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3754, __extension__
__PRETTY_FUNCTION__))
3754 "Operand types don't match!")(static_cast <bool> (getEffectiveSCEVType(Ops[i]->getType
()) == ETy && "Operand types don't match!") ? void (0
) : __assert_fail ("getEffectiveSCEVType(Ops[i]->getType()) == ETy && \"Operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3754, __extension__
__PRETTY_FUNCTION__))
;
3755 assert(Ops[0]->getType()->isPointerTy() ==(static_cast <bool> (Ops[0]->getType()->isPointerTy
() == Ops[i]->getType()->isPointerTy() && "min/max should be consistently pointerish"
) ? void (0) : __assert_fail ("Ops[0]->getType()->isPointerTy() == Ops[i]->getType()->isPointerTy() && \"min/max should be consistently pointerish\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3757, __extension__
__PRETTY_FUNCTION__))
3756 Ops[i]->getType()->isPointerTy() &&(static_cast <bool> (Ops[0]->getType()->isPointerTy
() == Ops[i]->getType()->isPointerTy() && "min/max should be consistently pointerish"
) ? void (0) : __assert_fail ("Ops[0]->getType()->isPointerTy() == Ops[i]->getType()->isPointerTy() && \"min/max should be consistently pointerish\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3757, __extension__
__PRETTY_FUNCTION__))
3757 "min/max should be consistently pointerish")(static_cast <bool> (Ops[0]->getType()->isPointerTy
() == Ops[i]->getType()->isPointerTy() && "min/max should be consistently pointerish"
) ? void (0) : __assert_fail ("Ops[0]->getType()->isPointerTy() == Ops[i]->getType()->isPointerTy() && \"min/max should be consistently pointerish\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3757, __extension__
__PRETTY_FUNCTION__))
;
3758 }
3759#endif
3760
3761 bool IsSigned = Kind == scSMaxExpr || Kind == scSMinExpr;
3762 bool IsMax = Kind == scSMaxExpr || Kind == scUMaxExpr;
3763
3764 // Sort by complexity, this groups all similar expression types together.
3765 GroupByComplexity(Ops, &LI, DT);
3766
3767 // Check if we have created the same expression before.
3768 if (const SCEV *S = findExistingSCEVInCache(Kind, Ops)) {
3769 return S;
3770 }
3771
3772 // If there are any constants, fold them together.
3773 unsigned Idx = 0;
3774 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
3775 ++Idx;
3776 assert(Idx < Ops.size())(static_cast <bool> (Idx < Ops.size()) ? void (0) : __assert_fail
("Idx < Ops.size()", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 3776, __extension__ __PRETTY_FUNCTION__))
;
3777 auto FoldOp = [&](const APInt &LHS, const APInt &RHS) {
3778 if (Kind == scSMaxExpr)
3779 return APIntOps::smax(LHS, RHS);
3780 else if (Kind == scSMinExpr)
3781 return APIntOps::smin(LHS, RHS);
3782 else if (Kind == scUMaxExpr)
3783 return APIntOps::umax(LHS, RHS);
3784 else if (Kind == scUMinExpr)
3785 return APIntOps::umin(LHS, RHS);
3786 llvm_unreachable("Unknown SCEV min/max opcode")::llvm::llvm_unreachable_internal("Unknown SCEV min/max opcode"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3786)
;
3787 };
3788
3789 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
3790 // We found two constants, fold them together!
3791 ConstantInt *Fold = ConstantInt::get(
3792 getContext(), FoldOp(LHSC->getAPInt(), RHSC->getAPInt()));
3793 Ops[0] = getConstant(Fold);
3794 Ops.erase(Ops.begin()+1); // Erase the folded element
3795 if (Ops.size() == 1) return Ops[0];
3796 LHSC = cast<SCEVConstant>(Ops[0]);
3797 }
3798
3799 bool IsMinV = LHSC->getValue()->isMinValue(IsSigned);
3800 bool IsMaxV = LHSC->getValue()->isMaxValue(IsSigned);
3801
3802 if (IsMax ? IsMinV : IsMaxV) {
3803 // If we are left with a constant minimum(/maximum)-int, strip it off.
3804 Ops.erase(Ops.begin());
3805 --Idx;
3806 } else if (IsMax ? IsMaxV : IsMinV) {
3807 // If we have a max(/min) with a constant maximum(/minimum)-int,
3808 // it will always be the extremum.
3809 return LHSC;
3810 }
3811
3812 if (Ops.size() == 1) return Ops[0];
3813 }
3814
3815 // Find the first operation of the same kind
3816 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < Kind)
3817 ++Idx;
3818
3819 // Check to see if one of the operands is of the same kind. If so, expand its
3820 // operands onto our operand list, and recurse to simplify.
3821 if (Idx < Ops.size()) {
3822 bool DeletedAny = false;
3823 while (Ops[Idx]->getSCEVType() == Kind) {
3824 const SCEVMinMaxExpr *SMME = cast<SCEVMinMaxExpr>(Ops[Idx]);
3825 Ops.erase(Ops.begin()+Idx);
3826 Ops.append(SMME->op_begin(), SMME->op_end());
3827 DeletedAny = true;
3828 }
3829
3830 if (DeletedAny)
3831 return getMinMaxExpr(Kind, Ops);
3832 }
3833
3834 // Okay, check to see if the same value occurs in the operand list twice. If
3835 // so, delete one. Since we sorted the list, these values are required to
3836 // be adjacent.
3837 llvm::CmpInst::Predicate GEPred =
3838 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
3839 llvm::CmpInst::Predicate LEPred =
3840 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
3841 llvm::CmpInst::Predicate FirstPred = IsMax ? GEPred : LEPred;
3842 llvm::CmpInst::Predicate SecondPred = IsMax ? LEPred : GEPred;
3843 for (unsigned i = 0, e = Ops.size() - 1; i != e; ++i) {
3844 if (Ops[i] == Ops[i + 1] ||
3845 isKnownViaNonRecursiveReasoning(FirstPred, Ops[i], Ops[i + 1])) {
3846 // X op Y op Y --> X op Y
3847 // X op Y --> X, if we know X, Y are ordered appropriately
3848 Ops.erase(Ops.begin() + i + 1, Ops.begin() + i + 2);
3849 --i;
3850 --e;
3851 } else if (isKnownViaNonRecursiveReasoning(SecondPred, Ops[i],
3852 Ops[i + 1])) {
3853 // X op Y --> Y, if we know X, Y are ordered appropriately
3854 Ops.erase(Ops.begin() + i, Ops.begin() + i + 1);
3855 --i;
3856 --e;
3857 }
3858 }
3859
3860 if (Ops.size() == 1) return Ops[0];
3861
3862 assert(!Ops.empty() && "Reduced smax down to nothing!")(static_cast <bool> (!Ops.empty() && "Reduced smax down to nothing!"
) ? void (0) : __assert_fail ("!Ops.empty() && \"Reduced smax down to nothing!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3862, __extension__
__PRETTY_FUNCTION__))
;
3863
3864 // Okay, it looks like we really DO need an expr. Check to see if we
3865 // already have one, otherwise create a new one.
3866 FoldingSetNodeID ID;
3867 ID.AddInteger(Kind);
3868 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
3869 ID.AddPointer(Ops[i]);
3870 void *IP = nullptr;
3871 const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
3872 if (ExistingSCEV)
3873 return ExistingSCEV;
3874 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
3875 std::uninitialized_copy(Ops.begin(), Ops.end(), O);
3876 SCEV *S = new (SCEVAllocator)
3877 SCEVMinMaxExpr(ID.Intern(SCEVAllocator), Kind, O, Ops.size());
3878
3879 UniqueSCEVs.InsertNode(S, IP);
3880 registerUser(S, Ops);
3881 return S;
3882}
3883
3884namespace {
3885
3886class SCEVSequentialMinMaxDeduplicatingVisitor final
3887 : public SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor,
3888 Optional<const SCEV *>> {
3889 using RetVal = Optional<const SCEV *>;
3890 using Base = SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor, RetVal>;
3891
3892 ScalarEvolution &SE;
3893 const SCEVTypes RootKind; // Must be a sequential min/max expression.
3894 const SCEVTypes NonSequentialRootKind; // Non-sequential variant of RootKind.
3895 SmallPtrSet<const SCEV *, 16> SeenOps;
3896
3897 bool canRecurseInto(SCEVTypes Kind) const {
3898 // We can only recurse into the SCEV expression of the same effective type
3899 // as the type of our root SCEV expression.
3900 return RootKind == Kind || NonSequentialRootKind == Kind;
3901 };
3902
3903 RetVal visitAnyMinMaxExpr(const SCEV *S) {
3904 assert((isa<SCEVMinMaxExpr>(S) || isa<SCEVSequentialMinMaxExpr>(S)) &&(static_cast <bool> ((isa<SCEVMinMaxExpr>(S) || isa
<SCEVSequentialMinMaxExpr>(S)) && "Only for min/max expressions."
) ? void (0) : __assert_fail ("(isa<SCEVMinMaxExpr>(S) || isa<SCEVSequentialMinMaxExpr>(S)) && \"Only for min/max expressions.\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3905, __extension__
__PRETTY_FUNCTION__))
3905 "Only for min/max expressions.")(static_cast <bool> ((isa<SCEVMinMaxExpr>(S) || isa
<SCEVSequentialMinMaxExpr>(S)) && "Only for min/max expressions."
) ? void (0) : __assert_fail ("(isa<SCEVMinMaxExpr>(S) || isa<SCEVSequentialMinMaxExpr>(S)) && \"Only for min/max expressions.\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 3905, __extension__
__PRETTY_FUNCTION__))
;
3906 SCEVTypes Kind = S->getSCEVType();
3907
3908 if (!canRecurseInto(Kind))
3909 return S;
3910
3911 auto *NAry = cast<SCEVNAryExpr>(S);
3912 SmallVector<const SCEV *> NewOps;
3913 bool Changed =
3914 visit(Kind, makeArrayRef(NAry->op_begin(), NAry->op_end()), NewOps);
3915
3916 if (!Changed)
3917 return S;
3918 if (NewOps.empty())
3919 return None;
3920
3921 return isa<SCEVSequentialMinMaxExpr>(S)
3922 ? SE.getSequentialMinMaxExpr(Kind, NewOps)
3923 : SE.getMinMaxExpr(Kind, NewOps);
3924 }
3925
3926 RetVal visit(const SCEV *S) {
3927 // Has the whole operand been seen already?
3928 if (!SeenOps.insert(S).second)
3929 return None;
3930 return Base::visit(S);
3931 }
3932
3933public:
3934 SCEVSequentialMinMaxDeduplicatingVisitor(ScalarEvolution &SE,
3935 SCEVTypes RootKind)
3936 : SE(SE), RootKind(RootKind),
3937 NonSequentialRootKind(
3938 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
3939 RootKind)) {}
3940
3941 bool /*Changed*/ visit(SCEVTypes Kind, ArrayRef<const SCEV *> OrigOps,
3942 SmallVectorImpl<const SCEV *> &NewOps) {
3943 bool Changed = false;
3944 SmallVector<const SCEV *> Ops;
3945 Ops.reserve(OrigOps.size());
3946
3947 for (const SCEV *Op : OrigOps) {
3948 RetVal NewOp = visit(Op);
3949 if (NewOp != Op)
3950 Changed = true;
3951 if (NewOp)
3952 Ops.emplace_back(*NewOp);
3953 }
3954
3955 if (Changed)
3956 NewOps = std::move(Ops);
3957 return Changed;
3958 }
3959
3960 RetVal visitConstant(const SCEVConstant *Constant) { return Constant; }
3961
3962 RetVal visitPtrToIntExpr(const SCEVPtrToIntExpr *Expr) { return Expr; }
3963
3964 RetVal visitTruncateExpr(const SCEVTruncateExpr *Expr) { return Expr; }
3965
3966 RetVal visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) { return Expr; }
3967
3968 RetVal visitSignExtendExpr(const SCEVSignExtendExpr *Expr) { return Expr; }
3969
3970 RetVal visitAddExpr(const SCEVAddExpr *Expr) { return Expr; }
3971
3972 RetVal visitMulExpr(const SCEVMulExpr *Expr) { return Expr; }
3973
3974 RetVal visitUDivExpr(const SCEVUDivExpr *Expr) { return Expr; }
3975
3976 RetVal visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
3977
3978 RetVal visitSMaxExpr(const SCEVSMaxExpr *Expr) {
3979 return visitAnyMinMaxExpr(Expr);
3980 }
3981
3982 RetVal visitUMaxExpr(const SCEVUMaxExpr *Expr) {
3983 return visitAnyMinMaxExpr(Expr);
3984 }
3985
3986 RetVal visitSMinExpr(const SCEVSMinExpr *Expr) {
3987 return visitAnyMinMaxExpr(Expr);
3988 }
3989
3990 RetVal visitUMinExpr(const SCEVUMinExpr *Expr) {
3991 return visitAnyMinMaxExpr(Expr);
3992 }
3993
3994 RetVal visitSequentialUMinExpr(const SCEVSequentialUMinExpr *Expr) {
3995 return visitAnyMinMaxExpr(Expr);
3996 }
3997
3998 RetVal visitUnknown(const SCEVUnknown *Expr) { return Expr; }
3999
4000 RetVal visitCouldNotCompute(const SCEVCouldNotCompute *Expr) { return Expr; }
4001};
4002
4003} // namespace
4004
4005const SCEV *
4006ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
4007 SmallVectorImpl<const SCEV *> &Ops) {
4008 assert(SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) &&(static_cast <bool> (SCEVSequentialMinMaxExpr::isSequentialMinMaxType
(Kind) && "Not a SCEVSequentialMinMaxExpr!") ? void (
0) : __assert_fail ("SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) && \"Not a SCEVSequentialMinMaxExpr!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4009, __extension__
__PRETTY_FUNCTION__))
4009 "Not a SCEVSequentialMinMaxExpr!")(static_cast <bool> (SCEVSequentialMinMaxExpr::isSequentialMinMaxType
(Kind) && "Not a SCEVSequentialMinMaxExpr!") ? void (
0) : __assert_fail ("SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) && \"Not a SCEVSequentialMinMaxExpr!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4009, __extension__
__PRETTY_FUNCTION__))
;
4010 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!")(static_cast <bool> (!Ops.empty() && "Cannot get empty (u|s)(min|max)!"
) ? void (0) : __assert_fail ("!Ops.empty() && \"Cannot get empty (u|s)(min|max)!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4010, __extension__
__PRETTY_FUNCTION__))
;
4011 if (Ops.size() == 1)
4012 return Ops[0];
4013 if (Ops.size() == 2 &&
4014 any_of(Ops, [](const SCEV *Op) { return isa<SCEVConstant>(Op); }))
4015 return getMinMaxExpr(
4016 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(Kind),
4017 Ops);
4018#ifndef NDEBUG
4019 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
4020 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4021 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&(static_cast <bool> (getEffectiveSCEVType(Ops[i]->getType
()) == ETy && "Operand types don't match!") ? void (0
) : __assert_fail ("getEffectiveSCEVType(Ops[i]->getType()) == ETy && \"Operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4022, __extension__
__PRETTY_FUNCTION__))
4022 "Operand types don't match!")(static_cast <bool> (getEffectiveSCEVType(Ops[i]->getType
()) == ETy && "Operand types don't match!") ? void (0
) : __assert_fail ("getEffectiveSCEVType(Ops[i]->getType()) == ETy && \"Operand types don't match!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4022, __extension__
__PRETTY_FUNCTION__))
;
4023 assert(Ops[0]->getType()->isPointerTy() ==(static_cast <bool> (Ops[0]->getType()->isPointerTy
() == Ops[i]->getType()->isPointerTy() && "min/max should be consistently pointerish"
) ? void (0) : __assert_fail ("Ops[0]->getType()->isPointerTy() == Ops[i]->getType()->isPointerTy() && \"min/max should be consistently pointerish\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4025, __extension__
__PRETTY_FUNCTION__))
4024 Ops[i]->getType()->isPointerTy() &&(static_cast <bool> (Ops[0]->getType()->isPointerTy
() == Ops[i]->getType()->isPointerTy() && "min/max should be consistently pointerish"
) ? void (0) : __assert_fail ("Ops[0]->getType()->isPointerTy() == Ops[i]->getType()->isPointerTy() && \"min/max should be consistently pointerish\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4025, __extension__
__PRETTY_FUNCTION__))
4025 "min/max should be consistently pointerish")(static_cast <bool> (Ops[0]->getType()->isPointerTy
() == Ops[i]->getType()->isPointerTy() && "min/max should be consistently pointerish"
) ? void (0) : __assert_fail ("Ops[0]->getType()->isPointerTy() == Ops[i]->getType()->isPointerTy() && \"min/max should be consistently pointerish\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4025, __extension__
__PRETTY_FUNCTION__))
;
4026 }
4027#endif
4028
4029 // Note that SCEVSequentialMinMaxExpr is *NOT* commutative,
4030 // so we can *NOT* do any kind of sorting of the expressions!
4031
4032 // Check if we have created the same expression before.
4033 if (const SCEV *S = findExistingSCEVInCache(Kind, Ops))
4034 return S;
4035
4036 // FIXME: there are *some* simplifications that we can do here.
4037
4038 // Keep only the first instance of an operand.
4039 {
4040 SCEVSequentialMinMaxDeduplicatingVisitor Deduplicator(*this, Kind);
4041 bool Changed = Deduplicator.visit(Kind, Ops, Ops);
4042 if (Changed)
4043 return getSequentialMinMaxExpr(Kind, Ops);
4044 }
4045
4046 // Check to see if one of the operands is of the same kind. If so, expand its
4047 // operands onto our operand list, and recurse to simplify.
4048 {
4049 unsigned Idx = 0;
4050 bool DeletedAny = false;
4051 while (Idx < Ops.size()) {
4052 if (Ops[Idx]->getSCEVType() != Kind) {
4053 ++Idx;
4054 continue;
4055 }
4056 const auto *SMME = cast<SCEVSequentialMinMaxExpr>(Ops[Idx]);
4057 Ops.erase(Ops.begin() + Idx);
4058 Ops.insert(Ops.begin() + Idx, SMME->op_begin(), SMME->op_end());
4059 DeletedAny = true;
4060 }
4061
4062 if (DeletedAny)
4063 return getSequentialMinMaxExpr(Kind, Ops);
4064 }
4065
4066 // Okay, it looks like we really DO need an expr. Check to see if we
4067 // already have one, otherwise create a new one.
4068 FoldingSetNodeID ID;
4069 ID.AddInteger(Kind);
4070 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
4071 ID.AddPointer(Ops[i]);
4072 void *IP = nullptr;
4073 const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
4074 if (ExistingSCEV)
4075 return ExistingSCEV;
4076
4077 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
4078 std::uninitialized_copy(Ops.begin(), Ops.end(), O);
4079 SCEV *S = new (SCEVAllocator)
4080 SCEVSequentialMinMaxExpr(ID.Intern(SCEVAllocator), Kind, O, Ops.size());
4081
4082 UniqueSCEVs.InsertNode(S, IP);
4083 registerUser(S, Ops);
4084 return S;
4085}
4086
4087const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS, const SCEV *RHS) {
4088 SmallVector<const SCEV *, 2> Ops = {LHS, RHS};
4089 return getSMaxExpr(Ops);
4090}
4091
4092const SCEV *ScalarEvolution::getSMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
4093 return getMinMaxExpr(scSMaxExpr, Ops);
4094}
4095
4096const SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS, const SCEV *RHS) {
4097 SmallVector<const SCEV *, 2> Ops = {LHS, RHS};
4098 return getUMaxExpr(Ops);
4099}
4100
4101const SCEV *ScalarEvolution::getUMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
4102 return getMinMaxExpr(scUMaxExpr, Ops);
4103}
4104
4105const SCEV *ScalarEvolution::getSMinExpr(const SCEV *LHS,
4106 const SCEV *RHS) {
4107 SmallVector<const SCEV *, 2> Ops = { LHS, RHS };
4108 return getSMinExpr(Ops);
4109}
4110
4111const SCEV *ScalarEvolution::getSMinExpr(SmallVectorImpl<const SCEV *> &Ops) {
4112 return getMinMaxExpr(scSMinExpr, Ops);
4113}
4114
4115const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS, const SCEV *RHS,
4116 bool Sequential) {
4117 SmallVector<const SCEV *, 2> Ops = { LHS, RHS };
4118 return getUMinExpr(Ops, Sequential);
4119}
4120
4121const SCEV *ScalarEvolution::getUMinExpr(SmallVectorImpl<const SCEV *> &Ops,
4122 bool Sequential) {
4123 return Sequential ? getSequentialMinMaxExpr(scSequentialUMinExpr, Ops)
4124 : getMinMaxExpr(scUMinExpr, Ops);
4125}
4126
4127const SCEV *
4128ScalarEvolution::getSizeOfScalableVectorExpr(Type *IntTy,
4129 ScalableVectorType *ScalableTy) {
4130 Constant *NullPtr = Constant::getNullValue(ScalableTy->getPointerTo());
4131 Constant *One = ConstantInt::get(IntTy, 1);
4132 Constant *GEP = ConstantExpr::getGetElementPtr(ScalableTy, NullPtr, One);
4133 // Note that the expression we created is the final expression, we don't
4134 // want to simplify it any further Also, if we call a normal getSCEV(),
4135 // we'll end up in an endless recursion. So just create an SCEVUnknown.
4136 return getUnknown(ConstantExpr::getPtrToInt(GEP, IntTy));
4137}
4138
4139const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) {
4140 if (auto *ScalableAllocTy = dyn_cast<ScalableVectorType>(AllocTy))
4141 return getSizeOfScalableVectorExpr(IntTy, ScalableAllocTy);
4142 // We can bypass creating a target-independent constant expression and then
4143 // folding it back into a ConstantInt. This is just a compile-time
4144 // optimization.
4145 return getConstant(IntTy, getDataLayout().getTypeAllocSize(AllocTy));
4146}
4147
4148const SCEV *ScalarEvolution::getStoreSizeOfExpr(Type *IntTy, Type *StoreTy) {
4149 if (auto *ScalableStoreTy = dyn_cast<ScalableVectorType>(StoreTy))
4150 return getSizeOfScalableVectorExpr(IntTy, ScalableStoreTy);
4151 // We can bypass creating a target-independent constant expression and then
4152 // folding it back into a ConstantInt. This is just a compile-time
4153 // optimization.
4154 return getConstant(IntTy, getDataLayout().getTypeStoreSize(StoreTy));
4155}
4156
4157const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy,
4158 StructType *STy,
4159 unsigned FieldNo) {
4160 // We can bypass creating a target-independent constant expression and then
4161 // folding it back into a ConstantInt. This is just a compile-time
4162 // optimization.
4163 return getConstant(
4164 IntTy, getDataLayout().getStructLayout(STy)->getElementOffset(FieldNo));
4165}
4166
4167const SCEV *ScalarEvolution::getUnknown(Value *V) {
4168 // Don't attempt to do anything other than create a SCEVUnknown object
4169 // here. createSCEV only calls getUnknown after checking for all other
4170 // interesting possibilities, and any other code that calls getUnknown
4171 // is doing so in order to hide a value from SCEV canonicalization.
4172
4173 FoldingSetNodeID ID;
4174 ID.AddInteger(scUnknown);
4175 ID.AddPointer(V);
4176 void *IP = nullptr;
4177 if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) {
4178 assert(cast<SCEVUnknown>(S)->getValue() == V &&(static_cast <bool> (cast<SCEVUnknown>(S)->getValue
() == V && "Stale SCEVUnknown in uniquing map!") ? void
(0) : __assert_fail ("cast<SCEVUnknown>(S)->getValue() == V && \"Stale SCEVUnknown in uniquing map!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4179, __extension__
__PRETTY_FUNCTION__))
4179 "Stale SCEVUnknown in uniquing map!")(static_cast <bool> (cast<SCEVUnknown>(S)->getValue
() == V && "Stale SCEVUnknown in uniquing map!") ? void
(0) : __assert_fail ("cast<SCEVUnknown>(S)->getValue() == V && \"Stale SCEVUnknown in uniquing map!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4179, __extension__
__PRETTY_FUNCTION__))
;
4180 return S;
4181 }
4182 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(SCEVAllocator), V, this,
4183 FirstUnknown);
4184 FirstUnknown = cast<SCEVUnknown>(S);
4185 UniqueSCEVs.InsertNode(S, IP);
4186 return S;
4187}
4188
4189//===----------------------------------------------------------------------===//
4190// Basic SCEV Analysis and PHI Idiom Recognition Code
4191//
4192
4193/// Test if values of the given type are analyzable within the SCEV
4194/// framework. This primarily includes integer types, and it can optionally
4195/// include pointer types if the ScalarEvolution class has access to
4196/// target-specific information.
4197bool ScalarEvolution::isSCEVable(Type *Ty) const {
4198 // Integers and pointers are always SCEVable.
4199 return Ty->isIntOrPtrTy();
4200}
4201
4202/// Return the size in bits of the specified type, for which isSCEVable must
4203/// return true.
4204uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
4205 assert(isSCEVable(Ty) && "Type is not SCEVable!")(static_cast <bool> (isSCEVable(Ty) && "Type is not SCEVable!"
) ? void (0) : __assert_fail ("isSCEVable(Ty) && \"Type is not SCEVable!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4205, __extension__
__PRETTY_FUNCTION__))
;
4206 if (Ty->isPointerTy())
4207 return getDataLayout().getIndexTypeSizeInBits(Ty);
4208 return getDataLayout().getTypeSizeInBits(Ty);
4209}
4210
4211/// Return a type with the same bitwidth as the given type and which represents
4212/// how SCEV will treat the given type, for which isSCEVable must return
4213/// true. For pointer types, this is the pointer index sized integer type.
4214Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
4215 assert(isSCEVable(Ty) && "Type is not SCEVable!")(static_cast <bool> (isSCEVable(Ty) && "Type is not SCEVable!"
) ? void (0) : __assert_fail ("isSCEVable(Ty) && \"Type is not SCEVable!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4215, __extension__
__PRETTY_FUNCTION__))
;
4216
4217 if (Ty->isIntegerTy())
4218 return Ty;
4219
4220 // The only other support type is pointer.
4221 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!")(static_cast <bool> (Ty->isPointerTy() && "Unexpected non-pointer non-integer type!"
) ? void (0) : __assert_fail ("Ty->isPointerTy() && \"Unexpected non-pointer non-integer type!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4221, __extension__
__PRETTY_FUNCTION__))
;
4222 return getDataLayout().getIndexType(Ty);
4223}
4224
4225Type *ScalarEvolution::getWiderType(Type *T1, Type *T2) const {
4226 return getTypeSizeInBits(T1) >= getTypeSizeInBits(T2) ? T1 : T2;
4227}
4228
4229bool ScalarEvolution::instructionCouldExistWitthOperands(const SCEV *A,
4230 const SCEV *B) {
4231 /// For a valid use point to exist, the defining scope of one operand
4232 /// must dominate the other.
4233 bool PreciseA, PreciseB;
4234 auto *ScopeA = getDefiningScopeBound({A}, PreciseA);
4235 auto *ScopeB = getDefiningScopeBound({B}, PreciseB);
4236 if (!PreciseA || !PreciseB)
4237 // Can't tell.
4238 return false;
4239 return (ScopeA == ScopeB) || DT.dominates(ScopeA, ScopeB) ||
4240 DT.dominates(ScopeB, ScopeA);
4241}
4242
4243
4244const SCEV *ScalarEvolution::getCouldNotCompute() {
4245 return CouldNotCompute.get();
4246}
4247
4248bool ScalarEvolution::checkValidity(const SCEV *S) const {
4249 bool ContainsNulls = SCEVExprContains(S, [](const SCEV *S) {
4250 auto *SU = dyn_cast<SCEVUnknown>(S);
4251 return SU && SU->getValue() == nullptr;
4252 });
4253
4254 return !ContainsNulls;
4255}
4256
4257bool ScalarEvolution::containsAddRecurrence(const SCEV *S) {
4258 HasRecMapType::iterator I = HasRecMap.find(S);
4259 if (I != HasRecMap.end())
4260 return I->second;
4261
4262 bool FoundAddRec =
4263 SCEVExprContains(S, [](const SCEV *S) { return isa<SCEVAddRecExpr>(S); });
4264 HasRecMap.insert({S, FoundAddRec});
4265 return FoundAddRec;
4266}
4267
4268/// Try to split a SCEVAddExpr into a pair of {SCEV, ConstantInt}.
4269/// If \p S is a SCEVAddExpr and is composed of a sub SCEV S' and an
4270/// offset I, then return {S', I}, else return {\p S, nullptr}.
4271static std::pair<const SCEV *, ConstantInt *> splitAddExpr(const SCEV *S) {
4272 const auto *Add = dyn_cast<SCEVAddExpr>(S);
4273 if (!Add)
4274 return {S, nullptr};
4275
4276 if (Add->getNumOperands() != 2)
4277 return {S, nullptr};
4278
4279 auto *ConstOp = dyn_cast<SCEVConstant>(Add->getOperand(0));
4280 if (!ConstOp)
4281 return {S, nullptr};
4282
4283 return {Add->getOperand(1), ConstOp->getValue()};
4284}
4285
4286/// Return the ValueOffsetPair set for \p S. \p S can be represented
4287/// by the value and offset from any ValueOffsetPair in the set.
4288ScalarEvolution::ValueOffsetPairSetVector *
4289ScalarEvolution::getSCEVValues(const SCEV *S) {
4290 ExprValueMapType::iterator SI = ExprValueMap.find_as(S);
4291 if (SI == ExprValueMap.end())
4292 return nullptr;
4293#ifndef NDEBUG
4294 if (VerifySCEVMap) {
4295 // Check there is no dangling Value in the set returned.
4296 for (const auto &VE : SI->second)
4297 assert(ValueExprMap.count(VE.first))(static_cast <bool> (ValueExprMap.count(VE.first)) ? void
(0) : __assert_fail ("ValueExprMap.count(VE.first)", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 4297, __extension__ __PRETTY_FUNCTION__))
;
4298 }
4299#endif
4300 return &SI->second;
4301}
4302
4303/// Erase Value from ValueExprMap and ExprValueMap. ValueExprMap.erase(V)
4304/// cannot be used separately. eraseValueFromMap should be used to remove
4305/// V from ValueExprMap and ExprValueMap at the same time.
4306void ScalarEvolution::eraseValueFromMap(Value *V) {
4307 ValueExprMapType::iterator I = ValueExprMap.find_as(V);
4308 if (I != ValueExprMap.end()) {
4309 const SCEV *S = I->second;
4310 // Remove {V, 0} from the set of ExprValueMap[S]
4311 if (auto *SV = getSCEVValues(S))
4312 SV->remove({V, nullptr});
4313
4314 // Remove {V, Offset} from the set of ExprValueMap[Stripped]
4315 const SCEV *Stripped;
4316 ConstantInt *Offset;
4317 std::tie(Stripped, Offset) = splitAddExpr(S);
4318 if (Offset != nullptr) {
4319 if (auto *SV = getSCEVValues(Stripped))
4320 SV->remove({V, Offset});
4321 }
4322 ValueExprMap.erase(V);
4323 }
4324}
4325
4326void ScalarEvolution::insertValueToMap(Value *V, const SCEV *S) {
4327 // A recursive query may have already computed the SCEV. It should be
4328 // equivalent, but may not necessarily be exactly the same, e.g. due to lazily
4329 // inferred nowrap flags.
4330 auto It = ValueExprMap.find_as(V);
4331 if (It == ValueExprMap.end()) {
4332 ValueExprMap.insert({SCEVCallbackVH(V, this), S});
4333 ExprValueMap[S].insert({V, nullptr});
4334 }
4335}
4336
4337/// Return an existing SCEV if it exists, otherwise analyze the expression and
4338/// create a new one.
4339const SCEV *ScalarEvolution::getSCEV(Value *V) {
4340 assert(isSCEVable(V->getType()) && "Value is not SCEVable!")(static_cast <bool> (isSCEVable(V->getType()) &&
"Value is not SCEVable!") ? void (0) : __assert_fail ("isSCEVable(V->getType()) && \"Value is not SCEVable!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4340, __extension__
__PRETTY_FUNCTION__))
;
4341
4342 const SCEV *S = getExistingSCEV(V);
4343 if (S == nullptr) {
4344 S = createSCEV(V);
4345 // During PHI resolution, it is possible to create two SCEVs for the same
4346 // V, so it is needed to double check whether V->S is inserted into
4347 // ValueExprMap before insert S->{V, 0} into ExprValueMap.
4348 std::pair<ValueExprMapType::iterator, bool> Pair =
4349 ValueExprMap.insert({SCEVCallbackVH(V, this), S});
4350 if (Pair.second) {
4351 ExprValueMap[S].insert({V, nullptr});
4352
4353 // If S == Stripped + Offset, add Stripped -> {V, Offset} into
4354 // ExprValueMap.
4355 const SCEV *Stripped = S;
4356 ConstantInt *Offset = nullptr;
4357 std::tie(Stripped, Offset) = splitAddExpr(S);
4358 // If stripped is SCEVUnknown, don't bother to save
4359 // Stripped -> {V, offset}. It doesn't simplify and sometimes even
4360 // increase the complexity of the expansion code.
4361 // If V is GetElementPtrInst, don't save Stripped -> {V, offset}
4362 // because it may generate add/sub instead of GEP in SCEV expansion.
4363 if (Offset != nullptr && !isa<SCEVUnknown>(Stripped) &&
4364 !isa<GetElementPtrInst>(V))
4365 ExprValueMap[Stripped].insert({V, Offset});
4366 }
4367 }
4368 return S;
4369}
4370
4371const SCEV *ScalarEvolution::getExistingSCEV(Value *V) {
4372 assert(isSCEVable(V->getType()) && "Value is not SCEVable!")(static_cast <bool> (isSCEVable(V->getType()) &&
"Value is not SCEVable!") ? void (0) : __assert_fail ("isSCEVable(V->getType()) && \"Value is not SCEVable!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4372, __extension__
__PRETTY_FUNCTION__))
;
4373
4374 ValueExprMapType::iterator I = ValueExprMap.find_as(V);
4375 if (I != ValueExprMap.end()) {
4376 const SCEV *S = I->second;
4377 assert(checkValidity(S) &&(static_cast <bool> (checkValidity(S) && "existing SCEV has not been properly invalidated"
) ? void (0) : __assert_fail ("checkValidity(S) && \"existing SCEV has not been properly invalidated\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4378, __extension__
__PRETTY_FUNCTION__))
4378 "existing SCEV has not been properly invalidated")(static_cast <bool> (checkValidity(S) && "existing SCEV has not been properly invalidated"
) ? void (0) : __assert_fail ("checkValidity(S) && \"existing SCEV has not been properly invalidated\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4378, __extension__
__PRETTY_FUNCTION__))
;
4379 return S;
4380 }
4381 return nullptr;
4382}
4383
4384/// Return a SCEV corresponding to -V = -1*V
4385const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V,
4386 SCEV::NoWrapFlags Flags) {
4387 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
4388 return getConstant(
4389 cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue())));
4390
4391 Type *Ty = V->getType();
4392 Ty = getEffectiveSCEVType(Ty);
4393 return getMulExpr(V, getMinusOne(Ty), Flags);
4394}
4395
4396/// If Expr computes ~A, return A else return nullptr
4397static const SCEV *MatchNotExpr(const SCEV *Expr) {
4398 const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Expr);
4399 if (!Add || Add->getNumOperands() != 2 ||
4400 !Add->getOperand(0)->isAllOnesValue())
4401 return nullptr;
4402
4403 const SCEVMulExpr *AddRHS = dyn_cast<SCEVMulExpr>(Add->getOperand(1));
4404 if (!AddRHS || AddRHS->getNumOperands() != 2 ||
4405 !AddRHS->getOperand(0)->isAllOnesValue())
4406 return nullptr;
4407
4408 return AddRHS->getOperand(1);
4409}
4410
4411/// Return a SCEV corresponding to ~V = -1-V
4412const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
4413 assert(!V->getType()->isPointerTy() && "Can't negate pointer")(static_cast <bool> (!V->getType()->isPointerTy()
&& "Can't negate pointer") ? void (0) : __assert_fail
("!V->getType()->isPointerTy() && \"Can't negate pointer\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4413, __extension__
__PRETTY_FUNCTION__))
;
4414
4415 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
4416 return getConstant(
4417 cast<ConstantInt>(ConstantExpr::getNot(VC->getValue())));
4418
4419 // Fold ~(u|s)(min|max)(~x, ~y) to (u|s)(max|min)(x, y)
4420 if (const SCEVMinMaxExpr *MME = dyn_cast<SCEVMinMaxExpr>(V)) {
4421 auto MatchMinMaxNegation = [&](const SCEVMinMaxExpr *MME) {
4422 SmallVector<const SCEV *, 2> MatchedOperands;
4423 for (const SCEV *Operand : MME->operands()) {
4424 const SCEV *Matched = MatchNotExpr(Operand);
4425 if (!Matched)
4426 return (const SCEV *)nullptr;
4427 MatchedOperands.push_back(Matched);
4428 }
4429 return getMinMaxExpr(SCEVMinMaxExpr::negate(MME->getSCEVType()),
4430 MatchedOperands);
4431 };
4432 if (const SCEV *Replaced = MatchMinMaxNegation(MME))
4433 return Replaced;
4434 }
4435
4436 Type *Ty = V->getType();
4437 Ty = getEffectiveSCEVType(Ty);
4438 return getMinusSCEV(getMinusOne(Ty), V);
4439}
4440
4441const SCEV *ScalarEvolution::removePointerBase(const SCEV *P) {
4442 assert(P->getType()->isPointerTy())(static_cast <bool> (P->getType()->isPointerTy())
? void (0) : __assert_fail ("P->getType()->isPointerTy()"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4442, __extension__
__PRETTY_FUNCTION__))
;
4443
4444 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(P)) {
4445 // The base of an AddRec is the first operand.
4446 SmallVector<const SCEV *> Ops{AddRec->operands()};
4447 Ops[0] = removePointerBase(Ops[0]);
4448 // Don't try to transfer nowrap flags for now. We could in some cases
4449 // (for example, if pointer operand of the AddRec is a SCEVUnknown).
4450 return getAddRecExpr(Ops, AddRec->getLoop(), SCEV::FlagAnyWrap);
4451 }
4452 if (auto *Add = dyn_cast<SCEVAddExpr>(P)) {
4453 // The base of an Add is the pointer operand.
4454 SmallVector<const SCEV *> Ops{Add->operands()};
4455 const SCEV **PtrOp = nullptr;
4456 for (const SCEV *&AddOp : Ops) {
4457 if (AddOp->getType()->isPointerTy()) {
4458 assert(!PtrOp && "Cannot have multiple pointer ops")(static_cast <bool> (!PtrOp && "Cannot have multiple pointer ops"
) ? void (0) : __assert_fail ("!PtrOp && \"Cannot have multiple pointer ops\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4458, __extension__
__PRETTY_FUNCTION__))
;
4459 PtrOp = &AddOp;
4460 }
4461 }
4462 *PtrOp = removePointerBase(*PtrOp);
4463 // Don't try to transfer nowrap flags for now. We could in some cases
4464 // (for example, if the pointer operand of the Add is a SCEVUnknown).
4465 return getAddExpr(Ops);
4466 }
4467 // Any other expression must be a pointer base.
4468 return getZero(P->getType());
4469}
4470
4471const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS,
4472 SCEV::NoWrapFlags Flags,
4473 unsigned Depth) {
4474 // Fast path: X - X --> 0.
4475 if (LHS == RHS)
4476 return getZero(LHS->getType());
4477
4478 // If we subtract two pointers with different pointer bases, bail.
4479 // Eventually, we're going to add an assertion to getMulExpr that we
4480 // can't multiply by a pointer.
4481 if (RHS->getType()->isPointerTy()) {
4482 if (!LHS->getType()->isPointerTy() ||
4483 getPointerBase(LHS) != getPointerBase(RHS))
4484 return getCouldNotCompute();
4485 LHS = removePointerBase(LHS);
4486 RHS = removePointerBase(RHS);
4487 }
4488
4489 // We represent LHS - RHS as LHS + (-1)*RHS. This transformation
4490 // makes it so that we cannot make much use of NUW.
4491 auto AddFlags = SCEV::FlagAnyWrap;
4492 const bool RHSIsNotMinSigned =
4493 !getSignedRangeMin(RHS).isMinSignedValue();
4494 if (hasFlags(Flags, SCEV::FlagNSW)) {
4495 // Let M be the minimum representable signed value. Then (-1)*RHS
4496 // signed-wraps if and only if RHS is M. That can happen even for
4497 // a NSW subtraction because e.g. (-1)*M signed-wraps even though
4498 // -1 - M does not. So to transfer NSW from LHS - RHS to LHS +
4499 // (-1)*RHS, we need to prove that RHS != M.
4500 //
4501 // If LHS is non-negative and we know that LHS - RHS does not
4502 // signed-wrap, then RHS cannot be M. So we can rule out signed-wrap
4503 // either by proving that RHS > M or that LHS >= 0.
4504 if (RHSIsNotMinSigned || isKnownNonNegative(LHS)) {
4505 AddFlags = SCEV::FlagNSW;
4506 }
4507 }
4508
4509 // FIXME: Find a correct way to transfer NSW to (-1)*M when LHS -
4510 // RHS is NSW and LHS >= 0.
4511 //
4512 // The difficulty here is that the NSW flag may have been proven
4513 // relative to a loop that is to be found in a recurrence in LHS and
4514 // not in RHS. Applying NSW to (-1)*M may then let the NSW have a
4515 // larger scope than intended.
4516 auto NegFlags = RHSIsNotMinSigned ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
4517
4518 return getAddExpr(LHS, getNegativeSCEV(RHS, NegFlags), AddFlags, Depth);
4519}
4520
4521const SCEV *ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty,
4522 unsigned Depth) {
4523 Type *SrcTy = V->getType();
4524 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&(static_cast <bool> (SrcTy->isIntOrPtrTy() &&
Ty->isIntOrPtrTy() && "Cannot truncate or zero extend with non-integer arguments!"
) ? void (0) : __assert_fail ("SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot truncate or zero extend with non-integer arguments!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4525, __extension__
__PRETTY_FUNCTION__))
4525 "Cannot truncate or zero extend with non-integer arguments!")(static_cast <bool> (SrcTy->isIntOrPtrTy() &&
Ty->isIntOrPtrTy() && "Cannot truncate or zero extend with non-integer arguments!"
) ? void (0) : __assert_fail ("SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot truncate or zero extend with non-integer arguments!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4525, __extension__
__PRETTY_FUNCTION__))
;
4526 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4527 return V; // No conversion
4528 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
4529 return getTruncateExpr(V, Ty, Depth);
4530 return getZeroExtendExpr(V, Ty, Depth);
4531}
4532
4533const SCEV *ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, Type *Ty,
4534 unsigned Depth) {
4535 Type *SrcTy = V->getType();
4536 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&(static_cast <bool> (SrcTy->isIntOrPtrTy() &&
Ty->isIntOrPtrTy() && "Cannot truncate or zero extend with non-integer arguments!"
) ? void (0) : __assert_fail ("SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot truncate or zero extend with non-integer arguments!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4537, __extension__
__PRETTY_FUNCTION__))
4537 "Cannot truncate or zero extend with non-integer arguments!")(static_cast <bool> (SrcTy->isIntOrPtrTy() &&
Ty->isIntOrPtrTy() && "Cannot truncate or zero extend with non-integer arguments!"
) ? void (0) : __assert_fail ("SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot truncate or zero extend with non-integer arguments!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4537, __extension__
__PRETTY_FUNCTION__))
;
4538 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4539 return V; // No conversion
4540 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
4541 return getTruncateExpr(V, Ty, Depth);
4542 return getSignExtendExpr(V, Ty, Depth);
4543}
4544
4545const SCEV *
4546ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) {
4547 Type *SrcTy = V->getType();
4548 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&(static_cast <bool> (SrcTy->isIntOrPtrTy() &&
Ty->isIntOrPtrTy() && "Cannot noop or zero extend with non-integer arguments!"
) ? void (0) : __assert_fail ("SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot noop or zero extend with non-integer arguments!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4549, __extension__
__PRETTY_FUNCTION__))
4549 "Cannot noop or zero extend with non-integer arguments!")(static_cast <bool> (SrcTy->isIntOrPtrTy() &&
Ty->isIntOrPtrTy() && "Cannot noop or zero extend with non-integer arguments!"
) ? void (0) : __assert_fail ("SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot noop or zero extend with non-integer arguments!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4549, __extension__
__PRETTY_FUNCTION__))
;
4550 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&(static_cast <bool> (getTypeSizeInBits(SrcTy) <= getTypeSizeInBits
(Ty) && "getNoopOrZeroExtend cannot truncate!") ? void
(0) : __assert_fail ("getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && \"getNoopOrZeroExtend cannot truncate!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4551, __extension__
__PRETTY_FUNCTION__))
4551 "getNoopOrZeroExtend cannot truncate!")(static_cast <bool> (getTypeSizeInBits(SrcTy) <= getTypeSizeInBits
(Ty) && "getNoopOrZeroExtend cannot truncate!") ? void
(0) : __assert_fail ("getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && \"getNoopOrZeroExtend cannot truncate!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4551, __extension__
__PRETTY_FUNCTION__))
;
4552 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4553 return V; // No conversion
4554 return getZeroExtendExpr(V, Ty);
4555}
4556
4557const SCEV *
4558ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) {
4559 Type *SrcTy = V->getType();
4560 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&(static_cast <bool> (SrcTy->isIntOrPtrTy() &&
Ty->isIntOrPtrTy() && "Cannot noop or sign extend with non-integer arguments!"
) ? void (0) : __assert_fail ("SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot noop or sign extend with non-integer arguments!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4561, __extension__
__PRETTY_FUNCTION__))
4561 "Cannot noop or sign extend with non-integer arguments!")(static_cast <bool> (SrcTy->isIntOrPtrTy() &&
Ty->isIntOrPtrTy() && "Cannot noop or sign extend with non-integer arguments!"
) ? void (0) : __assert_fail ("SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot noop or sign extend with non-integer arguments!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4561, __extension__
__PRETTY_FUNCTION__))
;
4562 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&(static_cast <bool> (getTypeSizeInBits(SrcTy) <= getTypeSizeInBits
(Ty) && "getNoopOrSignExtend cannot truncate!") ? void
(0) : __assert_fail ("getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && \"getNoopOrSignExtend cannot truncate!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4563, __extension__
__PRETTY_FUNCTION__))
4563 "getNoopOrSignExtend cannot truncate!")(static_cast <bool> (getTypeSizeInBits(SrcTy) <= getTypeSizeInBits
(Ty) && "getNoopOrSignExtend cannot truncate!") ? void
(0) : __assert_fail ("getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && \"getNoopOrSignExtend cannot truncate!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4563, __extension__
__PRETTY_FUNCTION__))
;
4564 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4565 return V; // No conversion
4566 return getSignExtendExpr(V, Ty);
4567}
4568
4569const SCEV *
4570ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) {
4571 Type *SrcTy = V->getType();
4572 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&(static_cast <bool> (SrcTy->isIntOrPtrTy() &&
Ty->isIntOrPtrTy() && "Cannot noop or any extend with non-integer arguments!"
) ? void (0) : __assert_fail ("SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot noop or any extend with non-integer arguments!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4573, __extension__
__PRETTY_FUNCTION__))
4573 "Cannot noop or any extend with non-integer arguments!")(static_cast <bool> (SrcTy->isIntOrPtrTy() &&
Ty->isIntOrPtrTy() && "Cannot noop or any extend with non-integer arguments!"
) ? void (0) : __assert_fail ("SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot noop or any extend with non-integer arguments!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4573, __extension__
__PRETTY_FUNCTION__))
;
4574 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&(static_cast <bool> (getTypeSizeInBits(SrcTy) <= getTypeSizeInBits
(Ty) && "getNoopOrAnyExtend cannot truncate!") ? void
(0) : __assert_fail ("getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && \"getNoopOrAnyExtend cannot truncate!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4575, __extension__
__PRETTY_FUNCTION__))
4575 "getNoopOrAnyExtend cannot truncate!")(static_cast <bool> (getTypeSizeInBits(SrcTy) <= getTypeSizeInBits
(Ty) && "getNoopOrAnyExtend cannot truncate!") ? void
(0) : __assert_fail ("getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && \"getNoopOrAnyExtend cannot truncate!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4575, __extension__
__PRETTY_FUNCTION__))
;
4576 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4577 return V; // No conversion
4578 return getAnyExtendExpr(V, Ty);
4579}
4580
4581const SCEV *
4582ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) {
4583 Type *SrcTy = V->getType();
4584 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&(static_cast <bool> (SrcTy->isIntOrPtrTy() &&
Ty->isIntOrPtrTy() && "Cannot truncate or noop with non-integer arguments!"
) ? void (0) : __assert_fail ("SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot truncate or noop with non-integer arguments!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4585, __extension__
__PRETTY_FUNCTION__))
4585 "Cannot truncate or noop with non-integer arguments!")(static_cast <bool> (SrcTy->isIntOrPtrTy() &&
Ty->isIntOrPtrTy() && "Cannot truncate or noop with non-integer arguments!"
) ? void (0) : __assert_fail ("SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && \"Cannot truncate or noop with non-integer arguments!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4585, __extension__
__PRETTY_FUNCTION__))
;
4586 assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) &&(static_cast <bool> (getTypeSizeInBits(SrcTy) >= getTypeSizeInBits
(Ty) && "getTruncateOrNoop cannot extend!") ? void (0
) : __assert_fail ("getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) && \"getTruncateOrNoop cannot extend!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4587, __extension__
__PRETTY_FUNCTION__))
4587 "getTruncateOrNoop cannot extend!")(static_cast <bool> (getTypeSizeInBits(SrcTy) >= getTypeSizeInBits
(Ty) && "getTruncateOrNoop cannot extend!") ? void (0
) : __assert_fail ("getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) && \"getTruncateOrNoop cannot extend!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4587, __extension__
__PRETTY_FUNCTION__))
;
4588 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
4589 return V; // No conversion
4590 return getTruncateExpr(V, Ty);
4591}
4592
4593const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS,
4594 const SCEV *RHS) {
4595 const SCEV *PromotedLHS = LHS;
4596 const SCEV *PromotedRHS = RHS;
4597
4598 if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType()))
4599 PromotedRHS = getZeroExtendExpr(RHS, LHS->getType());
4600 else
4601 PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType());
4602
4603 return getUMaxExpr(PromotedLHS, PromotedRHS);
4604}
4605
4606const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS,
4607 const SCEV *RHS,
4608 bool Sequential) {
4609 SmallVector<const SCEV *, 2> Ops = { LHS, RHS };
4610 return getUMinFromMismatchedTypes(Ops, Sequential);
4611}
4612
4613const SCEV *
4614ScalarEvolution::getUMinFromMismatchedTypes(SmallVectorImpl<const SCEV *> &Ops,
4615 bool Sequential) {
4616 assert(!Ops.empty() && "At least one operand must be!")(static_cast <bool> (!Ops.empty() && "At least one operand must be!"
) ? void (0) : __assert_fail ("!Ops.empty() && \"At least one operand must be!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4616, __extension__
__PRETTY_FUNCTION__))
;
4617 // Trivial case.
4618 if (Ops.size() == 1)
4619 return Ops[0];
4620
4621 // Find the max type first.
4622 Type *MaxType = nullptr;
4623 for (auto *S : Ops)
4624 if (MaxType)
4625 MaxType = getWiderType(MaxType, S->getType());
4626 else
4627 MaxType = S->getType();
4628 assert(MaxType && "Failed to find maximum type!")(static_cast <bool> (MaxType && "Failed to find maximum type!"
) ? void (0) : __assert_fail ("MaxType && \"Failed to find maximum type!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4628, __extension__
__PRETTY_FUNCTION__))
;
4629
4630 // Extend all ops to max type.
4631 SmallVector<const SCEV *, 2> PromotedOps;
4632 for (auto *S : Ops)
4633 PromotedOps.push_back(getNoopOrZeroExtend(S, MaxType));
4634
4635 // Generate umin.
4636 return getUMinExpr(PromotedOps, Sequential);
4637}
4638
4639const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) {
4640 // A pointer operand may evaluate to a nonpointer expression, such as null.
4641 if (!V->getType()->isPointerTy())
4642 return V;
4643
4644 while (true) {
4645 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(V)) {
4646 V = AddRec->getStart();
4647 } else if (auto *Add = dyn_cast<SCEVAddExpr>(V)) {
4648 const SCEV *PtrOp = nullptr;
4649 for (const SCEV *AddOp : Add->operands()) {
4650 if (AddOp->getType()->isPointerTy()) {
4651 assert(!PtrOp && "Cannot have multiple pointer ops")(static_cast <bool> (!PtrOp && "Cannot have multiple pointer ops"
) ? void (0) : __assert_fail ("!PtrOp && \"Cannot have multiple pointer ops\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4651, __extension__
__PRETTY_FUNCTION__))
;
4652 PtrOp = AddOp;
4653 }
4654 }
4655 assert(PtrOp && "Must have pointer op")(static_cast <bool> (PtrOp && "Must have pointer op"
) ? void (0) : __assert_fail ("PtrOp && \"Must have pointer op\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4655, __extension__
__PRETTY_FUNCTION__))
;
4656 V = PtrOp;
4657 } else // Not something we can look further into.
4658 return V;
4659 }
4660}
4661
4662/// Push users of the given Instruction onto the given Worklist.
4663static void PushDefUseChildren(Instruction *I,
4664 SmallVectorImpl<Instruction *> &Worklist,
4665 SmallPtrSetImpl<Instruction *> &Visited) {
4666 // Push the def-use children onto the Worklist stack.
4667 for (User *U : I->users()) {
4668 auto *UserInsn = cast<Instruction>(U);
4669 if (Visited.insert(UserInsn).second)
4670 Worklist.push_back(UserInsn);
4671 }
4672}
4673
4674namespace {
4675
4676/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its start
4677/// expression in case its Loop is L. If it is not L then
4678/// if IgnoreOtherLoops is true then use AddRec itself
4679/// otherwise rewrite cannot be done.
4680/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4681class SCEVInitRewriter : public SCEVRewriteVisitor<SCEVInitRewriter> {
4682public:
4683 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
4684 bool IgnoreOtherLoops = true) {
4685 SCEVInitRewriter Rewriter(L, SE);
4686 const SCEV *Result = Rewriter.visit(S);
4687 if (Rewriter.hasSeenLoopVariantSCEVUnknown())
4688 return SE.getCouldNotCompute();
4689 return Rewriter.hasSeenOtherLoops() && !IgnoreOtherLoops
4690 ? SE.getCouldNotCompute()
4691 : Result;
4692 }
4693
4694 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4695 if (!SE.isLoopInvariant(Expr, L))
4696 SeenLoopVariantSCEVUnknown = true;
4697 return Expr;
4698 }
4699
4700 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4701 // Only re-write AddRecExprs for this loop.
4702 if (Expr->getLoop() == L)
4703 return Expr->getStart();
4704 SeenOtherLoops = true;
4705 return Expr;
4706 }
4707
4708 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4709
4710 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4711
4712private:
4713 explicit SCEVInitRewriter(const Loop *L, ScalarEvolution &SE)
4714 : SCEVRewriteVisitor(SE), L(L) {}
4715
4716 const Loop *L;
4717 bool SeenLoopVariantSCEVUnknown = false;
4718 bool SeenOtherLoops = false;
4719};
4720
4721/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its post
4722/// increment expression in case its Loop is L. If it is not L then
4723/// use AddRec itself.
4724/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4725class SCEVPostIncRewriter : public SCEVRewriteVisitor<SCEVPostIncRewriter> {
4726public:
4727 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE) {
4728 SCEVPostIncRewriter Rewriter(L, SE);
4729 const SCEV *Result = Rewriter.visit(S);
4730 return Rewriter.hasSeenLoopVariantSCEVUnknown()
4731 ? SE.getCouldNotCompute()
4732 : Result;
4733 }
4734
4735 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4736 if (!SE.isLoopInvariant(Expr, L))
4737 SeenLoopVariantSCEVUnknown = true;
4738 return Expr;
4739 }
4740
4741 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4742 // Only re-write AddRecExprs for this loop.
4743 if (Expr->getLoop() == L)
4744 return Expr->getPostIncExpr(SE);
4745 SeenOtherLoops = true;
4746 return Expr;
4747 }
4748
4749 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4750
4751 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4752
4753private:
4754 explicit SCEVPostIncRewriter(const Loop *L, ScalarEvolution &SE)
4755 : SCEVRewriteVisitor(SE), L(L) {}
4756
4757 const Loop *L;
4758 bool SeenLoopVariantSCEVUnknown = false;
4759 bool SeenOtherLoops = false;
4760};
4761
4762/// This class evaluates the compare condition by matching it against the
4763/// condition of loop latch. If there is a match we assume a true value
4764/// for the condition while building SCEV nodes.
4765class SCEVBackedgeConditionFolder
4766 : public SCEVRewriteVisitor<SCEVBackedgeConditionFolder> {
4767public:
4768 static const SCEV *rewrite(const SCEV *S, const Loop *L,
4769 ScalarEvolution &SE) {
4770 bool IsPosBECond = false;
4771 Value *BECond = nullptr;
4772 if (BasicBlock *Latch = L->getLoopLatch()) {
4773 BranchInst *BI = dyn_cast<BranchInst>(Latch->getTerminator());
4774 if (BI && BI->isConditional()) {
4775 assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&(static_cast <bool> (BI->getSuccessor(0) != BI->getSuccessor
(1) && "Both outgoing branches should not target same header!"
) ? void (0) : __assert_fail ("BI->getSuccessor(0) != BI->getSuccessor(1) && \"Both outgoing branches should not target same header!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4776, __extension__
__PRETTY_FUNCTION__))
4776 "Both outgoing branches should not target same header!")(static_cast <bool> (BI->getSuccessor(0) != BI->getSuccessor
(1) && "Both outgoing branches should not target same header!"
) ? void (0) : __assert_fail ("BI->getSuccessor(0) != BI->getSuccessor(1) && \"Both outgoing branches should not target same header!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 4776, __extension__
__PRETTY_FUNCTION__))
;
4777 BECond = BI->getCondition();
4778 IsPosBECond = BI->getSuccessor(0) == L->getHeader();
4779 } else {
4780 return S;
4781 }
4782 }
4783 SCEVBackedgeConditionFolder Rewriter(L, BECond, IsPosBECond, SE);
4784 return Rewriter.visit(S);
4785 }
4786
4787 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4788 const SCEV *Result = Expr;
4789 bool InvariantF = SE.isLoopInvariant(Expr, L);
4790
4791 if (!InvariantF) {
4792 Instruction *I = cast<Instruction>(Expr->getValue());
4793 switch (I->getOpcode()) {
4794 case Instruction::Select: {
4795 SelectInst *SI = cast<SelectInst>(I);
4796 Optional<const SCEV *> Res =
4797 compareWithBackedgeCondition(SI->getCondition());
4798 if (Res.hasValue()) {
4799 bool IsOne = cast<SCEVConstant>(Res.getValue())->getValue()->isOne();
4800 Result = SE.getSCEV(IsOne ? SI->getTrueValue() : SI->getFalseValue());
4801 }
4802 break;
4803 }
4804 default: {
4805 Optional<const SCEV *> Res = compareWithBackedgeCondition(I);
4806 if (Res.hasValue())
4807 Result = Res.getValue();
4808 break;
4809 }
4810 }
4811 }
4812 return Result;
4813 }
4814
4815private:
4816 explicit SCEVBackedgeConditionFolder(const Loop *L, Value *BECond,
4817 bool IsPosBECond, ScalarEvolution &SE)
4818 : SCEVRewriteVisitor(SE), L(L), BackedgeCond(BECond),
4819 IsPositiveBECond(IsPosBECond) {}
4820
4821 Optional<const SCEV *> compareWithBackedgeCondition(Value *IC);
4822
4823 const Loop *L;
4824 /// Loop back condition.
4825 Value *BackedgeCond = nullptr;
4826 /// Set to true if loop back is on positive branch condition.
4827 bool IsPositiveBECond;
4828};
4829
4830Optional<const SCEV *>
4831SCEVBackedgeConditionFolder::compareWithBackedgeCondition(Value *IC) {
4832
4833 // If value matches the backedge condition for loop latch,
4834 // then return a constant evolution node based on loopback
4835 // branch taken.
4836 if (BackedgeCond == IC)
4837 return IsPositiveBECond ? SE.getOne(Type::getInt1Ty(SE.getContext()))
4838 : SE.getZero(Type::getInt1Ty(SE.getContext()));
4839 return None;
4840}
4841
4842class SCEVShiftRewriter : public SCEVRewriteVisitor<SCEVShiftRewriter> {
4843public:
4844 static const SCEV *rewrite(const SCEV *S, const Loop *L,
4845 ScalarEvolution &SE) {
4846 SCEVShiftRewriter Rewriter(L, SE);
4847 const SCEV *Result = Rewriter.visit(S);
4848 return Rewriter.isValid() ? Result : SE.getCouldNotCompute();
4849 }
4850
4851 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4852 // Only allow AddRecExprs for this loop.
4853 if (!SE.isLoopInvariant(Expr, L))
4854 Valid = false;
4855 return Expr;
4856 }
4857
4858 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4859 if (Expr->getLoop() == L && Expr->isAffine())
4860 return SE.getMinusSCEV(Expr, Expr->getStepRecurrence(SE));
4861 Valid = false;
4862 return Expr;
4863 }
4864
4865 bool isValid() { return Valid; }
4866
4867private:
4868 explicit SCEVShiftRewriter(const Loop *L, ScalarEvolution &SE)
4869 : SCEVRewriteVisitor(SE), L(L) {}
4870
4871 const Loop *L;
4872 bool Valid = true;
4873};
4874
4875} // end anonymous namespace
4876
4877SCEV::NoWrapFlags
4878ScalarEvolution::proveNoWrapViaConstantRanges(const SCEVAddRecExpr *AR) {
4879 if (!AR->isAffine())
4880 return SCEV::FlagAnyWrap;
4881
4882 using OBO = OverflowingBinaryOperator;
4883
4884 SCEV::NoWrapFlags Result = SCEV::FlagAnyWrap;
4885
4886 if (!AR->hasNoSignedWrap()) {
4887 ConstantRange AddRecRange = getSignedRange(AR);
4888 ConstantRange IncRange = getSignedRange(AR->getStepRecurrence(*this));
4889
4890 auto NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
4891 Instruction::Add, IncRange, OBO::NoSignedWrap);
4892 if (NSWRegion.contains(AddRecRange))
4893 Result = ScalarEvolution::setFlags(Result, SCEV::FlagNSW);
4894 }
4895
4896 if (!AR->hasNoUnsignedWrap()) {
4897 ConstantRange AddRecRange = getUnsignedRange(AR);
4898 ConstantRange IncRange = getUnsignedRange(AR->getStepRecurrence(*this));
4899
4900 auto NUWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
4901 Instruction::Add, IncRange, OBO::NoUnsignedWrap);
4902 if (NUWRegion.contains(AddRecRange))
4903 Result = ScalarEvolution::setFlags(Result, SCEV::FlagNUW);
4904 }
4905
4906 return Result;
4907}
4908
4909SCEV::NoWrapFlags
4910ScalarEvolution::proveNoSignedWrapViaInduction(const SCEVAddRecExpr *AR) {
4911 SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
4912
4913 if (AR->hasNoSignedWrap())
4914 return Result;
4915
4916 if (!AR->isAffine())
4917 return Result;
4918
4919 const SCEV *Step = AR->getStepRecurrence(*this);
4920 const Loop *L = AR->getLoop();
4921
4922 // Check whether the backedge-taken count is SCEVCouldNotCompute.
4923 // Note that this serves two purposes: It filters out loops that are
4924 // simply not analyzable, and it covers the case where this code is
4925 // being called from within backedge-taken count analysis, such that
4926 // attempting to ask for the backedge-taken count would likely result
4927 // in infinite recursion. In the later case, the analysis code will
4928 // cope with a conservative value, and it will take care to purge
4929 // that value once it has finished.
4930 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
4931
4932 // Normally, in the cases we can prove no-overflow via a
4933 // backedge guarding condition, we can also compute a backedge
4934 // taken count for the loop. The exceptions are assumptions and
4935 // guards present in the loop -- SCEV is not great at exploiting
4936 // these to compute max backedge taken counts, but can still use
4937 // these to prove lack of overflow. Use this fact to avoid
4938 // doing extra work that may not pay off.
4939
4940 if (isa<SCEVCouldNotCompute>(MaxBECount) && !HasGuards &&
4941 AC.assumptions().empty())
4942 return Result;
4943
4944 // If the backedge is guarded by a comparison with the pre-inc value the
4945 // addrec is safe. Also, if the entry is guarded by a comparison with the
4946 // start value and the backedge is guarded by a comparison with the post-inc
4947 // value, the addrec is safe.
4948 ICmpInst::Predicate Pred;
4949 const SCEV *OverflowLimit =
4950 getSignedOverflowLimitForStep(Step, &Pred, this);
4951 if (OverflowLimit &&
4952 (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) ||
4953 isKnownOnEveryIteration(Pred, AR, OverflowLimit))) {
4954 Result = setFlags(Result, SCEV::FlagNSW);
4955 }
4956 return Result;
4957}
4958SCEV::NoWrapFlags
4959ScalarEvolution::proveNoUnsignedWrapViaInduction(const SCEVAddRecExpr *AR) {
4960 SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
4961
4962 if (AR->hasNoUnsignedWrap())
4963 return Result;
4964
4965 if (!AR->isAffine())
4966 return Result;
4967
4968 const SCEV *Step = AR->getStepRecurrence(*this);
4969 unsigned BitWidth = getTypeSizeInBits(AR->getType());
4970 const Loop *L = AR->getLoop();
4971
4972 // Check whether the backedge-taken count is SCEVCouldNotCompute.
4973 // Note that this serves two purposes: It filters out loops that are
4974 // simply not analyzable, and it covers the case where this code is
4975 // being called from within backedge-taken count analysis, such that
4976 // attempting to ask for the backedge-taken count would likely result
4977 // in infinite recursion. In the later case, the analysis code will
4978 // cope with a conservative value, and it will take care to purge
4979 // that value once it has finished.
4980 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
4981
4982 // Normally, in the cases we can prove no-overflow via a
4983 // backedge guarding condition, we can also compute a backedge
4984 // taken count for the loop. The exceptions are assumptions and
4985 // guards present in the loop -- SCEV is not great at exploiting
4986 // these to compute max backedge taken counts, but can still use
4987 // these to prove lack of overflow. Use this fact to avoid
4988 // doing extra work that may not pay off.
4989
4990 if (isa<SCEVCouldNotCompute>(MaxBECount) && !HasGuards &&
4991 AC.assumptions().empty())
4992 return Result;
4993
4994 // If the backedge is guarded by a comparison with the pre-inc value the
4995 // addrec is safe. Also, if the entry is guarded by a comparison with the
4996 // start value and the backedge is guarded by a comparison with the post-inc
4997 // value, the addrec is safe.
4998 if (isKnownPositive(Step)) {
4999 const SCEV *N = getConstant(APInt::getMinValue(BitWidth) -
5000 getUnsignedRangeMax(Step));
5001 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, AR, N) ||
5002 isKnownOnEveryIteration(ICmpInst::ICMP_ULT, AR, N)) {
5003 Result = setFlags(Result, SCEV::FlagNUW);
5004 }
5005 }
5006
5007 return Result;
5008}
5009
5010namespace {
5011
5012/// Represents an abstract binary operation. This may exist as a
5013/// normal instruction or constant expression, or may have been
5014/// derived from an expression tree.
5015struct BinaryOp {
5016 unsigned Opcode;
5017 Value *LHS;
5018 Value *RHS;
5019 bool IsNSW = false;
5020 bool IsNUW = false;
5021
5022 /// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or
5023 /// constant expression.
5024 Operator *Op = nullptr;
5025
5026 explicit BinaryOp(Operator *Op)
5027 : Opcode(Op->getOpcode()), LHS(Op->getOperand(0)), RHS(Op->getOperand(1)),
5028 Op(Op) {
5029 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Op)) {
5030 IsNSW = OBO->hasNoSignedWrap();
5031 IsNUW = OBO->hasNoUnsignedWrap();
5032 }
5033 }
5034
5035 explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false,
5036 bool IsNUW = false)
5037 : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW) {}
5038};
5039
5040} // end anonymous namespace
5041
5042/// Try to map \p V into a BinaryOp, and return \c None on failure.
5043static Optional<BinaryOp> MatchBinaryOp(Value *V, DominatorTree &DT) {
5044 auto *Op = dyn_cast<Operator>(V);
5045 if (!Op)
5046 return None;
5047
5048 // Implementation detail: all the cleverness here should happen without
5049 // creating new SCEV expressions -- our caller knowns tricks to avoid creating
5050 // SCEV expressions when possible, and we should not break that.
5051
5052 switch (Op->getOpcode()) {
5053 case Instruction::Add:
5054 case Instruction::Sub:
5055 case Instruction::Mul:
5056 case Instruction::UDiv:
5057 case Instruction::URem:
5058 case Instruction::And:
5059 case Instruction::Or:
5060 case Instruction::AShr:
5061 case Instruction::Shl:
5062 return BinaryOp(Op);
5063
5064 case Instruction::Xor:
5065 if (auto *RHSC = dyn_cast<ConstantInt>(Op->getOperand(1)))
5066 // If the RHS of the xor is a signmask, then this is just an add.
5067 // Instcombine turns add of signmask into xor as a strength reduction step.
5068 if (RHSC->getValue().isSignMask())
5069 return BinaryOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1));
5070 return BinaryOp(Op);
5071
5072 case Instruction::LShr:
5073 // Turn logical shift right of a constant into a unsigned divide.
5074 if (ConstantInt *SA = dyn_cast<ConstantInt>(Op->getOperand(1))) {
5075 uint32_t BitWidth = cast<IntegerType>(Op->getType())->getBitWidth();
5076
5077 // If the shift count is not less than the bitwidth, the result of
5078 // the shift is undefined. Don't try to analyze it, because the
5079 // resolution chosen here may differ from the resolution chosen in
5080 // other parts of the compiler.
5081 if (SA->getValue().ult(BitWidth)) {
5082 Constant *X =
5083 ConstantInt::get(SA->getContext(),
5084 APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
5085 return BinaryOp(Instruction::UDiv, Op->getOperand(0), X);
5086 }
5087 }
5088 return BinaryOp(Op);
5089
5090 case Instruction::ExtractValue: {
5091 auto *EVI = cast<ExtractValueInst>(Op);
5092 if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0)
5093 break;
5094
5095 auto *WO = dyn_cast<WithOverflowInst>(EVI->getAggregateOperand());
5096 if (!WO)
5097 break;
5098
5099 Instruction::BinaryOps BinOp = WO->getBinaryOp();
5100 bool Signed = WO->isSigned();
5101 // TODO: Should add nuw/nsw flags for mul as well.
5102 if (BinOp == Instruction::Mul || !isOverflowIntrinsicNoWrap(WO, DT))
5103 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS());
5104
5105 // Now that we know that all uses of the arithmetic-result component of
5106 // CI are guarded by the overflow check, we can go ahead and pretend
5107 // that the arithmetic is non-overflowing.
5108 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS(),
5109 /* IsNSW = */ Signed, /* IsNUW = */ !Signed);
5110 }
5111
5112 default:
5113 break;
5114 }
5115
5116 // Recognise intrinsic loop.decrement.reg, and as this has exactly the same
5117 // semantics as a Sub, return a binary sub expression.
5118 if (auto *II = dyn_cast<IntrinsicInst>(V))
5119 if (II->getIntrinsicID() == Intrinsic::loop_decrement_reg)
5120 return BinaryOp(Instruction::Sub, II->getOperand(0), II->getOperand(1));
5121
5122 return None;
5123}
5124
5125/// Helper function to createAddRecFromPHIWithCasts. We have a phi
5126/// node whose symbolic (unknown) SCEV is \p SymbolicPHI, which is updated via
5127/// the loop backedge by a SCEVAddExpr, possibly also with a few casts on the
5128/// way. This function checks if \p Op, an operand of this SCEVAddExpr,
5129/// follows one of the following patterns:
5130/// Op == (SExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5131/// Op == (ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5132/// If the SCEV expression of \p Op conforms with one of the expected patterns
5133/// we return the type of the truncation operation, and indicate whether the
5134/// truncated type should be treated as signed/unsigned by setting
5135/// \p Signed to true/false, respectively.
5136static Type *isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI,
5137 bool &Signed, ScalarEvolution &SE) {
5138 // The case where Op == SymbolicPHI (that is, with no type conversions on
5139 // the way) is handled by the regular add recurrence creating logic and
5140 // would have already been triggered in createAddRecForPHI. Reaching it here
5141 // means that createAddRecFromPHI had failed for this PHI before (e.g.,
5142 // because one of the other operands of the SCEVAddExpr updating this PHI is
5143 // not invariant).
5144 //
5145 // Here we look for the case where Op = (ext(trunc(SymbolicPHI))), and in
5146 // this case predicates that allow us to prove that Op == SymbolicPHI will
5147 // be added.
5148 if (Op == SymbolicPHI)
5149 return nullptr;
5150
5151 unsigned SourceBits = SE.getTypeSizeInBits(SymbolicPHI->getType());
5152 unsigned NewBits = SE.getTypeSizeInBits(Op->getType());
5153 if (SourceBits != NewBits)
5154 return nullptr;
5155
5156 const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(Op);
5157 const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(Op);
5158 if (!SExt && !ZExt)
5159 return nullptr;
5160 const SCEVTruncateExpr *Trunc =
5161 SExt ? dyn_cast<SCEVTruncateExpr>(SExt->getOperand())
5162 : dyn_cast<SCEVTruncateExpr>(ZExt->getOperand());
5163 if (!Trunc)
5164 return nullptr;
5165 const SCEV *X = Trunc->getOperand();
5166 if (X != SymbolicPHI)
5167 return nullptr;
5168 Signed = SExt != nullptr;
5169 return Trunc->getType();
5170}
5171
5172static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) {
5173 if (!PN->getType()->isIntegerTy())
5174 return nullptr;
5175 const Loop *L = LI.getLoopFor(PN->getParent());
5176 if (!L || L->getHeader() != PN->getParent())
5177 return nullptr;
5178 return L;
5179}
5180
5181// Analyze \p SymbolicPHI, a SCEV expression of a phi node, and check if the
5182// computation that updates the phi follows the following pattern:
5183// (SExt/ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy) + InvariantAccum
5184// which correspond to a phi->trunc->sext/zext->add->phi update chain.
5185// If so, try to see if it can be rewritten as an AddRecExpr under some
5186// Predicates. If successful, return them as a pair. Also cache the results
5187// of the analysis.
5188//
5189// Example usage scenario:
5190// Say the Rewriter is called for the following SCEV:
5191// 8 * ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5192// where:
5193// %X = phi i64 (%Start, %BEValue)
5194// It will visitMul->visitAdd->visitSExt->visitTrunc->visitUnknown(%X),
5195// and call this function with %SymbolicPHI = %X.
5196//
5197// The analysis will find that the value coming around the backedge has
5198// the following SCEV:
5199// BEValue = ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5200// Upon concluding that this matches the desired pattern, the function
5201// will return the pair {NewAddRec, SmallPredsVec} where:
5202// NewAddRec = {%Start,+,%Step}
5203// SmallPredsVec = {P1, P2, P3} as follows:
5204// P1(WrapPred): AR: {trunc(%Start),+,(trunc %Step)}<nsw> Flags: <nssw>
5205// P2(EqualPred): %Start == (sext i32 (trunc i64 %Start to i32) to i64)
5206// P3(EqualPred): %Step == (sext i32 (trunc i64 %Step to i32) to i64)
5207// The returned pair means that SymbolicPHI can be rewritten into NewAddRec
5208// under the predicates {P1,P2,P3}.
5209// This predicated rewrite will be cached in PredicatedSCEVRewrites:
5210// PredicatedSCEVRewrites[{%X,L}] = {NewAddRec, {P1,P2,P3)}
5211//
5212// TODO's:
5213//
5214// 1) Extend the Induction descriptor to also support inductions that involve
5215// casts: When needed (namely, when we are called in the context of the
5216// vectorizer induction analysis), a Set of cast instructions will be
5217// populated by this method, and provided back to isInductionPHI. This is
5218// needed to allow the vectorizer to properly record them to be ignored by
5219// the cost model and to avoid vectorizing them (otherwise these casts,
5220// which are redundant under the runtime overflow checks, will be
5221// vectorized, which can be costly).
5222//
5223// 2) Support additional induction/PHISCEV patterns: We also want to support
5224// inductions where the sext-trunc / zext-trunc operations (partly) occur
5225// after the induction update operation (the induction increment):
5226//
5227// (Trunc iy (SExt/ZExt ix (%SymbolicPHI + InvariantAccum) to iy) to ix)
5228// which correspond to a phi->add->trunc->sext/zext->phi update chain.
5229//
5230// (Trunc iy ((SExt/ZExt ix (%SymbolicPhi) to iy) + InvariantAccum) to ix)
5231// which correspond to a phi->trunc->add->sext/zext->phi update chain.
5232//
5233// 3) Outline common code with createAddRecFromPHI to avoid duplication.
5234Optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5235ScalarEvolution::createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI) {
5236 SmallVector<const SCEVPredicate *, 3> Predicates;
5237
5238 // *** Part1: Analyze if we have a phi-with-cast pattern for which we can
5239 // return an AddRec expression under some predicate.
5240
5241 auto *PN = cast<PHINode>(SymbolicPHI->getValue());
5242 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5243 assert(L && "Expecting an integer loop header phi")(static_cast <bool> (L && "Expecting an integer loop header phi"
) ? void (0) : __assert_fail ("L && \"Expecting an integer loop header phi\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 5243, __extension__
__PRETTY_FUNCTION__))
;
5244
5245 // The loop may have multiple entrances or multiple exits; we can analyze
5246 // this phi as an addrec if it has a unique entry value and a unique
5247 // backedge value.
5248 Value *BEValueV = nullptr, *StartValueV = nullptr;
5249 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5250 Value *V = PN->getIncomingValue(i);
5251 if (L->contains(PN->getIncomingBlock(i))) {
5252 if (!BEValueV) {
5253 BEValueV = V;
5254 } else if (BEValueV != V) {
5255 BEValueV = nullptr;
5256 break;
5257 }
5258 } else if (!StartValueV) {
5259 StartValueV = V;
5260 } else if (StartValueV != V) {
5261 StartValueV = nullptr;
5262 break;
5263 }
5264 }
5265 if (!BEValueV || !StartValueV)
5266 return None;
5267
5268 const SCEV *BEValue = getSCEV(BEValueV);
5269
5270 // If the value coming around the backedge is an add with the symbolic
5271 // value we just inserted, possibly with casts that we can ignore under
5272 // an appropriate runtime guard, then we found a simple induction variable!
5273 const auto *Add = dyn_cast<SCEVAddExpr>(BEValue);
5274 if (!Add)
5275 return None;
5276
5277 // If there is a single occurrence of the symbolic value, possibly
5278 // casted, replace it with a recurrence.
5279 unsigned FoundIndex = Add->getNumOperands();
5280 Type *TruncTy = nullptr;
5281 bool Signed;
5282 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5283 if ((TruncTy =
5284 isSimpleCastedPHI(Add->getOperand(i), SymbolicPHI, Signed, *this)))
5285 if (FoundIndex == e) {
5286 FoundIndex = i;
5287 break;
5288 }
5289
5290 if (FoundIndex == Add->getNumOperands())
5291 return None;
5292
5293 // Create an add with everything but the specified operand.
5294 SmallVector<const SCEV *, 8> Ops;
5295 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5296 if (i != FoundIndex)
5297 Ops.push_back(Add->getOperand(i));
5298 const SCEV *Accum = getAddExpr(Ops);
5299
5300 // The runtime checks will not be valid if the step amount is
5301 // varying inside the loop.
5302 if (!isLoopInvariant(Accum, L))
5303 return None;
5304
5305 // *** Part2: Create the predicates
5306
5307 // Analysis was successful: we have a phi-with-cast pattern for which we
5308 // can return an AddRec expression under the following predicates:
5309 //
5310 // P1: A Wrap predicate that guarantees that Trunc(Start) + i*Trunc(Accum)
5311 // fits within the truncated type (does not overflow) for i = 0 to n-1.
5312 // P2: An Equal predicate that guarantees that
5313 // Start = (Ext ix (Trunc iy (Start) to ix) to iy)
5314 // P3: An Equal predicate that guarantees that
5315 // Accum = (Ext ix (Trunc iy (Accum) to ix) to iy)
5316 //
5317 // As we next prove, the above predicates guarantee that:
5318 // Start + i*Accum = (Ext ix (Trunc iy ( Start + i*Accum ) to ix) to iy)
5319 //
5320 //
5321 // More formally, we want to prove that:
5322 // Expr(i+1) = Start + (i+1) * Accum
5323 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5324 //
5325 // Given that:
5326 // 1) Expr(0) = Start
5327 // 2) Expr(1) = Start + Accum
5328 // = (Ext ix (Trunc iy (Start) to ix) to iy) + Accum :: from P2
5329 // 3) Induction hypothesis (step i):
5330 // Expr(i) = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum
5331 //
5332 // Proof:
5333 // Expr(i+1) =
5334 // = Start + (i+1)*Accum
5335 // = (Start + i*Accum) + Accum
5336 // = Expr(i) + Accum
5337 // = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum + Accum
5338 // :: from step i
5339 //
5340 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy) + Accum + Accum
5341 //
5342 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy)
5343 // + (Ext ix (Trunc iy (Accum) to ix) to iy)
5344 // + Accum :: from P3
5345 //
5346 // = (Ext ix (Trunc iy ((Start + (i-1)*Accum) + Accum) to ix) to iy)
5347 // + Accum :: from P1: Ext(x)+Ext(y)=>Ext(x+y)
5348 //
5349 // = (Ext ix (Trunc iy (Start + i*Accum) to ix) to iy) + Accum
5350 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5351 //
5352 // By induction, the same applies to all iterations 1<=i<n:
5353 //
5354
5355 // Create a truncated addrec for which we will add a no overflow check (P1).
5356 const SCEV *StartVal = getSCEV(StartValueV);
5357 const SCEV *PHISCEV =
5358 getAddRecExpr(getTruncateExpr(StartVal, TruncTy),
5359 getTruncateExpr(Accum, TruncTy), L, SCEV::FlagAnyWrap);
5360
5361 // PHISCEV can be either a SCEVConstant or a SCEVAddRecExpr.
5362 // ex: If truncated Accum is 0 and StartVal is a constant, then PHISCEV
5363 // will be constant.
5364 //
5365 // If PHISCEV is a constant, then P1 degenerates into P2 or P3, so we don't
5366 // add P1.
5367 if (const auto *AR = dyn_cast<SCEVAddRecExpr>(PHISCEV)) {
5368 SCEVWrapPredicate::IncrementWrapFlags AddedFlags =
5369 Signed ? SCEVWrapPredicate::IncrementNSSW
5370 : SCEVWrapPredicate::IncrementNUSW;
5371 const SCEVPredicate *AddRecPred = getWrapPredicate(AR, AddedFlags);
5372 Predicates.push_back(AddRecPred);
5373 }
5374
5375 // Create the Equal Predicates P2,P3:
5376
5377 // It is possible that the predicates P2 and/or P3 are computable at
5378 // compile time due to StartVal and/or Accum being constants.
5379 // If either one is, then we can check that now and escape if either P2
5380 // or P3 is false.
5381
5382 // Construct the extended SCEV: (Ext ix (Trunc iy (Expr) to ix) to iy)
5383 // for each of StartVal and Accum
5384 auto getExtendedExpr = [&](const SCEV *Expr,
5385 bool CreateSignExtend) -> const SCEV * {
5386 assert(isLoopInvariant(Expr, L) && "Expr is expected to be invariant")(static_cast <bool> (isLoopInvariant(Expr, L) &&
"Expr is expected to be invariant") ? void (0) : __assert_fail
("isLoopInvariant(Expr, L) && \"Expr is expected to be invariant\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 5386, __extension__
__PRETTY_FUNCTION__))
;
5387 const SCEV *TruncatedExpr = getTruncateExpr(Expr, TruncTy);
5388 const SCEV *ExtendedExpr =
5389 CreateSignExtend ? getSignExtendExpr(TruncatedExpr, Expr->getType())
5390 : getZeroExtendExpr(TruncatedExpr, Expr->getType());
5391 return ExtendedExpr;
5392 };
5393
5394 // Given:
5395 // ExtendedExpr = (Ext ix (Trunc iy (Expr) to ix) to iy
5396 // = getExtendedExpr(Expr)
5397 // Determine whether the predicate P: Expr == ExtendedExpr
5398 // is known to be false at compile time
5399 auto PredIsKnownFalse = [&](const SCEV *Expr,
5400 const SCEV *ExtendedExpr) -> bool {
5401 return Expr != ExtendedExpr &&
5402 isKnownPredicate(ICmpInst::ICMP_NE, Expr, ExtendedExpr);
5403 };
5404
5405 const SCEV *StartExtended = getExtendedExpr(StartVal, Signed);
5406 if (PredIsKnownFalse(StartVal, StartExtended)) {
5407 LLVM_DEBUG(dbgs() << "P2 is compile-time false\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << "P2 is compile-time false\n"
;; } } while (false)
;
5408 return None;
5409 }
5410
5411 // The Step is always Signed (because the overflow checks are either
5412 // NSSW or NUSW)
5413 const SCEV *AccumExtended = getExtendedExpr(Accum, /*CreateSignExtend=*/true);
5414 if (PredIsKnownFalse(Accum, AccumExtended)) {
5415 LLVM_DEBUG(dbgs() << "P3 is compile-time false\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << "P3 is compile-time false\n"
;; } } while (false)
;
5416 return None;
5417 }
5418
5419 auto AppendPredicate = [&](const SCEV *Expr,
5420 const SCEV *ExtendedExpr) -> void {
5421 if (Expr != ExtendedExpr &&
5422 !isKnownPredicate(ICmpInst::ICMP_EQ, Expr, ExtendedExpr)) {
5423 const SCEVPredicate *Pred = getEqualPredicate(Expr, ExtendedExpr);
5424 LLVM_DEBUG(dbgs() << "Added Predicate: " << *Pred)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << "Added Predicate: " <<
*Pred; } } while (false)
;
5425 Predicates.push_back(Pred);
5426 }
5427 };
5428
5429 AppendPredicate(StartVal, StartExtended);
5430 AppendPredicate(Accum, AccumExtended);
5431
5432 // *** Part3: Predicates are ready. Now go ahead and create the new addrec in
5433 // which the casts had been folded away. The caller can rewrite SymbolicPHI
5434 // into NewAR if it will also add the runtime overflow checks specified in
5435 // Predicates.
5436 auto *NewAR = getAddRecExpr(StartVal, Accum, L, SCEV::FlagAnyWrap);
5437
5438 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> PredRewrite =
5439 std::make_pair(NewAR, Predicates);
5440 // Remember the result of the analysis for this SCEV at this locayyytion.
5441 PredicatedSCEVRewrites[{SymbolicPHI, L}] = PredRewrite;
5442 return PredRewrite;
5443}
5444
5445Optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5446ScalarEvolution::createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI) {
5447 auto *PN = cast<PHINode>(SymbolicPHI->getValue());
5448 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5449 if (!L)
5450 return None;
5451
5452 // Check to see if we already analyzed this PHI.
5453 auto I = PredicatedSCEVRewrites.find({SymbolicPHI, L});
5454 if (I != PredicatedSCEVRewrites.end()) {
5455 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> Rewrite =
5456 I->second;
5457 // Analysis was done before and failed to create an AddRec:
5458 if (Rewrite.first == SymbolicPHI)
5459 return None;
5460 // Analysis was done before and succeeded to create an AddRec under
5461 // a predicate:
5462 assert(isa<SCEVAddRecExpr>(Rewrite.first) && "Expected an AddRec")(static_cast <bool> (isa<SCEVAddRecExpr>(Rewrite.
first) && "Expected an AddRec") ? void (0) : __assert_fail
("isa<SCEVAddRecExpr>(Rewrite.first) && \"Expected an AddRec\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 5462, __extension__
__PRETTY_FUNCTION__))
;
5463 assert(!(Rewrite.second).empty() && "Expected to find Predicates")(static_cast <bool> (!(Rewrite.second).empty() &&
"Expected to find Predicates") ? void (0) : __assert_fail ("!(Rewrite.second).empty() && \"Expected to find Predicates\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 5463, __extension__
__PRETTY_FUNCTION__))
;
5464 return Rewrite;
5465 }
5466
5467 Optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5468 Rewrite = createAddRecFromPHIWithCastsImpl(SymbolicPHI);
5469
5470 // Record in the cache that the analysis failed
5471 if (!Rewrite) {
5472 SmallVector<const SCEVPredicate *, 3> Predicates;
5473 PredicatedSCEVRewrites[{SymbolicPHI, L}] = {SymbolicPHI, Predicates};
5474 return None;
5475 }
5476
5477 return Rewrite;
5478}
5479
5480// FIXME: This utility is currently required because the Rewriter currently
5481// does not rewrite this expression:
5482// {0, +, (sext ix (trunc iy to ix) to iy)}
5483// into {0, +, %step},
5484// even when the following Equal predicate exists:
5485// "%step == (sext ix (trunc iy to ix) to iy)".
5486bool PredicatedScalarEvolution::areAddRecsEqualWithPreds(
5487 const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2) const {
5488 if (AR1 == AR2)
5489 return true;
5490
5491 auto areExprsEqual = [&](const SCEV *Expr1, const SCEV *Expr2) -> bool {
5492 if (Expr1 != Expr2 && !Preds.implies(SE.getEqualPredicate(Expr1, Expr2)) &&
5493 !Preds.implies(SE.getEqualPredicate(Expr2, Expr1)))
5494 return false;
5495 return true;
5496 };
5497
5498 if (!areExprsEqual(AR1->getStart(), AR2->getStart()) ||
5499 !areExprsEqual(AR1->getStepRecurrence(SE), AR2->getStepRecurrence(SE)))
5500 return false;
5501 return true;
5502}
5503
5504/// A helper function for createAddRecFromPHI to handle simple cases.
5505///
5506/// This function tries to find an AddRec expression for the simplest (yet most
5507/// common) cases: PN = PHI(Start, OP(Self, LoopInvariant)).
5508/// If it fails, createAddRecFromPHI will use a more general, but slow,
5509/// technique for finding the AddRec expression.
5510const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
5511 Value *BEValueV,
5512 Value *StartValueV) {
5513 const Loop *L = LI.getLoopFor(PN->getParent());
5514 assert(L && L->getHeader() == PN->getParent())(static_cast <bool> (L && L->getHeader() == PN
->getParent()) ? void (0) : __assert_fail ("L && L->getHeader() == PN->getParent()"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 5514, __extension__
__PRETTY_FUNCTION__))
;
5515 assert(BEValueV && StartValueV)(static_cast <bool> (BEValueV && StartValueV) ?
void (0) : __assert_fail ("BEValueV && StartValueV",
"llvm/lib/Analysis/ScalarEvolution.cpp", 5515, __extension__
__PRETTY_FUNCTION__))
;
5516
5517 auto BO = MatchBinaryOp(BEValueV, DT);
5518 if (!BO)
5519 return nullptr;
5520
5521 if (BO->Opcode != Instruction::Add)
5522 return nullptr;
5523
5524 const SCEV *Accum = nullptr;
5525 if (BO->LHS == PN && L->isLoopInvariant(BO->RHS))
5526 Accum = getSCEV(BO->RHS);
5527 else if (BO->RHS == PN && L->isLoopInvariant(BO->LHS))
5528 Accum = getSCEV(BO->LHS);
5529
5530 if (!Accum)
5531 return nullptr;
5532
5533 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
5534 if (BO->IsNUW)
5535 Flags = setFlags(Flags, SCEV::FlagNUW);
5536 if (BO->IsNSW)
5537 Flags = setFlags(Flags, SCEV::FlagNSW);
5538
5539 const SCEV *StartVal = getSCEV(StartValueV);
5540 const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags);
5541 insertValueToMap(PN, PHISCEV);
5542
5543 // We can add Flags to the post-inc expression only if we
5544 // know that it is *undefined behavior* for BEValueV to
5545 // overflow.
5546 if (auto *BEInst = dyn_cast<Instruction>(BEValueV)) {
5547 assert(isLoopInvariant(Accum, L) &&(static_cast <bool> (isLoopInvariant(Accum, L) &&
"Accum is defined outside L, but is not invariant?") ? void (
0) : __assert_fail ("isLoopInvariant(Accum, L) && \"Accum is defined outside L, but is not invariant?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 5548, __extension__
__PRETTY_FUNCTION__))
5548 "Accum is defined outside L, but is not invariant?")(static_cast <bool> (isLoopInvariant(Accum, L) &&
"Accum is defined outside L, but is not invariant?") ? void (
0) : __assert_fail ("isLoopInvariant(Accum, L) && \"Accum is defined outside L, but is not invariant?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 5548, __extension__
__PRETTY_FUNCTION__))
;
5549 if (isAddRecNeverPoison(BEInst, L))
5550 (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, Flags);
5551 }
5552
5553 return PHISCEV;
5554}
5555
5556const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
5557 const Loop *L = LI.getLoopFor(PN->getParent());
5558 if (!L || L->getHeader() != PN->getParent())
5559 return nullptr;
5560
5561 // The loop may have multiple entrances or multiple exits; we can analyze
5562 // this phi as an addrec if it has a unique entry value and a unique
5563 // backedge value.
5564 Value *BEValueV = nullptr, *StartValueV = nullptr;
5565 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5566 Value *V = PN->getIncomingValue(i);
5567 if (L->contains(PN->getIncomingBlock(i))) {
5568 if (!BEValueV) {
5569 BEValueV = V;
5570 } else if (BEValueV != V) {
5571 BEValueV = nullptr;
5572 break;
5573 }
5574 } else if (!StartValueV) {
5575 StartValueV = V;
5576 } else if (StartValueV != V) {
5577 StartValueV = nullptr;
5578 break;
5579 }
5580 }
5581 if (!BEValueV || !StartValueV)
5582 return nullptr;
5583
5584 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&(static_cast <bool> (ValueExprMap.find_as(PN) == ValueExprMap
.end() && "PHI node already processed?") ? void (0) :
__assert_fail ("ValueExprMap.find_as(PN) == ValueExprMap.end() && \"PHI node already processed?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 5585, __extension__
__PRETTY_FUNCTION__))
5585 "PHI node already processed?")(static_cast <bool> (ValueExprMap.find_as(PN) == ValueExprMap
.end() && "PHI node already processed?") ? void (0) :
__assert_fail ("ValueExprMap.find_as(PN) == ValueExprMap.end() && \"PHI node already processed?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 5585, __extension__
__PRETTY_FUNCTION__))
;
5586
5587 // First, try to find AddRec expression without creating a fictituos symbolic
5588 // value for PN.
5589 if (auto *S = createSimpleAffineAddRec(PN, BEValueV, StartValueV))
5590 return S;
5591
5592 // Handle PHI node value symbolically.
5593 const SCEV *SymbolicName = getUnknown(PN);
5594 insertValueToMap(PN, SymbolicName);
5595
5596 // Using this symbolic name for the PHI, analyze the value coming around
5597 // the back-edge.
5598 const SCEV *BEValue = getSCEV(BEValueV);
5599
5600 // NOTE: If BEValue is loop invariant, we know that the PHI node just
5601 // has a special value for the first iteration of the loop.
5602
5603 // If the value coming around the backedge is an add with the symbolic
5604 // value we just inserted, then we found a simple induction variable!
5605 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BEValue)) {
5606 // If there is a single occurrence of the symbolic value, replace it
5607 // with a recurrence.
5608 unsigned FoundIndex = Add->getNumOperands();
5609 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5610 if (Add->getOperand(i) == SymbolicName)
5611 if (FoundIndex == e) {
5612 FoundIndex = i;
5613 break;
5614 }
5615
5616 if (FoundIndex != Add->getNumOperands()) {
5617 // Create an add with everything but the specified operand.
5618 SmallVector<const SCEV *, 8> Ops;
5619 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5620 if (i != FoundIndex)
5621 Ops.push_back(SCEVBackedgeConditionFolder::rewrite(Add->getOperand(i),
5622 L, *this));
5623 const SCEV *Accum = getAddExpr(Ops);
5624
5625 // This is not a valid addrec if the step amount is varying each
5626 // loop iteration, but is not itself an addrec in this loop.
5627 if (isLoopInvariant(Accum, L) ||
5628 (isa<SCEVAddRecExpr>(Accum) &&
5629 cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) {
5630 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
5631
5632 if (auto BO = MatchBinaryOp(BEValueV, DT)) {
5633 if (BO->Opcode == Instruction::Add && BO->LHS == PN) {
5634 if (BO->IsNUW)
5635 Flags = setFlags(Flags, SCEV::FlagNUW);
5636 if (BO->IsNSW)
5637 Flags = setFlags(Flags, SCEV::FlagNSW);
5638 }
5639 } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(BEValueV)) {
5640 // If the increment is an inbounds GEP, then we know the address
5641 // space cannot be wrapped around. We cannot make any guarantee
5642 // about signed or unsigned overflow because pointers are
5643 // unsigned but we may have a negative index from the base
5644 // pointer. We can guarantee that no unsigned wrap occurs if the
5645 // indices form a positive value.
5646 if (GEP->isInBounds() && GEP->getOperand(0) == PN) {
5647 Flags = setFlags(Flags, SCEV::FlagNW);
5648
5649 const SCEV *Ptr = getSCEV(GEP->getPointerOperand());
5650 if (isKnownPositive(getMinusSCEV(getSCEV(GEP), Ptr)))
5651 Flags = setFlags(Flags, SCEV::FlagNUW);
5652 }
5653
5654 // We cannot transfer nuw and nsw flags from subtraction
5655 // operations -- sub nuw X, Y is not the same as add nuw X, -Y
5656 // for instance.
5657 }
5658
5659 const SCEV *StartVal = getSCEV(StartValueV);
5660 const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags);
5661
5662 // Okay, for the entire analysis of this edge we assumed the PHI
5663 // to be symbolic. We now need to go back and purge all of the
5664 // entries for the scalars that use the symbolic expression.
5665 forgetMemoizedResults(SymbolicName);
5666 insertValueToMap(PN, PHISCEV);
5667
5668 // We can add Flags to the post-inc expression only if we
5669 // know that it is *undefined behavior* for BEValueV to
5670 // overflow.
5671 if (auto *BEInst = dyn_cast<Instruction>(BEValueV))
5672 if (isLoopInvariant(Accum, L) && isAddRecNeverPoison(BEInst, L))
5673 (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, Flags);
5674
5675 return PHISCEV;
5676 }
5677 }
5678 } else {
5679 // Otherwise, this could be a loop like this:
5680 // i = 0; for (j = 1; ..; ++j) { .... i = j; }
5681 // In this case, j = {1,+,1} and BEValue is j.
5682 // Because the other in-value of i (0) fits the evolution of BEValue
5683 // i really is an addrec evolution.
5684 //
5685 // We can generalize this saying that i is the shifted value of BEValue
5686 // by one iteration:
5687 // PHI(f(0), f({1,+,1})) --> f({0,+,1})
5688 const SCEV *Shifted = SCEVShiftRewriter::rewrite(BEValue, L, *this);
5689 const SCEV *Start = SCEVInitRewriter::rewrite(Shifted, L, *this, false);
5690 if (Shifted != getCouldNotCompute() &&
5691 Start != getCouldNotCompute()) {
5692 const SCEV *StartVal = getSCEV(StartValueV);
5693 if (Start == StartVal) {
5694 // Okay, for the entire analysis of this edge we assumed the PHI
5695 // to be symbolic. We now need to go back and purge all of the
5696 // entries for the scalars that use the symbolic expression.
5697 forgetMemoizedResults(SymbolicName);
5698 insertValueToMap(PN, Shifted);
5699 return Shifted;
5700 }
5701 }
5702 }
5703
5704 // Remove the temporary PHI node SCEV that has been inserted while intending
5705 // to create an AddRecExpr for this PHI node. We can not keep this temporary
5706 // as it will prevent later (possibly simpler) SCEV expressions to be added
5707 // to the ValueExprMap.
5708 eraseValueFromMap(PN);
5709
5710 return nullptr;
5711}
5712
5713// Checks if the SCEV S is available at BB. S is considered available at BB
5714// if S can be materialized at BB without introducing a fault.
5715static bool IsAvailableOnEntry(const Loop *L, DominatorTree &DT, const SCEV *S,
5716 BasicBlock *BB) {
5717 struct CheckAvailable {
5718 bool TraversalDone = false;
5719 bool Available = true;
5720
5721 const Loop *L = nullptr; // The loop BB is in (can be nullptr)
5722 BasicBlock *BB = nullptr;
5723 DominatorTree &DT;
5724
5725 CheckAvailable(const Loop *L, BasicBlock *BB, DominatorTree &DT)
5726 : L(L), BB(BB), DT(DT) {}
5727
5728 bool setUnavailable() {
5729 TraversalDone = true;
5730 Available = false;
5731 return false;
5732 }
5733
5734 bool follow(const SCEV *S) {
5735 switch (S->getSCEVType()) {
5736 case scConstant:
5737 case scPtrToInt:
5738 case scTruncate:
5739 case scZeroExtend:
5740 case scSignExtend:
5741 case scAddExpr:
5742 case scMulExpr:
5743 case scUMaxExpr:
5744 case scSMaxExpr:
5745 case scUMinExpr:
5746 case scSMinExpr:
5747 case scSequentialUMinExpr:
5748 // These expressions are available if their operand(s) is/are.
5749 return true;
5750
5751 case scAddRecExpr: {
5752 // We allow add recurrences that are on the loop BB is in, or some
5753 // outer loop. This guarantees availability because the value of the
5754 // add recurrence at BB is simply the "current" value of the induction
5755 // variable. We can relax this in the future; for instance an add
5756 // recurrence on a sibling dominating loop is also available at BB.
5757 const auto *ARLoop = cast<SCEVAddRecExpr>(S)->getLoop();
5758 if (L && (ARLoop == L || ARLoop->contains(L)))
5759 return true;
5760
5761 return setUnavailable();
5762 }
5763
5764 case scUnknown: {
5765 // For SCEVUnknown, we check for simple dominance.
5766 const auto *SU = cast<SCEVUnknown>(S);
5767 Value *V = SU->getValue();
5768
5769 if (isa<Argument>(V))
5770 return false;
5771
5772 if (isa<Instruction>(V) && DT.dominates(cast<Instruction>(V), BB))
5773 return false;
5774
5775 return setUnavailable();
5776 }
5777
5778 case scUDivExpr:
5779 case scCouldNotCompute:
5780 // We do not try to smart about these at all.
5781 return setUnavailable();
5782 }
5783 llvm_unreachable("Unknown SCEV kind!")::llvm::llvm_unreachable_internal("Unknown SCEV kind!", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 5783)
;
5784 }
5785
5786 bool isDone() { return TraversalDone; }
5787 };
5788
5789 CheckAvailable CA(L, BB, DT);
5790 SCEVTraversal<CheckAvailable> ST(CA);
5791
5792 ST.visitAll(S);
5793 return CA.Available;
5794}
5795
5796// Try to match a control flow sequence that branches out at BI and merges back
5797// at Merge into a "C ? LHS : RHS" select pattern. Return true on a successful
5798// match.
5799static bool BrPHIToSelect(DominatorTree &DT, BranchInst *BI, PHINode *Merge,
5800 Value *&C, Value *&LHS, Value *&RHS) {
5801 C = BI->getCondition();
5802
5803 BasicBlockEdge LeftEdge(BI->getParent(), BI->getSuccessor(0));
5804 BasicBlockEdge RightEdge(BI->getParent(), BI->getSuccessor(1));
5805
5806 if (!LeftEdge.isSingleEdge())
5807 return false;
5808
5809 assert(RightEdge.isSingleEdge() && "Follows from LeftEdge.isSingleEdge()")(static_cast <bool> (RightEdge.isSingleEdge() &&
"Follows from LeftEdge.isSingleEdge()") ? void (0) : __assert_fail
("RightEdge.isSingleEdge() && \"Follows from LeftEdge.isSingleEdge()\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 5809, __extension__
__PRETTY_FUNCTION__))
;
5810
5811 Use &LeftUse = Merge->getOperandUse(0);
5812 Use &RightUse = Merge->getOperandUse(1);
5813
5814 if (DT.dominates(LeftEdge, LeftUse) && DT.dominates(RightEdge, RightUse)) {
5815 LHS = LeftUse;
5816 RHS = RightUse;
5817 return true;
5818 }
5819
5820 if (DT.dominates(LeftEdge, RightUse) && DT.dominates(RightEdge, LeftUse)) {
5821 LHS = RightUse;
5822 RHS = LeftUse;
5823 return true;
5824 }
5825
5826 return false;
5827}
5828
5829const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
5830 auto IsReachable =
5831 [&](BasicBlock *BB) { return DT.isReachableFromEntry(BB); };
5832 if (PN->getNumIncomingValues() == 2 && all_of(PN->blocks(), IsReachable)) {
5833 const Loop *L = LI.getLoopFor(PN->getParent());
5834
5835 // We don't want to break LCSSA, even in a SCEV expression tree.
5836 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
5837 if (LI.getLoopFor(PN->getIncomingBlock(i)) != L)
5838 return nullptr;
5839
5840 // Try to match
5841 //
5842 // br %cond, label %left, label %right
5843 // left:
5844 // br label %merge
5845 // right:
5846 // br label %merge
5847 // merge:
5848 // V = phi [ %x, %left ], [ %y, %right ]
5849 //
5850 // as "select %cond, %x, %y"
5851
5852 BasicBlock *IDom = DT[PN->getParent()]->getIDom()->getBlock();
5853 assert(IDom && "At least the entry block should dominate PN")(static_cast <bool> (IDom && "At least the entry block should dominate PN"
) ? void (0) : __assert_fail ("IDom && \"At least the entry block should dominate PN\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 5853, __extension__
__PRETTY_FUNCTION__))
;
5854
5855 auto *BI = dyn_cast<BranchInst>(IDom->getTerminator());
5856 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
5857
5858 if (BI && BI->isConditional() &&
5859 BrPHIToSelect(DT, BI, PN, Cond, LHS, RHS) &&
5860 IsAvailableOnEntry(L, DT, getSCEV(LHS), PN->getParent()) &&
5861 IsAvailableOnEntry(L, DT, getSCEV(RHS), PN->getParent()))
5862 return createNodeForSelectOrPHI(PN, Cond, LHS, RHS);
5863 }
5864
5865 return nullptr;
5866}
5867
5868const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
5869 if (const SCEV *S = createAddRecFromPHI(PN))
5870 return S;
5871
5872 if (const SCEV *S = createNodeFromSelectLikePHI(PN))
5873 return S;
5874
5875 // If the PHI has a single incoming value, follow that value, unless the
5876 // PHI's incoming blocks are in a different loop, in which case doing so
5877 // risks breaking LCSSA form. Instcombine would normally zap these, but
5878 // it doesn't have DominatorTree information, so it may miss cases.
5879 if (Value *V = SimplifyInstruction(PN, {getDataLayout(), &TLI, &DT, &AC}))
5880 if (LI.replacementPreservesLCSSAForm(PN, V))
5881 return getSCEV(V);
5882
5883 // If it's not a loop phi, we can't handle it yet.
5884 return getUnknown(PN);
5885}
5886
5887const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Instruction *I,
5888 Value *Cond,
5889 Value *TrueVal,
5890 Value *FalseVal) {
5891 // Handle "constant" branch or select. This can occur for instance when a
5892 // loop pass transforms an inner loop and moves on to process the outer loop.
5893 if (auto *CI = dyn_cast<ConstantInt>(Cond))
5894 return getSCEV(CI->isOne() ? TrueVal : FalseVal);
5895
5896 // Try to match some simple smax or umax patterns.
5897 auto *ICI = dyn_cast<ICmpInst>(Cond);
5898 if (!ICI)
5899 return getUnknown(I);
5900
5901 Value *LHS = ICI->getOperand(0);
5902 Value *RHS = ICI->getOperand(1);
5903
5904 switch (ICI->getPredicate()) {
5905 case ICmpInst::ICMP_SLT:
5906 case ICmpInst::ICMP_SLE:
5907 case ICmpInst::ICMP_ULT:
5908 case ICmpInst::ICMP_ULE:
5909 std::swap(LHS, RHS);
5910 LLVM_FALLTHROUGH[[gnu::fallthrough]];
5911 case ICmpInst::ICMP_SGT:
5912 case ICmpInst::ICMP_SGE:
5913 case ICmpInst::ICMP_UGT:
5914 case ICmpInst::ICMP_UGE:
5915 // a > b ? a+x : b+x -> max(a, b)+x
5916 // a > b ? b+x : a+x -> min(a, b)+x
5917 if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(I->getType())) {
5918 bool Signed = ICI->isSigned();
5919 const SCEV *LA = getSCEV(TrueVal);
5920 const SCEV *RA = getSCEV(FalseVal);
5921 const SCEV *LS = getSCEV(LHS);
5922 const SCEV *RS = getSCEV(RHS);
5923 if (LA->getType()->isPointerTy()) {
5924 // FIXME: Handle cases where LS/RS are pointers not equal to LA/RA.
5925 // Need to make sure we can't produce weird expressions involving
5926 // negated pointers.
5927 if (LA == LS && RA == RS)
5928 return Signed ? getSMaxExpr(LS, RS) : getUMaxExpr(LS, RS);
5929 if (LA == RS && RA == LS)
5930 return Signed ? getSMinExpr(LS, RS) : getUMinExpr(LS, RS);
5931 }
5932 auto CoerceOperand = [&](const SCEV *Op) -> const SCEV * {
5933 if (Op->getType()->isPointerTy()) {
5934 Op = getLosslessPtrToIntExpr(Op);
5935 if (isa<SCEVCouldNotCompute>(Op))
5936 return Op;
5937 }
5938 if (Signed)
5939 Op = getNoopOrSignExtend(Op, I->getType());
5940 else
5941 Op = getNoopOrZeroExtend(Op, I->getType());
5942 return Op;
5943 };
5944 LS = CoerceOperand(LS);
5945 RS = CoerceOperand(RS);
5946 if (isa<SCEVCouldNotCompute>(LS) || isa<SCEVCouldNotCompute>(RS))
5947 break;
5948 const SCEV *LDiff = getMinusSCEV(LA, LS);
5949 const SCEV *RDiff = getMinusSCEV(RA, RS);
5950 if (LDiff == RDiff)
5951 return getAddExpr(Signed ? getSMaxExpr(LS, RS) : getUMaxExpr(LS, RS),
5952 LDiff);
5953 LDiff = getMinusSCEV(LA, RS);
5954 RDiff = getMinusSCEV(RA, LS);
5955 if (LDiff == RDiff)
5956 return getAddExpr(Signed ? getSMinExpr(LS, RS) : getUMinExpr(LS, RS),
5957 LDiff);
5958 }
5959 break;
5960 case ICmpInst::ICMP_NE:
5961 // n != 0 ? n+x : 1+x -> umax(n, 1)+x
5962 if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(I->getType()) &&
5963 isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero()) {
5964 const SCEV *One = getOne(I->getType());
5965 const SCEV *LS = getNoopOrZeroExtend(getSCEV(LHS), I->getType());
5966 const SCEV *LA = getSCEV(TrueVal);
5967 const SCEV *RA = getSCEV(FalseVal);
5968 const SCEV *LDiff = getMinusSCEV(LA, LS);
5969 const SCEV *RDiff = getMinusSCEV(RA, One);
5970 if (LDiff == RDiff)
5971 return getAddExpr(getUMaxExpr(One, LS), LDiff);
5972 }
5973 break;
5974 case ICmpInst::ICMP_EQ:
5975 // n == 0 ? 1+x : n+x -> umax(n, 1)+x
5976 if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(I->getType()) &&
5977 isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero()) {
5978 const SCEV *One = getOne(I->getType());
5979 const SCEV *LS = getNoopOrZeroExtend(getSCEV(LHS), I->getType());
5980 const SCEV *LA = getSCEV(TrueVal);
5981 const SCEV *RA = getSCEV(FalseVal);
5982 const SCEV *LDiff = getMinusSCEV(LA, One);
5983 const SCEV *RDiff = getMinusSCEV(RA, LS);
5984 if (LDiff == RDiff)
5985 return getAddExpr(getUMaxExpr(One, LS), LDiff);
5986 }
5987 break;
5988 default:
5989 break;
5990 }
5991
5992 return getUnknown(I);
5993}
5994
5995/// Expand GEP instructions into add and multiply operations. This allows them
5996/// to be analyzed by regular SCEV code.
5997const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
5998 // Don't attempt to analyze GEPs over unsized objects.
5999 if (!GEP->getSourceElementType()->isSized())
6000 return getUnknown(GEP);
6001
6002 SmallVector<const SCEV *, 4> IndexExprs;
6003 for (Value *Index : GEP->indices())
6004 IndexExprs.push_back(getSCEV(Index));
6005 return getGEPExpr(GEP, IndexExprs);
6006}
6007
6008uint32_t ScalarEvolution::GetMinTrailingZerosImpl(const SCEV *S) {
6009 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
6010 return C->getAPInt().countTrailingZeros();
6011
6012 if (const SCEVPtrToIntExpr *I = dyn_cast<SCEVPtrToIntExpr>(S))
6013 return GetMinTrailingZeros(I->getOperand());
6014
6015 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(S))
6016 return std::min(GetMinTrailingZeros(T->getOperand()),
6017 (uint32_t)getTypeSizeInBits(T->getType()));
6018
6019 if (const SCEVZeroExtendExpr *E = dyn_cast<SCEVZeroExtendExpr>(S)) {
6020 uint32_t OpRes = GetMinTrailingZeros(E->getOperand());
6021 return OpRes == getTypeSizeInBits(E->getOperand()->getType())
6022 ? getTypeSizeInBits(E->getType())
6023 : OpRes;
6024 }
6025
6026 if (const SCEVSignExtendExpr *E = dyn_cast<SCEVSignExtendExpr>(S)) {
6027 uint32_t OpRes = GetMinTrailingZeros(E->getOperand());
6028 return OpRes == getTypeSizeInBits(E->getOperand()->getType())
6029 ? getTypeSizeInBits(E->getType())
6030 : OpRes;
6031 }
6032
6033 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(S)) {
6034 // The result is the min of all operands results.
6035 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0));
6036 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i)
6037 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i)));
6038 return MinOpRes;
6039 }
6040
6041 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) {
6042 // The result is the sum of all operands results.
6043 uint32_t SumOpRes = GetMinTrailingZeros(M->getOperand(0));
6044 uint32_t BitWidth = getTypeSizeInBits(M->getType());
6045 for (unsigned i = 1, e = M->getNumOperands();
6046 SumOpRes != BitWidth && i != e; ++i)
6047 SumOpRes =
6048 std::min(SumOpRes + GetMinTrailingZeros(M->getOperand(i)), BitWidth);
6049 return SumOpRes;
6050 }
6051
6052 if (const SCEVAddRecExpr *A = dyn_cast<SCEVAddRecExpr>(S)) {
6053 // The result is the min of all operands results.
6054 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0));
6055 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i)
6056 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i)));
6057 return MinOpRes;
6058 }
6059
6060 if (const SCEVSMaxExpr *M = dyn_cast<SCEVSMaxExpr>(S)) {
6061 // The result is the min of all operands results.
6062 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0));
6063 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i)
6064 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i)));
6065 return MinOpRes;
6066 }
6067
6068 if (const SCEVUMaxExpr *M = dyn_cast<SCEVUMaxExpr>(S)) {
6069 // The result is the min of all operands results.
6070 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0));
6071 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i)
6072 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i)));
6073 return MinOpRes;
6074 }
6075
6076 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
6077 // For a SCEVUnknown, ask ValueTracking.
6078 KnownBits Known = computeKnownBits(U->getValue(), getDataLayout(), 0, &AC, nullptr, &DT);
6079 return Known.countMinTrailingZeros();
6080 }
6081
6082 // SCEVUDivExpr
6083 return 0;
6084}
6085
6086uint32_t ScalarEvolution::GetMinTrailingZeros(const SCEV *S) {
6087 auto I = MinTrailingZerosCache.find(S);
6088 if (I != MinTrailingZerosCache.end())
6089 return I->second;
6090
6091 uint32_t Result = GetMinTrailingZerosImpl(S);
6092 auto InsertPair = MinTrailingZerosCache.insert({S, Result});
6093 assert(InsertPair.second && "Should insert a new key")(static_cast <bool> (InsertPair.second && "Should insert a new key"
) ? void (0) : __assert_fail ("InsertPair.second && \"Should insert a new key\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6093, __extension__
__PRETTY_FUNCTION__))
;
6094 return InsertPair.first->second;
6095}
6096
6097/// Helper method to assign a range to V from metadata present in the IR.
6098static Optional<ConstantRange> GetRangeFromMetadata(Value *V) {
6099 if (Instruction *I = dyn_cast<Instruction>(V))
6100 if (MDNode *MD = I->getMetadata(LLVMContext::MD_range))
6101 return getConstantRangeFromMetadata(*MD);
6102
6103 return None;
6104}
6105
6106void ScalarEvolution::setNoWrapFlags(SCEVAddRecExpr *AddRec,
6107 SCEV::NoWrapFlags Flags) {
6108 if (AddRec->getNoWrapFlags(Flags) != Flags) {
6109 AddRec->setNoWrapFlags(Flags);
6110 UnsignedRanges.erase(AddRec);
6111 SignedRanges.erase(AddRec);
6112 }
6113}
6114
6115ConstantRange ScalarEvolution::
6116getRangeForUnknownRecurrence(const SCEVUnknown *U) {
6117 const DataLayout &DL = getDataLayout();
6118
6119 unsigned BitWidth = getTypeSizeInBits(U->getType());
6120 const ConstantRange FullSet(BitWidth, /*isFullSet=*/true);
6121
6122 // Match a simple recurrence of the form: <start, ShiftOp, Step>, and then
6123 // use information about the trip count to improve our available range. Note
6124 // that the trip count independent cases are already handled by known bits.
6125 // WARNING: The definition of recurrence used here is subtly different than
6126 // the one used by AddRec (and thus most of this file). Step is allowed to
6127 // be arbitrarily loop varying here, where AddRec allows only loop invariant
6128 // and other addrecs in the same loop (for non-affine addrecs). The code
6129 // below intentionally handles the case where step is not loop invariant.
6130 auto *P = dyn_cast<PHINode>(U->getValue());
6131 if (!P)
6132 return FullSet;
6133
6134 // Make sure that no Phi input comes from an unreachable block. Otherwise,
6135 // even the values that are not available in these blocks may come from them,
6136 // and this leads to false-positive recurrence test.
6137 for (auto *Pred : predecessors(P->getParent()))
6138 if (!DT.isReachableFromEntry(Pred))
6139 return FullSet;
6140
6141 BinaryOperator *BO;
6142 Value *Start, *Step;
6143 if (!matchSimpleRecurrence(P, BO, Start, Step))
6144 return FullSet;
6145
6146 // If we found a recurrence in reachable code, we must be in a loop. Note
6147 // that BO might be in some subloop of L, and that's completely okay.
6148 auto *L = LI.getLoopFor(P->getParent());
6149 assert(L && L->getHeader() == P->getParent())(static_cast <bool> (L && L->getHeader() == P
->getParent()) ? void (0) : __assert_fail ("L && L->getHeader() == P->getParent()"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6149, __extension__
__PRETTY_FUNCTION__))
;
6150 if (!L->contains(BO->getParent()))
6151 // NOTE: This bailout should be an assert instead. However, asserting
6152 // the condition here exposes a case where LoopFusion is querying SCEV
6153 // with malformed loop information during the midst of the transform.
6154 // There doesn't appear to be an obvious fix, so for the moment bailout
6155 // until the caller issue can be fixed. PR49566 tracks the bug.
6156 return FullSet;
6157
6158 // TODO: Extend to other opcodes such as mul, and div
6159 switch (BO->getOpcode()) {
6160 default:
6161 return FullSet;
6162 case Instruction::AShr:
6163 case Instruction::LShr:
6164 case Instruction::Shl:
6165 break;
6166 };
6167
6168 if (BO->getOperand(0) != P)
6169 // TODO: Handle the power function forms some day.
6170 return FullSet;
6171
6172 unsigned TC = getSmallConstantMaxTripCount(L);
6173 if (!TC || TC >= BitWidth)
6174 return FullSet;
6175
6176 auto KnownStart = computeKnownBits(Start, DL, 0, &AC, nullptr, &DT);
6177 auto KnownStep = computeKnownBits(Step, DL, 0, &AC, nullptr, &DT);
6178 assert(KnownStart.getBitWidth() == BitWidth &&(static_cast <bool> (KnownStart.getBitWidth() == BitWidth
&& KnownStep.getBitWidth() == BitWidth) ? void (0) :
__assert_fail ("KnownStart.getBitWidth() == BitWidth && KnownStep.getBitWidth() == BitWidth"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6179, __extension__
__PRETTY_FUNCTION__))
6179 KnownStep.getBitWidth() == BitWidth)(static_cast <bool> (KnownStart.getBitWidth() == BitWidth
&& KnownStep.getBitWidth() == BitWidth) ? void (0) :
__assert_fail ("KnownStart.getBitWidth() == BitWidth && KnownStep.getBitWidth() == BitWidth"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6179, __extension__
__PRETTY_FUNCTION__))
;
6180
6181 // Compute total shift amount, being careful of overflow and bitwidths.
6182 auto MaxShiftAmt = KnownStep.getMaxValue();
6183 APInt TCAP(BitWidth, TC-1);
6184 bool Overflow = false;
6185 auto TotalShift = MaxShiftAmt.umul_ov(TCAP, Overflow);
6186 if (Overflow)
6187 return FullSet;
6188
6189 switch (BO->getOpcode()) {
6190 default:
6191 llvm_unreachable("filtered out above")::llvm::llvm_unreachable_internal("filtered out above", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 6191)
;
6192 case Instruction::AShr: {
6193 // For each ashr, three cases:
6194 // shift = 0 => unchanged value
6195 // saturation => 0 or -1
6196 // other => a value closer to zero (of the same sign)
6197 // Thus, the end value is closer to zero than the start.
6198 auto KnownEnd = KnownBits::ashr(KnownStart,
6199 KnownBits::makeConstant(TotalShift));
6200 if (KnownStart.isNonNegative())
6201 // Analogous to lshr (simply not yet canonicalized)
6202 return ConstantRange::getNonEmpty(KnownEnd.getMinValue(),
6203 KnownStart.getMaxValue() + 1);
6204 if (KnownStart.isNegative())
6205 // End >=u Start && End <=s Start
6206 return ConstantRange::getNonEmpty(KnownStart.getMinValue(),
6207 KnownEnd.getMaxValue() + 1);
6208 break;
6209 }
6210 case Instruction::LShr: {
6211 // For each lshr, three cases:
6212 // shift = 0 => unchanged value
6213 // saturation => 0
6214 // other => a smaller positive number
6215 // Thus, the low end of the unsigned range is the last value produced.
6216 auto KnownEnd = KnownBits::lshr(KnownStart,
6217 KnownBits::makeConstant(TotalShift));
6218 return ConstantRange::getNonEmpty(KnownEnd.getMinValue(),
6219 KnownStart.getMaxValue() + 1);
6220 }
6221 case Instruction::Shl: {
6222 // Iff no bits are shifted out, value increases on every shift.
6223 auto KnownEnd = KnownBits::shl(KnownStart,
6224 KnownBits::makeConstant(TotalShift));
6225 if (TotalShift.ult(KnownStart.countMinLeadingZeros()))
6226 return ConstantRange(KnownStart.getMinValue(),
6227 KnownEnd.getMaxValue() + 1);
6228 break;
6229 }
6230 };
6231 return FullSet;
6232}
6233
6234/// Determine the range for a particular SCEV. If SignHint is
6235/// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges
6236/// with a "cleaner" unsigned (resp. signed) representation.
6237const ConstantRange &
6238ScalarEvolution::getRangeRef(const SCEV *S,
6239 ScalarEvolution::RangeSignHint SignHint) {
6240 DenseMap<const SCEV *, ConstantRange> &Cache =
6241 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6242 : SignedRanges;
6243 ConstantRange::PreferredRangeType RangeType =
6244 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED
6245 ? ConstantRange::Unsigned : ConstantRange::Signed;
6246
6247 // See if we've computed this range already.
6248 DenseMap<const SCEV *, ConstantRange>::iterator I = Cache.find(S);
6249 if (I != Cache.end())
6250 return I->second;
6251
6252 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
6253 return setRange(C, SignHint, ConstantRange(C->getAPInt()));
6254
6255 unsigned BitWidth = getTypeSizeInBits(S->getType());
6256 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
6257 using OBO = OverflowingBinaryOperator;
6258
6259 // If the value has known zeros, the maximum value will have those known zeros
6260 // as well.
6261 uint32_t TZ = GetMinTrailingZeros(S);
6262 if (TZ != 0) {
6263 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED)
6264 ConservativeResult =
6265 ConstantRange(APInt::getMinValue(BitWidth),
6266 APInt::getMaxValue(BitWidth).lshr(TZ).shl(TZ) + 1);
6267 else
6268 ConservativeResult = ConstantRange(
6269 APInt::getSignedMinValue(BitWidth),
6270 APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1);
6271 }
6272
6273 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
6274 ConstantRange X = getRangeRef(Add->getOperand(0), SignHint);
6275 unsigned WrapType = OBO::AnyWrap;
6276 if (Add->hasNoSignedWrap())
6277 WrapType |= OBO::NoSignedWrap;
6278 if (Add->hasNoUnsignedWrap())
6279 WrapType |= OBO::NoUnsignedWrap;
6280 for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i)
6281 X = X.addWithNoWrap(getRangeRef(Add->getOperand(i), SignHint),
6282 WrapType, RangeType);
6283 return setRange(Add, SignHint,
6284 ConservativeResult.intersectWith(X, RangeType));
6285 }
6286
6287 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
6288 ConstantRange X = getRangeRef(Mul->getOperand(0), SignHint);
6289 for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i)
6290 X = X.multiply(getRangeRef(Mul->getOperand(i), SignHint));
6291 return setRange(Mul, SignHint,
6292 ConservativeResult.intersectWith(X, RangeType));
6293 }
6294
6295 if (isa<SCEVMinMaxExpr>(S) || isa<SCEVSequentialMinMaxExpr>(S)) {
6296 Intrinsic::ID ID;
6297 switch (S->getSCEVType()) {
6298 case scUMaxExpr:
6299 ID = Intrinsic::umax;
6300 break;
6301 case scSMaxExpr:
6302 ID = Intrinsic::smax;
6303 break;
6304 case scUMinExpr:
6305 case scSequentialUMinExpr:
6306 ID = Intrinsic::umin;
6307 break;
6308 case scSMinExpr:
6309 ID = Intrinsic::smin;
6310 break;
6311 default:
6312 llvm_unreachable("Unknown SCEVMinMaxExpr/SCEVSequentialMinMaxExpr.")::llvm::llvm_unreachable_internal("Unknown SCEVMinMaxExpr/SCEVSequentialMinMaxExpr."
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6312)
;
6313 }
6314
6315 const auto *NAry = cast<SCEVNAryExpr>(S);
6316 ConstantRange X = getRangeRef(NAry->getOperand(0), SignHint);
6317 for (unsigned i = 1, e = NAry->getNumOperands(); i != e; ++i)
6318 X = X.intrinsic(ID, {X, getRangeRef(NAry->getOperand(i), SignHint)});
6319 return setRange(S, SignHint,
6320 ConservativeResult.intersectWith(X, RangeType));
6321 }
6322
6323 if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) {
6324 ConstantRange X = getRangeRef(UDiv->getLHS(), SignHint);
6325 ConstantRange Y = getRangeRef(UDiv->getRHS(), SignHint);
6326 return setRange(UDiv, SignHint,
6327 ConservativeResult.intersectWith(X.udiv(Y), RangeType));
6328 }
6329
6330 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) {
6331 ConstantRange X = getRangeRef(ZExt->getOperand(), SignHint);
6332 return setRange(ZExt, SignHint,
6333 ConservativeResult.intersectWith(X.zeroExtend(BitWidth),
6334 RangeType));
6335 }
6336
6337 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) {
6338 ConstantRange X = getRangeRef(SExt->getOperand(), SignHint);
6339 return setRange(SExt, SignHint,
6340 ConservativeResult.intersectWith(X.signExtend(BitWidth),
6341 RangeType));
6342 }
6343
6344 if (const SCEVPtrToIntExpr *PtrToInt = dyn_cast<SCEVPtrToIntExpr>(S)) {
6345 ConstantRange X = getRangeRef(PtrToInt->getOperand(), SignHint);
6346 return setRange(PtrToInt, SignHint, X);
6347 }
6348
6349 if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) {
6350 ConstantRange X = getRangeRef(Trunc->getOperand(), SignHint);
6351 return setRange(Trunc, SignHint,
6352 ConservativeResult.intersectWith(X.truncate(BitWidth),
6353 RangeType));
6354 }
6355
6356 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) {
6357 // If there's no unsigned wrap, the value will never be less than its
6358 // initial value.
6359 if (AddRec->hasNoUnsignedWrap()) {
6360 APInt UnsignedMinValue = getUnsignedRangeMin(AddRec->getStart());
6361 if (!UnsignedMinValue.isZero())
6362 ConservativeResult = ConservativeResult.intersectWith(
6363 ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), RangeType);
6364 }
6365
6366 // If there's no signed wrap, and all the operands except initial value have
6367 // the same sign or zero, the value won't ever be:
6368 // 1: smaller than initial value if operands are non negative,
6369 // 2: bigger than initial value if operands are non positive.
6370 // For both cases, value can not cross signed min/max boundary.
6371 if (AddRec->hasNoSignedWrap()) {
6372 bool AllNonNeg = true;
6373 bool AllNonPos = true;
6374 for (unsigned i = 1, e = AddRec->getNumOperands(); i != e; ++i) {
6375 if (!isKnownNonNegative(AddRec->getOperand(i)))
6376 AllNonNeg = false;
6377 if (!isKnownNonPositive(AddRec->getOperand(i)))
6378 AllNonPos = false;
6379 }
6380 if (AllNonNeg)
6381 ConservativeResult = ConservativeResult.intersectWith(
6382 ConstantRange::getNonEmpty(getSignedRangeMin(AddRec->getStart()),
6383 APInt::getSignedMinValue(BitWidth)),
6384 RangeType);
6385 else if (AllNonPos)
6386 ConservativeResult = ConservativeResult.intersectWith(
6387 ConstantRange::getNonEmpty(
6388 APInt::getSignedMinValue(BitWidth),
6389 getSignedRangeMax(AddRec->getStart()) + 1),
6390 RangeType);
6391 }
6392
6393 // TODO: non-affine addrec
6394 if (AddRec->isAffine()) {
6395 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(AddRec->getLoop());
6396 if (!isa<SCEVCouldNotCompute>(MaxBECount) &&
6397 getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) {
6398 auto RangeFromAffine = getRangeForAffineAR(
6399 AddRec->getStart(), AddRec->getStepRecurrence(*this), MaxBECount,
6400 BitWidth);
6401 ConservativeResult =
6402 ConservativeResult.intersectWith(RangeFromAffine, RangeType);
6403
6404 auto RangeFromFactoring = getRangeViaFactoring(
6405 AddRec->getStart(), AddRec->getStepRecurrence(*this), MaxBECount,
6406 BitWidth);
6407 ConservativeResult =
6408 ConservativeResult.intersectWith(RangeFromFactoring, RangeType);
6409 }
6410
6411 // Now try symbolic BE count and more powerful methods.
6412 if (UseExpensiveRangeSharpening) {
6413 const SCEV *SymbolicMaxBECount =
6414 getSymbolicMaxBackedgeTakenCount(AddRec->getLoop());
6415 if (!isa<SCEVCouldNotCompute>(SymbolicMaxBECount) &&
6416 getTypeSizeInBits(MaxBECount->getType()) <= BitWidth &&
6417 AddRec->hasNoSelfWrap()) {
6418 auto RangeFromAffineNew = getRangeForAffineNoSelfWrappingAR(
6419 AddRec, SymbolicMaxBECount, BitWidth, SignHint);
6420 ConservativeResult =
6421 ConservativeResult.intersectWith(RangeFromAffineNew, RangeType);
6422 }
6423 }
6424 }
6425
6426 return setRange(AddRec, SignHint, std::move(ConservativeResult));
6427 }
6428
6429 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
6430
6431 // Check if the IR explicitly contains !range metadata.
6432 Optional<ConstantRange> MDRange = GetRangeFromMetadata(U->getValue());
6433 if (MDRange.hasValue())
6434 ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue(),
6435 RangeType);
6436
6437 // Use facts about recurrences in the underlying IR. Note that add
6438 // recurrences are AddRecExprs and thus don't hit this path. This
6439 // primarily handles shift recurrences.
6440 auto CR = getRangeForUnknownRecurrence(U);
6441 ConservativeResult = ConservativeResult.intersectWith(CR);
6442
6443 // See if ValueTracking can give us a useful range.
6444 const DataLayout &DL = getDataLayout();
6445 KnownBits Known = computeKnownBits(U->getValue(), DL, 0, &AC, nullptr, &DT);
6446 if (Known.getBitWidth() != BitWidth)
6447 Known = Known.zextOrTrunc(BitWidth);
6448
6449 // ValueTracking may be able to compute a tighter result for the number of
6450 // sign bits than for the value of those sign bits.
6451 unsigned NS = ComputeNumSignBits(U->getValue(), DL, 0, &AC, nullptr, &DT);
6452 if (U->getType()->isPointerTy()) {
6453 // If the pointer size is larger than the index size type, this can cause
6454 // NS to be larger than BitWidth. So compensate for this.
6455 unsigned ptrSize = DL.getPointerTypeSizeInBits(U->getType());
6456 int ptrIdxDiff = ptrSize - BitWidth;
6457 if (ptrIdxDiff > 0 && ptrSize > BitWidth && NS > (unsigned)ptrIdxDiff)
6458 NS -= ptrIdxDiff;
6459 }
6460
6461 if (NS > 1) {
6462 // If we know any of the sign bits, we know all of the sign bits.
6463 if (!Known.Zero.getHiBits(NS).isZero())
6464 Known.Zero.setHighBits(NS);
6465 if (!Known.One.getHiBits(NS).isZero())
6466 Known.One.setHighBits(NS);
6467 }
6468
6469 if (Known.getMinValue() != Known.getMaxValue() + 1)
6470 ConservativeResult = ConservativeResult.intersectWith(
6471 ConstantRange(Known.getMinValue(), Known.getMaxValue() + 1),
6472 RangeType);
6473 if (NS > 1)
6474 ConservativeResult = ConservativeResult.intersectWith(
6475 ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1),
6476 APInt::getSignedMaxValue(BitWidth).ashr(NS - 1) + 1),
6477 RangeType);
6478
6479 // A range of Phi is a subset of union of all ranges of its input.
6480 if (const PHINode *Phi = dyn_cast<PHINode>(U->getValue())) {
6481 // Make sure that we do not run over cycled Phis.
6482 if (PendingPhiRanges.insert(Phi).second) {
6483 ConstantRange RangeFromOps(BitWidth, /*isFullSet=*/false);
6484 for (auto &Op : Phi->operands()) {
6485 auto OpRange = getRangeRef(getSCEV(Op), SignHint);
6486 RangeFromOps = RangeFromOps.unionWith(OpRange);
6487 // No point to continue if we already have a full set.
6488 if (RangeFromOps.isFullSet())
6489 break;
6490 }
6491 ConservativeResult =
6492 ConservativeResult.intersectWith(RangeFromOps, RangeType);
6493 bool Erased = PendingPhiRanges.erase(Phi);
6494 assert(Erased && "Failed to erase Phi properly?")(static_cast <bool> (Erased && "Failed to erase Phi properly?"
) ? void (0) : __assert_fail ("Erased && \"Failed to erase Phi properly?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6494, __extension__
__PRETTY_FUNCTION__))
;
6495 (void) Erased;
6496 }
6497 }
6498
6499 return setRange(U, SignHint, std::move(ConservativeResult));
6500 }
6501
6502 return setRange(S, SignHint, std::move(ConservativeResult));
6503}
6504
6505// Given a StartRange, Step and MaxBECount for an expression compute a range of
6506// values that the expression can take. Initially, the expression has a value
6507// from StartRange and then is changed by Step up to MaxBECount times. Signed
6508// argument defines if we treat Step as signed or unsigned.
6509static ConstantRange getRangeForAffineARHelper(APInt Step,
6510 const ConstantRange &StartRange,
6511 const APInt &MaxBECount,
6512 unsigned BitWidth, bool Signed) {
6513 // If either Step or MaxBECount is 0, then the expression won't change, and we
6514 // just need to return the initial range.
6515 if (Step == 0 || MaxBECount == 0)
6516 return StartRange;
6517
6518 // If we don't know anything about the initial value (i.e. StartRange is
6519 // FullRange), then we don't know anything about the final range either.
6520 // Return FullRange.
6521 if (StartRange.isFullSet())
6522 return ConstantRange::getFull(BitWidth);
6523
6524 // If Step is signed and negative, then we use its absolute value, but we also
6525 // note that we're moving in the opposite direction.
6526 bool Descending = Signed && Step.isNegative();
6527
6528 if (Signed)
6529 // This is correct even for INT_SMIN. Let's look at i8 to illustrate this:
6530 // abs(INT_SMIN) = abs(-128) = abs(0x80) = -0x80 = 0x80 = 128.
6531 // This equations hold true due to the well-defined wrap-around behavior of
6532 // APInt.
6533 Step = Step.abs();
6534
6535 // Check if Offset is more than full span of BitWidth. If it is, the
6536 // expression is guaranteed to overflow.
6537 if (APInt::getMaxValue(StartRange.getBitWidth()).udiv(Step).ult(MaxBECount))
6538 return ConstantRange::getFull(BitWidth);
6539
6540 // Offset is by how much the expression can change. Checks above guarantee no
6541 // overflow here.
6542 APInt Offset = Step * MaxBECount;
6543
6544 // Minimum value of the final range will match the minimal value of StartRange
6545 // if the expression is increasing and will be decreased by Offset otherwise.
6546 // Maximum value of the final range will match the maximal value of StartRange
6547 // if the expression is decreasing and will be increased by Offset otherwise.
6548 APInt StartLower = StartRange.getLower();
6549 APInt StartUpper = StartRange.getUpper() - 1;
6550 APInt MovedBoundary = Descending ? (StartLower - std::move(Offset))
6551 : (StartUpper + std::move(Offset));
6552
6553 // It's possible that the new minimum/maximum value will fall into the initial
6554 // range (due to wrap around). This means that the expression can take any
6555 // value in this bitwidth, and we have to return full range.
6556 if (StartRange.contains(MovedBoundary))
6557 return ConstantRange::getFull(BitWidth);
6558
6559 APInt NewLower =
6560 Descending ? std::move(MovedBoundary) : std::move(StartLower);
6561 APInt NewUpper =
6562 Descending ? std::move(StartUpper) : std::move(MovedBoundary);
6563 NewUpper += 1;
6564
6565 // No overflow detected, return [StartLower, StartUpper + Offset + 1) range.
6566 return ConstantRange::getNonEmpty(std::move(NewLower), std::move(NewUpper));
6567}
6568
6569ConstantRange ScalarEvolution::getRangeForAffineAR(const SCEV *Start,
6570 const SCEV *Step,
6571 const SCEV *MaxBECount,
6572 unsigned BitWidth) {
6573 assert(!isa<SCEVCouldNotCompute>(MaxBECount) &&(static_cast <bool> (!isa<SCEVCouldNotCompute>(MaxBECount
) && getTypeSizeInBits(MaxBECount->getType()) <=
BitWidth && "Precondition!") ? void (0) : __assert_fail
("!isa<SCEVCouldNotCompute>(MaxBECount) && getTypeSizeInBits(MaxBECount->getType()) <= BitWidth && \"Precondition!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6575, __extension__
__PRETTY_FUNCTION__))
6574 getTypeSizeInBits(MaxBECount->getType()) <= BitWidth &&(static_cast <bool> (!isa<SCEVCouldNotCompute>(MaxBECount
) && getTypeSizeInBits(MaxBECount->getType()) <=
BitWidth && "Precondition!") ? void (0) : __assert_fail
("!isa<SCEVCouldNotCompute>(MaxBECount) && getTypeSizeInBits(MaxBECount->getType()) <= BitWidth && \"Precondition!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6575, __extension__
__PRETTY_FUNCTION__))
6575 "Precondition!")(static_cast <bool> (!isa<SCEVCouldNotCompute>(MaxBECount
) && getTypeSizeInBits(MaxBECount->getType()) <=
BitWidth && "Precondition!") ? void (0) : __assert_fail
("!isa<SCEVCouldNotCompute>(MaxBECount) && getTypeSizeInBits(MaxBECount->getType()) <= BitWidth && \"Precondition!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6575, __extension__
__PRETTY_FUNCTION__))
;
6576
6577 MaxBECount = getNoopOrZeroExtend(MaxBECount, Start->getType());
6578 APInt MaxBECountValue = getUnsignedRangeMax(MaxBECount);
6579
6580 // First, consider step signed.
6581 ConstantRange StartSRange = getSignedRange(Start);
6582 ConstantRange StepSRange = getSignedRange(Step);
6583
6584 // If Step can be both positive and negative, we need to find ranges for the
6585 // maximum absolute step values in both directions and union them.
6586 ConstantRange SR =
6587 getRangeForAffineARHelper(StepSRange.getSignedMin(), StartSRange,
6588 MaxBECountValue, BitWidth, /* Signed = */ true);
6589 SR = SR.unionWith(getRangeForAffineARHelper(StepSRange.getSignedMax(),
6590 StartSRange, MaxBECountValue,
6591 BitWidth, /* Signed = */ true));
6592
6593 // Next, consider step unsigned.
6594 ConstantRange UR = getRangeForAffineARHelper(
6595 getUnsignedRangeMax(Step), getUnsignedRange(Start),
6596 MaxBECountValue, BitWidth, /* Signed = */ false);
6597
6598 // Finally, intersect signed and unsigned ranges.
6599 return SR.intersectWith(UR, ConstantRange::Smallest);
6600}
6601
6602ConstantRange ScalarEvolution::getRangeForAffineNoSelfWrappingAR(
6603 const SCEVAddRecExpr *AddRec, const SCEV *MaxBECount, unsigned BitWidth,
6604 ScalarEvolution::RangeSignHint SignHint) {
6605 assert(AddRec->isAffine() && "Non-affine AddRecs are not suppored!\n")(static_cast <bool> (AddRec->isAffine() && "Non-affine AddRecs are not suppored!\n"
) ? void (0) : __assert_fail ("AddRec->isAffine() && \"Non-affine AddRecs are not suppored!\\n\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6605, __extension__
__PRETTY_FUNCTION__))
;
6606 assert(AddRec->hasNoSelfWrap() &&(static_cast <bool> (AddRec->hasNoSelfWrap() &&
"This only works for non-self-wrapping AddRecs!") ? void (0)
: __assert_fail ("AddRec->hasNoSelfWrap() && \"This only works for non-self-wrapping AddRecs!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6607, __extension__
__PRETTY_FUNCTION__))
6607 "This only works for non-self-wrapping AddRecs!")(static_cast <bool> (AddRec->hasNoSelfWrap() &&
"This only works for non-self-wrapping AddRecs!") ? void (0)
: __assert_fail ("AddRec->hasNoSelfWrap() && \"This only works for non-self-wrapping AddRecs!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6607, __extension__
__PRETTY_FUNCTION__))
;
6608 const bool IsSigned = SignHint == HINT_RANGE_SIGNED;
6609 const SCEV *Step = AddRec->getStepRecurrence(*this);
6610 // Only deal with constant step to save compile time.
6611 if (!isa<SCEVConstant>(Step))
6612 return ConstantRange::getFull(BitWidth);
6613 // Let's make sure that we can prove that we do not self-wrap during
6614 // MaxBECount iterations. We need this because MaxBECount is a maximum
6615 // iteration count estimate, and we might infer nw from some exit for which we
6616 // do not know max exit count (or any other side reasoning).
6617 // TODO: Turn into assert at some point.
6618 if (getTypeSizeInBits(MaxBECount->getType()) >
6619 getTypeSizeInBits(AddRec->getType()))
6620 return ConstantRange::getFull(BitWidth);
6621 MaxBECount = getNoopOrZeroExtend(MaxBECount, AddRec->getType());
6622 const SCEV *RangeWidth = getMinusOne(AddRec->getType());
6623 const SCEV *StepAbs = getUMinExpr(Step, getNegativeSCEV(Step));
6624 const SCEV *MaxItersWithoutWrap = getUDivExpr(RangeWidth, StepAbs);
6625 if (!isKnownPredicateViaConstantRanges(ICmpInst::ICMP_ULE, MaxBECount,
6626 MaxItersWithoutWrap))
6627 return ConstantRange::getFull(BitWidth);
6628
6629 ICmpInst::Predicate LEPred =
6630 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
6631 ICmpInst::Predicate GEPred =
6632 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
6633 const SCEV *End = AddRec->evaluateAtIteration(MaxBECount, *this);
6634
6635 // We know that there is no self-wrap. Let's take Start and End values and
6636 // look at all intermediate values V1, V2, ..., Vn that IndVar takes during
6637 // the iteration. They either lie inside the range [Min(Start, End),
6638 // Max(Start, End)] or outside it:
6639 //
6640 // Case 1: RangeMin ... Start V1 ... VN End ... RangeMax;
6641 // Case 2: RangeMin Vk ... V1 Start ... End Vn ... Vk + 1 RangeMax;
6642 //
6643 // No self wrap flag guarantees that the intermediate values cannot be BOTH
6644 // outside and inside the range [Min(Start, End), Max(Start, End)]. Using that
6645 // knowledge, let's try to prove that we are dealing with Case 1. It is so if
6646 // Start <= End and step is positive, or Start >= End and step is negative.
6647 const SCEV *Start = AddRec->getStart();
6648 ConstantRange StartRange = getRangeRef(Start, SignHint);
6649 ConstantRange EndRange = getRangeRef(End, SignHint);
6650 ConstantRange RangeBetween = StartRange.unionWith(EndRange);
6651 // If they already cover full iteration space, we will know nothing useful
6652 // even if we prove what we want to prove.
6653 if (RangeBetween.isFullSet())
6654 return RangeBetween;
6655 // Only deal with ranges that do not wrap (i.e. RangeMin < RangeMax).
6656 bool IsWrappedSet = IsSigned ? RangeBetween.isSignWrappedSet()
6657 : RangeBetween.isWrappedSet();
6658 if (IsWrappedSet)
6659 return ConstantRange::getFull(BitWidth);
6660
6661 if (isKnownPositive(Step) &&
6662 isKnownPredicateViaConstantRanges(LEPred, Start, End))
6663 return RangeBetween;
6664 else if (isKnownNegative(Step) &&
6665 isKnownPredicateViaConstantRanges(GEPred, Start, End))
6666 return RangeBetween;
6667 return ConstantRange::getFull(BitWidth);
6668}
6669
6670ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start,
6671 const SCEV *Step,
6672 const SCEV *MaxBECount,
6673 unsigned BitWidth) {
6674 // RangeOf({C?A:B,+,C?P:Q}) == RangeOf(C?{A,+,P}:{B,+,Q})
6675 // == RangeOf({A,+,P}) union RangeOf({B,+,Q})
6676
6677 struct SelectPattern {
6678 Value *Condition = nullptr;
6679 APInt TrueValue;
6680 APInt FalseValue;
6681
6682 explicit SelectPattern(ScalarEvolution &SE, unsigned BitWidth,
6683 const SCEV *S) {
6684 Optional<unsigned> CastOp;
6685 APInt Offset(BitWidth, 0);
6686
6687 assert(SE.getTypeSizeInBits(S->getType()) == BitWidth &&(static_cast <bool> (SE.getTypeSizeInBits(S->getType
()) == BitWidth && "Should be!") ? void (0) : __assert_fail
("SE.getTypeSizeInBits(S->getType()) == BitWidth && \"Should be!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6688, __extension__
__PRETTY_FUNCTION__))
6688 "Should be!")(static_cast <bool> (SE.getTypeSizeInBits(S->getType
()) == BitWidth && "Should be!") ? void (0) : __assert_fail
("SE.getTypeSizeInBits(S->getType()) == BitWidth && \"Should be!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6688, __extension__
__PRETTY_FUNCTION__))
;
6689
6690 // Peel off a constant offset:
6691 if (auto *SA = dyn_cast<SCEVAddExpr>(S)) {
6692 // In the future we could consider being smarter here and handle
6693 // {Start+Step,+,Step} too.
6694 if (SA->getNumOperands() != 2 || !isa<SCEVConstant>(SA->getOperand(0)))
6695 return;
6696
6697 Offset = cast<SCEVConstant>(SA->getOperand(0))->getAPInt();
6698 S = SA->getOperand(1);
6699 }
6700
6701 // Peel off a cast operation
6702 if (auto *SCast = dyn_cast<SCEVIntegralCastExpr>(S)) {
6703 CastOp = SCast->getSCEVType();
6704 S = SCast->getOperand();
6705 }
6706
6707 using namespace llvm::PatternMatch;
6708
6709 auto *SU = dyn_cast<SCEVUnknown>(S);
6710 const APInt *TrueVal, *FalseVal;
6711 if (!SU ||
6712 !match(SU->getValue(), m_Select(m_Value(Condition), m_APInt(TrueVal),
6713 m_APInt(FalseVal)))) {
6714 Condition = nullptr;
6715 return;
6716 }
6717
6718 TrueValue = *TrueVal;
6719 FalseValue = *FalseVal;
6720
6721 // Re-apply the cast we peeled off earlier
6722 if (CastOp.hasValue())
6723 switch (*CastOp) {
6724 default:
6725 llvm_unreachable("Unknown SCEV cast type!")::llvm::llvm_unreachable_internal("Unknown SCEV cast type!", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 6725)
;
6726
6727 case scTruncate:
6728 TrueValue = TrueValue.trunc(BitWidth);
6729 FalseValue = FalseValue.trunc(BitWidth);
6730 break;
6731 case scZeroExtend:
6732 TrueValue = TrueValue.zext(BitWidth);
6733 FalseValue = FalseValue.zext(BitWidth);
6734 break;
6735 case scSignExtend:
6736 TrueValue = TrueValue.sext(BitWidth);
6737 FalseValue = FalseValue.sext(BitWidth);
6738 break;
6739 }
6740
6741 // Re-apply the constant offset we peeled off earlier
6742 TrueValue += Offset;
6743 FalseValue += Offset;
6744 }
6745
6746 bool isRecognized() { return Condition != nullptr; }
6747 };
6748
6749 SelectPattern StartPattern(*this, BitWidth, Start);
6750 if (!StartPattern.isRecognized())
6751 return ConstantRange::getFull(BitWidth);
6752
6753 SelectPattern StepPattern(*this, BitWidth, Step);
6754 if (!StepPattern.isRecognized())
6755 return ConstantRange::getFull(BitWidth);
6756
6757 if (StartPattern.Condition != StepPattern.Condition) {
6758 // We don't handle this case today; but we could, by considering four
6759 // possibilities below instead of two. I'm not sure if there are cases where
6760 // that will help over what getRange already does, though.
6761 return ConstantRange::getFull(BitWidth);
6762 }
6763
6764 // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to
6765 // construct arbitrary general SCEV expressions here. This function is called
6766 // from deep in the call stack, and calling getSCEV (on a sext instruction,
6767 // say) can end up caching a suboptimal value.
6768
6769 // FIXME: without the explicit `this` receiver below, MSVC errors out with
6770 // C2352 and C2512 (otherwise it isn't needed).
6771
6772 const SCEV *TrueStart = this->getConstant(StartPattern.TrueValue);
6773 const SCEV *TrueStep = this->getConstant(StepPattern.TrueValue);
6774 const SCEV *FalseStart = this->getConstant(StartPattern.FalseValue);
6775 const SCEV *FalseStep = this->getConstant(StepPattern.FalseValue);
6776
6777 ConstantRange TrueRange =
6778 this->getRangeForAffineAR(TrueStart, TrueStep, MaxBECount, BitWidth);
6779 ConstantRange FalseRange =
6780 this->getRangeForAffineAR(FalseStart, FalseStep, MaxBECount, BitWidth);
6781
6782 return TrueRange.unionWith(FalseRange);
6783}
6784
6785SCEV::NoWrapFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) {
6786 if (isa<ConstantExpr>(V)) return SCEV::FlagAnyWrap;
6787 const BinaryOperator *BinOp = cast<BinaryOperator>(V);
6788
6789 // Return early if there are no flags to propagate to the SCEV.
6790 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
6791 if (BinOp->hasNoUnsignedWrap())
6792 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
6793 if (BinOp->hasNoSignedWrap())
6794 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW);
6795 if (Flags == SCEV::FlagAnyWrap)
6796 return SCEV::FlagAnyWrap;
6797
6798 return isSCEVExprNeverPoison(BinOp) ? Flags : SCEV::FlagAnyWrap;
6799}
6800
6801const Instruction *
6802ScalarEvolution::getNonTrivialDefiningScopeBound(const SCEV *S) {
6803 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(S))
6804 return &*AddRec->getLoop()->getHeader()->begin();
6805 if (auto *U = dyn_cast<SCEVUnknown>(S))
6806 if (auto *I = dyn_cast<Instruction>(U->getValue()))
6807 return I;
6808 return nullptr;
6809}
6810
6811/// Fills \p Ops with unique operands of \p S, if it has operands. If not,
6812/// \p Ops remains unmodified.
6813static void collectUniqueOps(const SCEV *S,
6814 SmallVectorImpl<const SCEV *> &Ops) {
6815 SmallPtrSet<const SCEV *, 4> Unique;
6816 auto InsertUnique = [&](const SCEV *S) {
6817 if (Unique.insert(S).second)
6818 Ops.push_back(S);
6819 };
6820 if (auto *S2 = dyn_cast<SCEVCastExpr>(S))
6821 for (auto *Op : S2->operands())
6822 InsertUnique(Op);
6823 else if (auto *S2 = dyn_cast<SCEVNAryExpr>(S))
6824 for (auto *Op : S2->operands())
6825 InsertUnique(Op);
6826 else if (auto *S2 = dyn_cast<SCEVUDivExpr>(S))
6827 for (auto *Op : S2->operands())
6828 InsertUnique(Op);
6829}
6830
6831const Instruction *
6832ScalarEvolution::getDefiningScopeBound(ArrayRef<const SCEV *> Ops,
6833 bool &Precise) {
6834 Precise = true;
6835 // Do a bounded search of the def relation of the requested SCEVs.
6836 SmallSet<const SCEV *, 16> Visited;
6837 SmallVector<const SCEV *> Worklist;
6838 auto pushOp = [&](const SCEV *S) {
6839 if (!Visited.insert(S).second)
6840 return;
6841 // Threshold of 30 here is arbitrary.
6842 if (Visited.size() > 30) {
6843 Precise = false;
6844 return;
6845 }
6846 Worklist.push_back(S);
6847 };
6848
6849 for (auto *S : Ops)
6850 pushOp(S);
6851
6852 const Instruction *Bound = nullptr;
6853 while (!Worklist.empty()) {
6854 auto *S = Worklist.pop_back_val();
6855 if (auto *DefI = getNonTrivialDefiningScopeBound(S)) {
6856 if (!Bound || DT.dominates(Bound, DefI))
6857 Bound = DefI;
6858 } else {
6859 SmallVector<const SCEV *, 4> Ops;
6860 collectUniqueOps(S, Ops);
6861 for (auto *Op : Ops)
6862 pushOp(Op);
6863 }
6864 }
6865 return Bound ? Bound : &*F.getEntryBlock().begin();
6866}
6867
6868const Instruction *
6869ScalarEvolution::getDefiningScopeBound(ArrayRef<const SCEV *> Ops) {
6870 bool Discard;
6871 return getDefiningScopeBound(Ops, Discard);
6872}
6873
6874bool ScalarEvolution::isGuaranteedToTransferExecutionTo(const Instruction *A,
6875 const Instruction *B) {
6876 if (A->getParent() == B->getParent() &&
6877 isGuaranteedToTransferExecutionToSuccessor(A->getIterator(),
6878 B->getIterator()))
6879 return true;
6880
6881 auto *BLoop = LI.getLoopFor(B->getParent());
6882 if (BLoop && BLoop->getHeader() == B->getParent() &&
6883 BLoop->getLoopPreheader() == A->getParent() &&
6884 isGuaranteedToTransferExecutionToSuccessor(A->getIterator(),
6885 A->getParent()->end()) &&
6886 isGuaranteedToTransferExecutionToSuccessor(B->getParent()->begin(),
6887 B->getIterator()))
6888 return true;
6889 return false;
6890}
6891
6892
6893bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) {
6894 // Only proceed if we can prove that I does not yield poison.
6895 if (!programUndefinedIfPoison(I))
6896 return false;
6897
6898 // At this point we know that if I is executed, then it does not wrap
6899 // according to at least one of NSW or NUW. If I is not executed, then we do
6900 // not know if the calculation that I represents would wrap. Multiple
6901 // instructions can map to the same SCEV. If we apply NSW or NUW from I to
6902 // the SCEV, we must guarantee no wrapping for that SCEV also when it is
6903 // derived from other instructions that map to the same SCEV. We cannot make
6904 // that guarantee for cases where I is not executed. So we need to find a
6905 // upper bound on the defining scope for the SCEV, and prove that I is
6906 // executed every time we enter that scope. When the bounding scope is a
6907 // loop (the common case), this is equivalent to proving I executes on every
6908 // iteration of that loop.
6909 SmallVector<const SCEV *> SCEVOps;
6910 for (const Use &Op : I->operands()) {
6911 // I could be an extractvalue from a call to an overflow intrinsic.
6912 // TODO: We can do better here in some cases.
6913 if (isSCEVable(Op->getType()))
6914 SCEVOps.push_back(getSCEV(Op));
6915 }
6916 auto *DefI = getDefiningScopeBound(SCEVOps);
6917 return isGuaranteedToTransferExecutionTo(DefI, I);
6918}
6919
6920bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) {
6921 // If we know that \c I can never be poison period, then that's enough.
6922 if (isSCEVExprNeverPoison(I))
6923 return true;
6924
6925 // For an add recurrence specifically, we assume that infinite loops without
6926 // side effects are undefined behavior, and then reason as follows:
6927 //
6928 // If the add recurrence is poison in any iteration, it is poison on all
6929 // future iterations (since incrementing poison yields poison). If the result
6930 // of the add recurrence is fed into the loop latch condition and the loop
6931 // does not contain any throws or exiting blocks other than the latch, we now
6932 // have the ability to "choose" whether the backedge is taken or not (by
6933 // choosing a sufficiently evil value for the poison feeding into the branch)
6934 // for every iteration including and after the one in which \p I first became
6935 // poison. There are two possibilities (let's call the iteration in which \p
6936 // I first became poison as K):
6937 //
6938 // 1. In the set of iterations including and after K, the loop body executes
6939 // no side effects. In this case executing the backege an infinte number
6940 // of times will yield undefined behavior.
6941 //
6942 // 2. In the set of iterations including and after K, the loop body executes
6943 // at least one side effect. In this case, that specific instance of side
6944 // effect is control dependent on poison, which also yields undefined
6945 // behavior.
6946
6947 auto *ExitingBB = L->getExitingBlock();
6948 auto *LatchBB = L->getLoopLatch();
6949 if (!ExitingBB || !LatchBB || ExitingBB != LatchBB)
6950 return false;
6951
6952 SmallPtrSet<const Instruction *, 16> Pushed;
6953 SmallVector<const Instruction *, 8> PoisonStack;
6954
6955 // We start by assuming \c I, the post-inc add recurrence, is poison. Only
6956 // things that are known to be poison under that assumption go on the
6957 // PoisonStack.
6958 Pushed.insert(I);
6959 PoisonStack.push_back(I);
6960
6961 bool LatchControlDependentOnPoison = false;
6962 while (!PoisonStack.empty() && !LatchControlDependentOnPoison) {
6963 const Instruction *Poison = PoisonStack.pop_back_val();
6964
6965 for (auto *PoisonUser : Poison->users()) {
6966 if (propagatesPoison(cast<Operator>(PoisonUser))) {
6967 if (Pushed.insert(cast<Instruction>(PoisonUser)).second)
6968 PoisonStack.push_back(cast<Instruction>(PoisonUser));
6969 } else if (auto *BI = dyn_cast<BranchInst>(PoisonUser)) {
6970 assert(BI->isConditional() && "Only possibility!")(static_cast <bool> (BI->isConditional() && "Only possibility!"
) ? void (0) : __assert_fail ("BI->isConditional() && \"Only possibility!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 6970, __extension__
__PRETTY_FUNCTION__))
;
6971 if (BI->getParent() == LatchBB) {
6972 LatchControlDependentOnPoison = true;
6973 break;
6974 }
6975 }
6976 }
6977 }
6978
6979 return LatchControlDependentOnPoison && loopHasNoAbnormalExits(L);
6980}
6981
6982ScalarEvolution::LoopProperties
6983ScalarEvolution::getLoopProperties(const Loop *L) {
6984 using LoopProperties = ScalarEvolution::LoopProperties;
6985
6986 auto Itr = LoopPropertiesCache.find(L);
6987 if (Itr == LoopPropertiesCache.end()) {
6988 auto HasSideEffects = [](Instruction *I) {
6989 if (auto *SI = dyn_cast<StoreInst>(I))
6990 return !SI->isSimple();
6991
6992 return I->mayThrow() || I->mayWriteToMemory();
6993 };
6994
6995 LoopProperties LP = {/* HasNoAbnormalExits */ true,
6996 /*HasNoSideEffects*/ true};
6997
6998 for (auto *BB : L->getBlocks())
6999 for (auto &I : *BB) {
7000 if (!isGuaranteedToTransferExecutionToSuccessor(&I))
7001 LP.HasNoAbnormalExits = false;
7002 if (HasSideEffects(&I))
7003 LP.HasNoSideEffects = false;
7004 if (!LP.HasNoAbnormalExits && !LP.HasNoSideEffects)
7005 break; // We're already as pessimistic as we can get.
7006 }
7007
7008 auto InsertPair = LoopPropertiesCache.insert({L, LP});
7009 assert(InsertPair.second && "We just checked!")(static_cast <bool> (InsertPair.second && "We just checked!"
) ? void (0) : __assert_fail ("InsertPair.second && \"We just checked!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7009, __extension__
__PRETTY_FUNCTION__))
;
7010 Itr = InsertPair.first;
7011 }
7012
7013 return Itr->second;
7014}
7015
7016bool ScalarEvolution::loopIsFiniteByAssumption(const Loop *L) {
7017 // A mustprogress loop without side effects must be finite.
7018 // TODO: The check used here is very conservative. It's only *specific*
7019 // side effects which are well defined in infinite loops.
7020 return isMustProgress(L) && loopHasNoSideEffects(L);
7021}
7022
7023const SCEV *ScalarEvolution::createSCEV(Value *V) {
7024 if (!isSCEVable(V->getType()))
7025 return getUnknown(V);
7026
7027 if (Instruction *I = dyn_cast<Instruction>(V)) {
7028 // Don't attempt to analyze instructions in blocks that aren't
7029 // reachable. Such instructions don't matter, and they aren't required
7030 // to obey basic rules for definitions dominating uses which this
7031 // analysis depends on.
7032 if (!DT.isReachableFromEntry(I->getParent()))
7033 return getUnknown(UndefValue::get(V->getType()));
7034 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(V))
7035 return getConstant(CI);
7036 else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V))
7037 return GA->isInterposable() ? getUnknown(V) : getSCEV(GA->getAliasee());
7038 else if (!isa<ConstantExpr>(V))
7039 return getUnknown(V);
7040
7041 Operator *U = cast<Operator>(V);
7042 if (auto BO = MatchBinaryOp(U, DT)) {
7043 switch (BO->Opcode) {
7044 case Instruction::Add: {
7045 // The simple thing to do would be to just call getSCEV on both operands
7046 // and call getAddExpr with the result. However if we're looking at a
7047 // bunch of things all added together, this can be quite inefficient,
7048 // because it leads to N-1 getAddExpr calls for N ultimate operands.
7049 // Instead, gather up all the operands and make a single getAddExpr call.
7050 // LLVM IR canonical form means we need only traverse the left operands.
7051 SmallVector<const SCEV *, 4> AddOps;
7052 do {
7053 if (BO->Op) {
7054 if (auto *OpSCEV = getExistingSCEV(BO->Op)) {
7055 AddOps.push_back(OpSCEV);
7056 break;
7057 }
7058
7059 // If a NUW or NSW flag can be applied to the SCEV for this
7060 // addition, then compute the SCEV for this addition by itself
7061 // with a separate call to getAddExpr. We need to do that
7062 // instead of pushing the operands of the addition onto AddOps,
7063 // since the flags are only known to apply to this particular
7064 // addition - they may not apply to other additions that can be
7065 // formed with operands from AddOps.
7066 const SCEV *RHS = getSCEV(BO->RHS);
7067 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(BO->Op);
7068 if (Flags != SCEV::FlagAnyWrap) {
7069 const SCEV *LHS = getSCEV(BO->LHS);
7070 if (BO->Opcode == Instruction::Sub)
7071 AddOps.push_back(getMinusSCEV(LHS, RHS, Flags));
7072 else
7073 AddOps.push_back(getAddExpr(LHS, RHS, Flags));
7074 break;
7075 }
7076 }
7077
7078 if (BO->Opcode == Instruction::Sub)
7079 AddOps.push_back(getNegativeSCEV(getSCEV(BO->RHS)));
7080 else
7081 AddOps.push_back(getSCEV(BO->RHS));
7082
7083 auto NewBO = MatchBinaryOp(BO->LHS, DT);
7084 if (!NewBO || (NewBO->Opcode != Instruction::Add &&
7085 NewBO->Opcode != Instruction::Sub)) {
7086 AddOps.push_back(getSCEV(BO->LHS));
7087 break;
7088 }
7089 BO = NewBO;
7090 } while (true);
7091
7092 return getAddExpr(AddOps);
7093 }
7094
7095 case Instruction::Mul: {
7096 SmallVector<const SCEV *, 4> MulOps;
7097 do {
7098 if (BO->Op) {
7099 if (auto *OpSCEV = getExistingSCEV(BO->Op)) {
7100 MulOps.push_back(OpSCEV);
7101 break;
7102 }
7103
7104 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(BO->Op);
7105 if (Flags != SCEV::FlagAnyWrap) {
7106 MulOps.push_back(
7107 getMulExpr(getSCEV(BO->LHS), getSCEV(BO->RHS), Flags));
7108 break;
7109 }
7110 }
7111
7112 MulOps.push_back(getSCEV(BO->RHS));
7113 auto NewBO = MatchBinaryOp(BO->LHS, DT);
7114 if (!NewBO || NewBO->Opcode != Instruction::Mul) {
7115 MulOps.push_back(getSCEV(BO->LHS));
7116 break;
7117 }
7118 BO = NewBO;
7119 } while (true);
7120
7121 return getMulExpr(MulOps);
7122 }
7123 case Instruction::UDiv:
7124 return getUDivExpr(getSCEV(BO->LHS), getSCEV(BO->RHS));
7125 case Instruction::URem:
7126 return getURemExpr(getSCEV(BO->LHS), getSCEV(BO->RHS));
7127 case Instruction::Sub: {
7128 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
7129 if (BO->Op)
7130 Flags = getNoWrapFlagsFromUB(BO->Op);
7131 return getMinusSCEV(getSCEV(BO->LHS), getSCEV(BO->RHS), Flags);
7132 }
7133 case Instruction::And:
7134 // For an expression like x&255 that merely masks off the high bits,
7135 // use zext(trunc(x)) as the SCEV expression.
7136 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) {
7137 if (CI->isZero())
7138 return getSCEV(BO->RHS);
7139 if (CI->isMinusOne())
7140 return getSCEV(BO->LHS);
7141 const APInt &A = CI->getValue();
7142
7143 // Instcombine's ShrinkDemandedConstant may strip bits out of
7144 // constants, obscuring what would otherwise be a low-bits mask.
7145 // Use computeKnownBits to compute what ShrinkDemandedConstant
7146 // knew about to reconstruct a low-bits mask value.
7147 unsigned LZ = A.countLeadingZeros();
7148 unsigned TZ = A.countTrailingZeros();
7149 unsigned BitWidth = A.getBitWidth();
7150 KnownBits Known(BitWidth);
7151 computeKnownBits(BO->LHS, Known, getDataLayout(),
7152 0, &AC, nullptr, &DT);
7153
7154 APInt EffectiveMask =
7155 APInt::getLowBitsSet(BitWidth, BitWidth - LZ - TZ).shl(TZ);
7156 if ((LZ != 0 || TZ != 0) && !((~A & ~Known.Zero) & EffectiveMask)) {
7157 const SCEV *MulCount = getConstant(APInt::getOneBitSet(BitWidth, TZ));
7158 const SCEV *LHS = getSCEV(BO->LHS);
7159 const SCEV *ShiftedLHS = nullptr;
7160 if (auto *LHSMul = dyn_cast<SCEVMulExpr>(LHS)) {
7161 if (auto *OpC = dyn_cast<SCEVConstant>(LHSMul->getOperand(0))) {
7162 // For an expression like (x * 8) & 8, simplify the multiply.
7163 unsigned MulZeros = OpC->getAPInt().countTrailingZeros();
7164 unsigned GCD = std::min(MulZeros, TZ);
7165 APInt DivAmt = APInt::getOneBitSet(BitWidth, TZ - GCD);
7166 SmallVector<const SCEV*, 4> MulOps;
7167 MulOps.push_back(getConstant(OpC->getAPInt().lshr(GCD)));
7168 MulOps.append(LHSMul->op_begin() + 1, LHSMul->op_end());
7169 auto *NewMul = getMulExpr(MulOps, LHSMul->getNoWrapFlags());
7170 ShiftedLHS = getUDivExpr(NewMul, getConstant(DivAmt));
7171 }
7172 }
7173 if (!ShiftedLHS)
7174 ShiftedLHS = getUDivExpr(LHS, MulCount);
7175 return getMulExpr(
7176 getZeroExtendExpr(
7177 getTruncateExpr(ShiftedLHS,
7178 IntegerType::get(getContext(), BitWidth - LZ - TZ)),
7179 BO->LHS->getType()),
7180 MulCount);
7181 }
7182 }
7183 break;
7184
7185 case Instruction::Or:
7186 // If the RHS of the Or is a constant, we may have something like:
7187 // X*4+1 which got turned into X*4|1. Handle this as an Add so loop
7188 // optimizations will transparently handle this case.
7189 //
7190 // In order for this transformation to be safe, the LHS must be of the
7191 // form X*(2^n) and the Or constant must be less than 2^n.
7192 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) {
7193 const SCEV *LHS = getSCEV(BO->LHS);
7194 const APInt &CIVal = CI->getValue();
7195 if (GetMinTrailingZeros(LHS) >=
7196 (CIVal.getBitWidth() - CIVal.countLeadingZeros())) {
7197 // Build a plain add SCEV.
7198 return getAddExpr(LHS, getSCEV(CI),
7199 (SCEV::NoWrapFlags)(SCEV::FlagNUW | SCEV::FlagNSW));
7200 }
7201 }
7202 break;
7203
7204 case Instruction::Xor:
7205 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) {
7206 // If the RHS of xor is -1, then this is a not operation.
7207 if (CI->isMinusOne())
7208 return getNotSCEV(getSCEV(BO->LHS));
7209
7210 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
7211 // This is a variant of the check for xor with -1, and it handles
7212 // the case where instcombine has trimmed non-demanded bits out
7213 // of an xor with -1.
7214 if (auto *LBO = dyn_cast<BinaryOperator>(BO->LHS))
7215 if (ConstantInt *LCI = dyn_cast<ConstantInt>(LBO->getOperand(1)))
7216 if (LBO->getOpcode() == Instruction::And &&
7217 LCI->getValue() == CI->getValue())
7218 if (const SCEVZeroExtendExpr *Z =
7219 dyn_cast<SCEVZeroExtendExpr>(getSCEV(BO->LHS))) {
7220 Type *UTy = BO->LHS->getType();
7221 const SCEV *Z0 = Z->getOperand();
7222 Type *Z0Ty = Z0->getType();
7223 unsigned Z0TySize = getTypeSizeInBits(Z0Ty);
7224
7225 // If C is a low-bits mask, the zero extend is serving to
7226 // mask off the high bits. Complement the operand and
7227 // re-apply the zext.
7228 if (CI->getValue().isMask(Z0TySize))
7229 return getZeroExtendExpr(getNotSCEV(Z0), UTy);
7230
7231 // If C is a single bit, it may be in the sign-bit position
7232 // before the zero-extend. In this case, represent the xor
7233 // using an add, which is equivalent, and re-apply the zext.
7234 APInt Trunc = CI->getValue().trunc(Z0TySize);
7235 if (Trunc.zext(getTypeSizeInBits(UTy)) == CI->getValue() &&
7236 Trunc.isSignMask())
7237 return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)),
7238 UTy);
7239 }
7240 }
7241 break;
7242
7243 case Instruction::Shl:
7244 // Turn shift left of a constant amount into a multiply.
7245 if (ConstantInt *SA = dyn_cast<ConstantInt>(BO->RHS)) {
7246 uint32_t BitWidth = cast<IntegerType>(SA->getType())->getBitWidth();
7247
7248 // If the shift count is not less than the bitwidth, the result of
7249 // the shift is undefined. Don't try to analyze it, because the
7250 // resolution chosen here may differ from the resolution chosen in
7251 // other parts of the compiler.
7252 if (SA->getValue().uge(BitWidth))
7253 break;
7254
7255 // We can safely preserve the nuw flag in all cases. It's also safe to
7256 // turn a nuw nsw shl into a nuw nsw mul. However, nsw in isolation
7257 // requires special handling. It can be preserved as long as we're not
7258 // left shifting by bitwidth - 1.
7259 auto Flags = SCEV::FlagAnyWrap;
7260 if (BO->Op) {
7261 auto MulFlags = getNoWrapFlagsFromUB(BO->Op);
7262 if ((MulFlags & SCEV::FlagNSW) &&
7263 ((MulFlags & SCEV::FlagNUW) || SA->getValue().ult(BitWidth - 1)))
7264 Flags = (SCEV::NoWrapFlags)(Flags | SCEV::FlagNSW);
7265 if (MulFlags & SCEV::FlagNUW)
7266 Flags = (SCEV::NoWrapFlags)(Flags | SCEV::FlagNUW);
7267 }
7268
7269 Constant *X = ConstantInt::get(
7270 getContext(), APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
7271 return getMulExpr(getSCEV(BO->LHS), getSCEV(X), Flags);
7272 }
7273 break;
7274
7275 case Instruction::AShr: {
7276 // AShr X, C, where C is a constant.
7277 ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS);
7278 if (!CI)
7279 break;
7280
7281 Type *OuterTy = BO->LHS->getType();
7282 uint64_t BitWidth = getTypeSizeInBits(OuterTy);
7283 // If the shift count is not less than the bitwidth, the result of
7284 // the shift is undefined. Don't try to analyze it, because the
7285 // resolution chosen here may differ from the resolution chosen in
7286 // other parts of the compiler.
7287 if (CI->getValue().uge(BitWidth))
7288 break;
7289
7290 if (CI->isZero())
7291 return getSCEV(BO->LHS); // shift by zero --> noop
7292
7293 uint64_t AShrAmt = CI->getZExtValue();
7294 Type *TruncTy = IntegerType::get(getContext(), BitWidth - AShrAmt);
7295
7296 Operator *L = dyn_cast<Operator>(BO->LHS);
7297 if (L && L->getOpcode() == Instruction::Shl) {
7298 // X = Shl A, n
7299 // Y = AShr X, m
7300 // Both n and m are constant.
7301
7302 const SCEV *ShlOp0SCEV = getSCEV(L->getOperand(0));
7303 if (L->getOperand(1) == BO->RHS)
7304 // For a two-shift sext-inreg, i.e. n = m,
7305 // use sext(trunc(x)) as the SCEV expression.
7306 return getSignExtendExpr(
7307 getTruncateExpr(ShlOp0SCEV, TruncTy), OuterTy);
7308
7309 ConstantInt *ShlAmtCI = dyn_cast<ConstantInt>(L->getOperand(1));
7310 if (ShlAmtCI && ShlAmtCI->getValue().ult(BitWidth)) {
7311 uint64_t ShlAmt = ShlAmtCI->getZExtValue();
7312 if (ShlAmt > AShrAmt) {
7313 // When n > m, use sext(mul(trunc(x), 2^(n-m)))) as the SCEV
7314 // expression. We already checked that ShlAmt < BitWidth, so
7315 // the multiplier, 1 << (ShlAmt - AShrAmt), fits into TruncTy as
7316 // ShlAmt - AShrAmt < Amt.
7317 APInt Mul = APInt::getOneBitSet(BitWidth - AShrAmt,
7318 ShlAmt - AShrAmt);
7319 return getSignExtendExpr(
7320 getMulExpr(getTruncateExpr(ShlOp0SCEV, TruncTy),
7321 getConstant(Mul)), OuterTy);
7322 }
7323 }
7324 }
7325 break;
7326 }
7327 }
7328 }
7329
7330 switch (U->getOpcode()) {
7331 case Instruction::Trunc:
7332 return getTruncateExpr(getSCEV(U->getOperand(0)), U->getType());
7333
7334 case Instruction::ZExt:
7335 return getZeroExtendExpr(getSCEV(U->getOperand(0)), U->getType());
7336
7337 case Instruction::SExt:
7338 if (auto BO = MatchBinaryOp(U->getOperand(0), DT)) {
7339 // The NSW flag of a subtract does not always survive the conversion to
7340 // A + (-1)*B. By pushing sign extension onto its operands we are much
7341 // more likely to preserve NSW and allow later AddRec optimisations.
7342 //
7343 // NOTE: This is effectively duplicating this logic from getSignExtend:
7344 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
7345 // but by that point the NSW information has potentially been lost.
7346 if (BO->Opcode == Instruction::Sub && BO->IsNSW) {
7347 Type *Ty = U->getType();
7348 auto *V1 = getSignExtendExpr(getSCEV(BO->LHS), Ty);
7349 auto *V2 = getSignExtendExpr(getSCEV(BO->RHS), Ty);
7350 return getMinusSCEV(V1, V2, SCEV::FlagNSW);
7351 }
7352 }
7353 return getSignExtendExpr(getSCEV(U->getOperand(0)), U->getType());
7354
7355 case Instruction::BitCast:
7356 // BitCasts are no-op casts so we just eliminate the cast.
7357 if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType()))
7358 return getSCEV(U->getOperand(0));
7359 break;
7360
7361 case Instruction::PtrToInt: {
7362 // Pointer to integer cast is straight-forward, so do model it.
7363 const SCEV *Op = getSCEV(U->getOperand(0));
7364 Type *DstIntTy = U->getType();
7365 // But only if effective SCEV (integer) type is wide enough to represent
7366 // all possible pointer values.
7367 const SCEV *IntOp = getPtrToIntExpr(Op, DstIntTy);
7368 if (isa<SCEVCouldNotCompute>(IntOp))
7369 return getUnknown(V);
7370 return IntOp;
7371 }
7372 case Instruction::IntToPtr:
7373 // Just don't deal with inttoptr casts.
7374 return getUnknown(V);
7375
7376 case Instruction::SDiv:
7377 // If both operands are non-negative, this is just an udiv.
7378 if (isKnownNonNegative(getSCEV(U->getOperand(0))) &&
7379 isKnownNonNegative(getSCEV(U->getOperand(1))))
7380 return getUDivExpr(getSCEV(U->getOperand(0)), getSCEV(U->getOperand(1)));
7381 break;
7382
7383 case Instruction::SRem:
7384 // If both operands are non-negative, this is just an urem.
7385 if (isKnownNonNegative(getSCEV(U->getOperand(0))) &&
7386 isKnownNonNegative(getSCEV(U->getOperand(1))))
7387 return getURemExpr(getSCEV(U->getOperand(0)), getSCEV(U->getOperand(1)));
7388 break;
7389
7390 case Instruction::GetElementPtr:
7391 return createNodeForGEP(cast<GEPOperator>(U));
7392
7393 case Instruction::PHI:
7394 return createNodeForPHI(cast<PHINode>(U));
7395
7396 case Instruction::Select:
7397 // U can also be a select constant expr, which let fall through. Since
7398 // createNodeForSelect only works for a condition that is an `ICmpInst`, and
7399 // constant expressions cannot have instructions as operands, we'd have
7400 // returned getUnknown for a select constant expressions anyway.
7401 if (isa<Instruction>(U))
7402 return createNodeForSelectOrPHI(cast<Instruction>(U), U->getOperand(0),
7403 U->getOperand(1), U->getOperand(2));
7404 break;
7405
7406 case Instruction::Call:
7407 case Instruction::Invoke:
7408 if (Value *RV = cast<CallBase>(U)->getReturnedArgOperand())
7409 return getSCEV(RV);
7410
7411 if (auto *II = dyn_cast<IntrinsicInst>(U)) {
7412 switch (II->getIntrinsicID()) {
7413 case Intrinsic::abs:
7414 return getAbsExpr(
7415 getSCEV(II->getArgOperand(0)),
7416 /*IsNSW=*/cast<ConstantInt>(II->getArgOperand(1))->isOne());
7417 case Intrinsic::umax:
7418 return getUMaxExpr(getSCEV(II->getArgOperand(0)),
7419 getSCEV(II->getArgOperand(1)));
7420 case Intrinsic::umin:
7421 return getUMinExpr(getSCEV(II->getArgOperand(0)),
7422 getSCEV(II->getArgOperand(1)));
7423 case Intrinsic::smax:
7424 return getSMaxExpr(getSCEV(II->getArgOperand(0)),
7425 getSCEV(II->getArgOperand(1)));
7426 case Intrinsic::smin:
7427 return getSMinExpr(getSCEV(II->getArgOperand(0)),
7428 getSCEV(II->getArgOperand(1)));
7429 case Intrinsic::usub_sat: {
7430 const SCEV *X = getSCEV(II->getArgOperand(0));
7431 const SCEV *Y = getSCEV(II->getArgOperand(1));
7432 const SCEV *ClampedY = getUMinExpr(X, Y);
7433 return getMinusSCEV(X, ClampedY, SCEV::FlagNUW);
7434 }
7435 case Intrinsic::uadd_sat: {
7436 const SCEV *X = getSCEV(II->getArgOperand(0));
7437 const SCEV *Y = getSCEV(II->getArgOperand(1));
7438 const SCEV *ClampedX = getUMinExpr(X, getNotSCEV(Y));
7439 return getAddExpr(ClampedX, Y, SCEV::FlagNUW);
7440 }
7441 case Intrinsic::start_loop_iterations:
7442 // A start_loop_iterations is just equivalent to the first operand for
7443 // SCEV purposes.
7444 return getSCEV(II->getArgOperand(0));
7445 default:
7446 break;
7447 }
7448 }
7449 break;
7450 }
7451
7452 return getUnknown(V);
7453}
7454
7455//===----------------------------------------------------------------------===//
7456// Iteration Count Computation Code
7457//
7458
7459const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount,
7460 bool Extend) {
7461 if (isa<SCEVCouldNotCompute>(ExitCount))
7462 return getCouldNotCompute();
7463
7464 auto *ExitCountType = ExitCount->getType();
7465 assert(ExitCountType->isIntegerTy())(static_cast <bool> (ExitCountType->isIntegerTy()) ?
void (0) : __assert_fail ("ExitCountType->isIntegerTy()",
"llvm/lib/Analysis/ScalarEvolution.cpp", 7465, __extension__
__PRETTY_FUNCTION__))
;
7466
7467 if (!Extend)
7468 return getAddExpr(ExitCount, getOne(ExitCountType));
7469
7470 auto *WiderType = Type::getIntNTy(ExitCountType->getContext(),
7471 1 + ExitCountType->getScalarSizeInBits());
7472 return getAddExpr(getNoopOrZeroExtend(ExitCount, WiderType),
7473 getOne(WiderType));
7474}
7475
7476static unsigned getConstantTripCount(const SCEVConstant *ExitCount) {
7477 if (!ExitCount)
7478 return 0;
7479
7480 ConstantInt *ExitConst = ExitCount->getValue();
7481
7482 // Guard against huge trip counts.
7483 if (ExitConst->getValue().getActiveBits() > 32)
7484 return 0;
7485
7486 // In case of integer overflow, this returns 0, which is correct.
7487 return ((unsigned)ExitConst->getZExtValue()) + 1;
7488}
7489
7490unsigned ScalarEvolution::getSmallConstantTripCount(const Loop *L) {
7491 auto *ExitCount = dyn_cast<SCEVConstant>(getBackedgeTakenCount(L, Exact));
7492 return getConstantTripCount(ExitCount);
7493}
7494
7495unsigned
7496ScalarEvolution::getSmallConstantTripCount(const Loop *L,
7497 const BasicBlock *ExitingBlock) {
7498 assert(ExitingBlock && "Must pass a non-null exiting block!")(static_cast <bool> (ExitingBlock && "Must pass a non-null exiting block!"
) ? void (0) : __assert_fail ("ExitingBlock && \"Must pass a non-null exiting block!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7498, __extension__
__PRETTY_FUNCTION__))
;
7499 assert(L->isLoopExiting(ExitingBlock) &&(static_cast <bool> (L->isLoopExiting(ExitingBlock) &&
"Exiting block must actually branch out of the loop!") ? void
(0) : __assert_fail ("L->isLoopExiting(ExitingBlock) && \"Exiting block must actually branch out of the loop!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7500, __extension__
__PRETTY_FUNCTION__))
7500 "Exiting block must actually branch out of the loop!")(static_cast <bool> (L->isLoopExiting(ExitingBlock) &&
"Exiting block must actually branch out of the loop!") ? void
(0) : __assert_fail ("L->isLoopExiting(ExitingBlock) && \"Exiting block must actually branch out of the loop!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7500, __extension__
__PRETTY_FUNCTION__))
;
7501 const SCEVConstant *ExitCount =
7502 dyn_cast<SCEVConstant>(getExitCount(L, ExitingBlock));
7503 return getConstantTripCount(ExitCount);
7504}
7505
7506unsigned ScalarEvolution::getSmallConstantMaxTripCount(const Loop *L) {
7507 const auto *MaxExitCount =
7508 dyn_cast<SCEVConstant>(getConstantMaxBackedgeTakenCount(L));
7509 return getConstantTripCount(MaxExitCount);
7510}
7511
7512const SCEV *ScalarEvolution::getConstantMaxTripCountFromArray(const Loop *L) {
7513 // We can't infer from Array in Irregular Loop.
7514 // FIXME: It's hard to infer loop bound from array operated in Nested Loop.
7515 if (!L->isLoopSimplifyForm() || !L->isInnermost())
1
Assuming the condition is false
2
Assuming the condition is false
3
Taking false branch
7516 return getCouldNotCompute();
7517
7518 // FIXME: To make the scene more typical, we only analysis loops that have
7519 // one exiting block and that block must be the latch. To make it easier to
7520 // capture loops that have memory access and memory access will be executed
7521 // in each iteration.
7522 const BasicBlock *LoopLatch = L->getLoopLatch();
7523 assert(LoopLatch && "See defination of simplify form loop.")(static_cast <bool> (LoopLatch && "See defination of simplify form loop."
) ? void (0) : __assert_fail ("LoopLatch && \"See defination of simplify form loop.\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7523, __extension__
__PRETTY_FUNCTION__))
;
4
Assuming 'LoopLatch' is non-null
5
'?' condition is true
7524 if (L->getExitingBlock() != LoopLatch)
6
Assuming the condition is false
7
Taking false branch
7525 return getCouldNotCompute();
7526
7527 const DataLayout &DL = getDataLayout();
7528 SmallVector<const SCEV *> InferCountColl;
7529 for (auto *BB : L->getBlocks()) {
8
Assuming '__begin1' is not equal to '__end1'
7530 // Go here, we can know that Loop is a single exiting and simplified form
7531 // loop. Make sure that infer from Memory Operation in those BBs must be
7532 // executed in loop. First step, we can make sure that max execution time
7533 // of MemAccessBB in loop represents latch max excution time.
7534 // If MemAccessBB does not dom Latch, skip.
7535 // Entry
7536 // │
7537 // ┌─────▼─────┐
7538 // │Loop Header◄─────┐
7539 // └──┬──────┬─┘ │
7540 // │ │ │
7541 // ┌────────▼──┐ ┌─▼─────┐ │
7542 // │MemAccessBB│ │OtherBB│ │
7543 // └────────┬──┘ └─┬─────┘ │
7544 // │ │ │
7545 // ┌─▼──────▼─┐ │
7546 // │Loop Latch├─────┘
7547 // └────┬─────┘
7548 // ▼
7549 // Exit
7550 if (!DT.dominates(BB, LoopLatch))
9
Assuming the condition is false
10
Taking false branch
7551 continue;
7552
7553 for (Instruction &Inst : *BB) {
7554 // Find Memory Operation Instruction.
7555 auto *GEP = getLoadStorePointerOperand(&Inst);
11
Calling 'getLoadStorePointerOperand'
28
Returning from 'getLoadStorePointerOperand'
7556 if (!GEP
28.1
'GEP' is non-null
28.1
'GEP' is non-null
28.1
'GEP' is non-null
28.1
'GEP' is non-null
)
29
Taking false branch
7557 continue;
7558
7559 auto *ElemSize = dyn_cast<SCEVConstant>(getElementSize(&Inst));
7560 // Do not infer from scalar type, eg."ElemSize = sizeof()".
7561 if (!ElemSize)
30
Assuming 'ElemSize' is non-null
31
Taking false branch
7562 continue;
7563
7564 // Use a existing polynomial recurrence on the trip count.
7565 auto *AddRec = dyn_cast<SCEVAddRecExpr>(getSCEV(GEP));
32
Assuming the object is a 'SCEVAddRecExpr'
7566 if (!AddRec
32.1
'AddRec' is non-null
32.1
'AddRec' is non-null
32.1
'AddRec' is non-null
32.1
'AddRec' is non-null
)
33
Taking false branch
7567 continue;
7568 auto *ArrBase = dyn_cast<SCEVUnknown>(getPointerBase(AddRec));
34
Assuming the object is a 'SCEVUnknown'
7569 auto *Step = dyn_cast<SCEVConstant>(AddRec->getStepRecurrence(*this));
35
Assuming the object is a 'SCEVConstant'
7570 if (!ArrBase
35.1
'ArrBase' is non-null
35.1
'ArrBase' is non-null
35.1
'ArrBase' is non-null
35.1
'ArrBase' is non-null
|| !Step
35.2
'Step' is non-null
35.2
'Step' is non-null
35.2
'Step' is non-null
35.2
'Step' is non-null
)
7571 continue;
7572 assert(isLoopInvariant(ArrBase, L) && "See addrec definition")(static_cast <bool> (isLoopInvariant(ArrBase, L) &&
"See addrec definition") ? void (0) : __assert_fail ("isLoopInvariant(ArrBase, L) && \"See addrec definition\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7572, __extension__
__PRETTY_FUNCTION__))
;
36
Taking false branch
37
'?' condition is true
7573
7574 // Only handle { %array + step },
7575 // FIXME: {(SCEVAddRecExpr) + step } could not be analysed here.
7576 if (AddRec->getStart() != ArrBase)
38
Assuming the condition is false
7577 continue;
7578
7579 // Memory operation pattern which have gaps.
7580 // Or repeat memory opreation.
7581 // And index of GEP wraps arround.
7582 if (Step->getAPInt().getActiveBits() > 32 ||
39
Assuming the condition is false
55
Taking false branch
7583 Step->getAPInt().getZExtValue() !=
40
Assuming the condition is false
7584 ElemSize->getAPInt().getZExtValue() ||
7585 Step->isZero() || Step->getAPInt().isNegative())
41
Calling 'SCEV::isZero'
45
Returning from 'SCEV::isZero'
46
Calling 'APInt::isNegative'
54
Returning from 'APInt::isNegative'
7586 continue;
7587
7588 // Only infer from stack array which has certain size.
7589 // Make sure alloca instruction is not excuted in loop.
7590 AllocaInst *AllocateInst = dyn_cast<AllocaInst>(ArrBase->getValue());
56
Assuming the object is a 'AllocaInst'
7591 if (!AllocateInst
56.1
'AllocateInst' is non-null
56.1
'AllocateInst' is non-null
56.1
'AllocateInst' is non-null
56.1
'AllocateInst' is non-null
|| L->contains(AllocateInst->getParent()))
57
Assuming the condition is false
58
Taking false branch
7592 continue;
7593
7594 // Make sure only handle normal array.
7595 auto *Ty = dyn_cast<ArrayType>(AllocateInst->getAllocatedType());
59
Assuming the object is a 'ArrayType'
7596 auto *ArrSize = dyn_cast<ConstantInt>(AllocateInst->getArraySize());
7597 if (!Ty
59.1
'Ty' is non-null
59.1
'Ty' is non-null
59.1
'Ty' is non-null
59.1
'Ty' is non-null
|| !ArrSize || !ArrSize->isOne())
60
Assuming 'ArrSize' is non-null
61
Calling 'ConstantInt::isOne'
72
Returning from 'ConstantInt::isOne'
73
Taking false branch
7598 continue;
7599 // Also make sure step was increased the same with sizeof allocated
7600 // element type.
7601 const PointerType *GEPT = dyn_cast<PointerType>(GEP->getType());
74
Assuming the object is not a 'PointerType'
75
'GEPT' initialized to a null pointer value
7602 if (Ty->getElementType() != GEPT->getElementType())
76
Called C++ object pointer is null
7603 continue;
7604
7605 // FIXME: Since gep indices are silently zext to the indexing type,
7606 // we will have a narrow gep index which wraps around rather than
7607 // increasing strictly, we shoule ensure that step is increasing
7608 // strictly by the loop iteration.
7609 // Now we can infer a max execution time by MemLength/StepLength.
7610 const SCEV *MemSize =
7611 getConstant(Step->getType(), DL.getTypeAllocSize(Ty));
7612 auto *MaxExeCount =
7613 dyn_cast<SCEVConstant>(getUDivCeilSCEV(MemSize, Step));
7614 if (!MaxExeCount || MaxExeCount->getAPInt().getActiveBits() > 32)
7615 continue;
7616
7617 // If the loop reaches the maximum number of executions, we can not
7618 // access bytes starting outside the statically allocated size without
7619 // being immediate UB. But it is allowed to enter loop header one more
7620 // time.
7621 auto *InferCount = dyn_cast<SCEVConstant>(
7622 getAddExpr(MaxExeCount, getOne(MaxExeCount->getType())));
7623 // Discard the maximum number of execution times under 32bits.
7624 if (!InferCount || InferCount->getAPInt().getActiveBits() > 32)
7625 continue;
7626
7627 InferCountColl.push_back(InferCount);
7628 }
7629 }
7630
7631 if (InferCountColl.size() == 0)
7632 return getCouldNotCompute();
7633
7634 return getUMinFromMismatchedTypes(InferCountColl);
7635}
7636
7637unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L) {
7638 SmallVector<BasicBlock *, 8> ExitingBlocks;
7639 L->getExitingBlocks(ExitingBlocks);
7640
7641 Optional<unsigned> Res = None;
7642 for (auto *ExitingBB : ExitingBlocks) {
7643 unsigned Multiple = getSmallConstantTripMultiple(L, ExitingBB);
7644 if (!Res)
7645 Res = Multiple;
7646 Res = (unsigned)GreatestCommonDivisor64(*Res, Multiple);
7647 }
7648 return Res.getValueOr(1);
7649}
7650
7651unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
7652 const SCEV *ExitCount) {
7653 if (ExitCount == getCouldNotCompute())
7654 return 1;
7655
7656 // Get the trip count
7657 const SCEV *TCExpr = getTripCountFromExitCount(ExitCount);
7658
7659 const SCEVConstant *TC = dyn_cast<SCEVConstant>(TCExpr);
7660 if (!TC)
7661 // Attempt to factor more general cases. Returns the greatest power of
7662 // two divisor. If overflow happens, the trip count expression is still
7663 // divisible by the greatest power of 2 divisor returned.
7664 return 1U << std::min((uint32_t)31,
7665 GetMinTrailingZeros(applyLoopGuards(TCExpr, L)));
7666
7667 ConstantInt *Result = TC->getValue();
7668
7669 // Guard against huge trip counts (this requires checking
7670 // for zero to handle the case where the trip count == -1 and the
7671 // addition wraps).
7672 if (!Result || Result->getValue().getActiveBits() > 32 ||
7673 Result->getValue().getActiveBits() == 0)
7674 return 1;
7675
7676 return (unsigned)Result->getZExtValue();
7677}
7678
7679/// Returns the largest constant divisor of the trip count of this loop as a
7680/// normal unsigned value, if possible. This means that the actual trip count is
7681/// always a multiple of the returned value (don't forget the trip count could
7682/// very well be zero as well!).
7683///
7684/// Returns 1 if the trip count is unknown or not guaranteed to be the
7685/// multiple of a constant (which is also the case if the trip count is simply
7686/// constant, use getSmallConstantTripCount for that case), Will also return 1
7687/// if the trip count is very large (>= 2^32).
7688///
7689/// As explained in the comments for getSmallConstantTripCount, this assumes
7690/// that control exits the loop via ExitingBlock.
7691unsigned
7692ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
7693 const BasicBlock *ExitingBlock) {
7694 assert(ExitingBlock && "Must pass a non-null exiting block!")(static_cast <bool> (ExitingBlock && "Must pass a non-null exiting block!"
) ? void (0) : __assert_fail ("ExitingBlock && \"Must pass a non-null exiting block!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7694, __extension__
__PRETTY_FUNCTION__))
;
7695 assert(L->isLoopExiting(ExitingBlock) &&(static_cast <bool> (L->isLoopExiting(ExitingBlock) &&
"Exiting block must actually branch out of the loop!") ? void
(0) : __assert_fail ("L->isLoopExiting(ExitingBlock) && \"Exiting block must actually branch out of the loop!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7696, __extension__
__PRETTY_FUNCTION__))
7696 "Exiting block must actually branch out of the loop!")(static_cast <bool> (L->isLoopExiting(ExitingBlock) &&
"Exiting block must actually branch out of the loop!") ? void
(0) : __assert_fail ("L->isLoopExiting(ExitingBlock) && \"Exiting block must actually branch out of the loop!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7696, __extension__
__PRETTY_FUNCTION__))
;
7697 const SCEV *ExitCount = getExitCount(L, ExitingBlock);
7698 return getSmallConstantTripMultiple(L, ExitCount);
7699}
7700
7701const SCEV *ScalarEvolution::getExitCount(const Loop *L,
7702 const BasicBlock *ExitingBlock,
7703 ExitCountKind Kind) {
7704 switch (Kind) {
7705 case Exact:
7706 case SymbolicMaximum:
7707 return getBackedgeTakenInfo(L).getExact(ExitingBlock, this);
7708 case ConstantMaximum:
7709 return getBackedgeTakenInfo(L).getConstantMax(ExitingBlock, this);
7710 };
7711 llvm_unreachable("Invalid ExitCountKind!")::llvm::llvm_unreachable_internal("Invalid ExitCountKind!", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 7711)
;
7712}
7713
7714const SCEV *
7715ScalarEvolution::getPredicatedBackedgeTakenCount(const Loop *L,
7716 SCEVUnionPredicate &Preds) {
7717 return getPredicatedBackedgeTakenInfo(L).getExact(L, this, &Preds);
7718}
7719
7720const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L,
7721 ExitCountKind Kind) {
7722 switch (Kind) {
7723 case Exact:
7724 return getBackedgeTakenInfo(L).getExact(L, this);
7725 case ConstantMaximum:
7726 return getBackedgeTakenInfo(L).getConstantMax(this);
7727 case SymbolicMaximum:
7728 return getBackedgeTakenInfo(L).getSymbolicMax(L, this);
7729 };
7730 llvm_unreachable("Invalid ExitCountKind!")::llvm::llvm_unreachable_internal("Invalid ExitCountKind!", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 7730)
;
7731}
7732
7733bool ScalarEvolution::isBackedgeTakenCountMaxOrZero(const Loop *L) {
7734 return getBackedgeTakenInfo(L).isConstantMaxOrZero(this);
7735}
7736
7737/// Push PHI nodes in the header of the given loop onto the given Worklist.
7738static void PushLoopPHIs(const Loop *L,
7739 SmallVectorImpl<Instruction *> &Worklist,
7740 SmallPtrSetImpl<Instruction *> &Visited) {
7741 BasicBlock *Header = L->getHeader();
7742
7743 // Push all Loop-header PHIs onto the Worklist stack.
7744 for (PHINode &PN : Header->phis())
7745 if (Visited.insert(&PN).second)
7746 Worklist.push_back(&PN);
7747}
7748
7749const ScalarEvolution::BackedgeTakenInfo &
7750ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) {
7751 auto &BTI = getBackedgeTakenInfo(L);
7752 if (BTI.hasFullInfo())
7753 return BTI;
7754
7755 auto Pair = PredicatedBackedgeTakenCounts.insert({L, BackedgeTakenInfo()});
7756
7757 if (!Pair.second)
7758 return Pair.first->second;
7759
7760 BackedgeTakenInfo Result =
7761 computeBackedgeTakenCount(L, /*AllowPredicates=*/true);
7762
7763 return PredicatedBackedgeTakenCounts.find(L)->second = std::move(Result);
7764}
7765
7766ScalarEvolution::BackedgeTakenInfo &
7767ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
7768 // Initially insert an invalid entry for this loop. If the insertion
7769 // succeeds, proceed to actually compute a backedge-taken count and
7770 // update the value. The temporary CouldNotCompute value tells SCEV
7771 // code elsewhere that it shouldn't attempt to request a new
7772 // backedge-taken count, which could result in infinite recursion.
7773 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
7774 BackedgeTakenCounts.insert({L, BackedgeTakenInfo()});
7775 if (!Pair.second)
7776 return Pair.first->second;
7777
7778 // computeBackedgeTakenCount may allocate memory for its result. Inserting it
7779 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
7780 // must be cleared in this scope.
7781 BackedgeTakenInfo Result = computeBackedgeTakenCount(L);
7782
7783 // In product build, there are no usage of statistic.
7784 (void)NumTripCountsComputed;
7785 (void)NumTripCountsNotComputed;
7786#if LLVM_ENABLE_STATS1 || !defined(NDEBUG)
7787 const SCEV *BEExact = Result.getExact(L, this);
7788 if (BEExact != getCouldNotCompute()) {
7789 assert(isLoopInvariant(BEExact, L) &&(static_cast <bool> (isLoopInvariant(BEExact, L) &&
isLoopInvariant(Result.getConstantMax(this), L) && "Computed backedge-taken count isn't loop invariant for loop!"
) ? void (0) : __assert_fail ("isLoopInvariant(BEExact, L) && isLoopInvariant(Result.getConstantMax(this), L) && \"Computed backedge-taken count isn't loop invariant for loop!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7791, __extension__
__PRETTY_FUNCTION__))
7790 isLoopInvariant(Result.getConstantMax(this), L) &&(static_cast <bool> (isLoopInvariant(BEExact, L) &&
isLoopInvariant(Result.getConstantMax(this), L) && "Computed backedge-taken count isn't loop invariant for loop!"
) ? void (0) : __assert_fail ("isLoopInvariant(BEExact, L) && isLoopInvariant(Result.getConstantMax(this), L) && \"Computed backedge-taken count isn't loop invariant for loop!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7791, __extension__
__PRETTY_FUNCTION__))
7791 "Computed backedge-taken count isn't loop invariant for loop!")(static_cast <bool> (isLoopInvariant(BEExact, L) &&
isLoopInvariant(Result.getConstantMax(this), L) && "Computed backedge-taken count isn't loop invariant for loop!"
) ? void (0) : __assert_fail ("isLoopInvariant(BEExact, L) && isLoopInvariant(Result.getConstantMax(this), L) && \"Computed backedge-taken count isn't loop invariant for loop!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7791, __extension__
__PRETTY_FUNCTION__))
;
7792 ++NumTripCountsComputed;
7793 } else if (Result.getConstantMax(this) == getCouldNotCompute() &&
7794 isa<PHINode>(L->getHeader()->begin())) {
7795 // Only count loops that have phi nodes as not being computable.
7796 ++NumTripCountsNotComputed;
7797 }
7798#endif // LLVM_ENABLE_STATS || !defined(NDEBUG)
7799
7800 // Now that we know more about the trip count for this loop, forget any
7801 // existing SCEV values for PHI nodes in this loop since they are only
7802 // conservative estimates made without the benefit of trip count
7803 // information. This invalidation is not necessary for correctness, and is
7804 // only done to produce more precise results.
7805 if (Result.hasAnyInfo()) {
7806 // Invalidate any expression using an addrec in this loop.
7807 SmallVector<const SCEV *, 8> ToForget;
7808 auto LoopUsersIt = LoopUsers.find(L);
7809 if (LoopUsersIt != LoopUsers.end())
7810 append_range(ToForget, LoopUsersIt->second);
7811 forgetMemoizedResults(ToForget);
7812
7813 // Invalidate constant-evolved loop header phis.
7814 for (PHINode &PN : L->getHeader()->phis())
7815 ConstantEvolutionLoopExitValue.erase(&PN);
7816 }
7817
7818 // Re-lookup the insert position, since the call to
7819 // computeBackedgeTakenCount above could result in a
7820 // recusive call to getBackedgeTakenInfo (on a different
7821 // loop), which would invalidate the iterator computed
7822 // earlier.
7823 return BackedgeTakenCounts.find(L)->second = std::move(Result);
7824}
7825
7826void ScalarEvolution::forgetAllLoops() {
7827 // This method is intended to forget all info about loops. It should
7828 // invalidate caches as if the following happened:
7829 // - The trip counts of all loops have changed arbitrarily
7830 // - Every llvm::Value has been updated in place to produce a different
7831 // result.
7832 BackedgeTakenCounts.clear();
7833 PredicatedBackedgeTakenCounts.clear();
7834 BECountUsers.clear();
7835 LoopPropertiesCache.clear();
7836 ConstantEvolutionLoopExitValue.clear();
7837 ValueExprMap.clear();
7838 ValuesAtScopes.clear();
7839 ValuesAtScopesUsers.clear();
7840 LoopDispositions.clear();
7841 BlockDispositions.clear();
7842 UnsignedRanges.clear();
7843 SignedRanges.clear();
7844 ExprValueMap.clear();
7845 HasRecMap.clear();
7846 MinTrailingZerosCache.clear();
7847 PredicatedSCEVRewrites.clear();
7848}
7849
7850void ScalarEvolution::forgetLoop(const Loop *L) {
7851 SmallVector<const Loop *, 16> LoopWorklist(1, L);
7852 SmallVector<Instruction *, 32> Worklist;
7853 SmallPtrSet<Instruction *, 16> Visited;
7854 SmallVector<const SCEV *, 16> ToForget;
7855
7856 // Iterate over all the loops and sub-loops to drop SCEV information.
7857 while (!LoopWorklist.empty()) {
7858 auto *CurrL = LoopWorklist.pop_back_val();
7859
7860 // Drop any stored trip count value.
7861 forgetBackedgeTakenCounts(CurrL, /* Predicated */ false);
7862 forgetBackedgeTakenCounts(CurrL, /* Predicated */ true);
7863
7864 // Drop information about predicated SCEV rewrites for this loop.
7865 for (auto I = PredicatedSCEVRewrites.begin();
7866 I != PredicatedSCEVRewrites.end();) {
7867 std::pair<const SCEV *, const Loop *> Entry = I->first;
7868 if (Entry.second == CurrL)
7869 PredicatedSCEVRewrites.erase(I++);
7870 else
7871 ++I;
7872 }
7873
7874 auto LoopUsersItr = LoopUsers.find(CurrL);
7875 if (LoopUsersItr != LoopUsers.end()) {
7876 ToForget.insert(ToForget.end(), LoopUsersItr->second.begin(),
7877 LoopUsersItr->second.end());
7878 LoopUsers.erase(LoopUsersItr);
7879 }
7880
7881 // Drop information about expressions based on loop-header PHIs.
7882 PushLoopPHIs(CurrL, Worklist, Visited);
7883
7884 while (!Worklist.empty()) {
7885 Instruction *I = Worklist.pop_back_val();
7886
7887 ValueExprMapType::iterator It =
7888 ValueExprMap.find_as(static_cast<Value *>(I));
7889 if (It != ValueExprMap.end()) {
7890 eraseValueFromMap(It->first);
7891 ToForget.push_back(It->second);
7892 if (PHINode *PN = dyn_cast<PHINode>(I))
7893 ConstantEvolutionLoopExitValue.erase(PN);
7894 }
7895
7896 PushDefUseChildren(I, Worklist, Visited);
7897 }
7898
7899 LoopPropertiesCache.erase(CurrL);
7900 // Forget all contained loops too, to avoid dangling entries in the
7901 // ValuesAtScopes map.
7902 LoopWorklist.append(CurrL->begin(), CurrL->end());
7903 }
7904 forgetMemoizedResults(ToForget);
7905}
7906
7907void ScalarEvolution::forgetTopmostLoop(const Loop *L) {
7908 while (Loop *Parent = L->getParentLoop())
7909 L = Parent;
7910 forgetLoop(L);
7911}
7912
7913void ScalarEvolution::forgetValue(Value *V) {
7914 Instruction *I = dyn_cast<Instruction>(V);
7915 if (!I) return;
7916
7917 // Drop information about expressions based on loop-header PHIs.
7918 SmallVector<Instruction *, 16> Worklist;
7919 SmallPtrSet<Instruction *, 8> Visited;
7920 SmallVector<const SCEV *, 8> ToForget;
7921 Worklist.push_back(I);
7922 Visited.insert(I);
7923
7924 while (!Worklist.empty()) {
7925 I = Worklist.pop_back_val();
7926 ValueExprMapType::iterator It =
7927 ValueExprMap.find_as(static_cast<Value *>(I));
7928 if (It != ValueExprMap.end()) {
7929 eraseValueFromMap(It->first);
7930 ToForget.push_back(It->second);
7931 if (PHINode *PN = dyn_cast<PHINode>(I))
7932 ConstantEvolutionLoopExitValue.erase(PN);
7933 }
7934
7935 PushDefUseChildren(I, Worklist, Visited);
7936 }
7937 forgetMemoizedResults(ToForget);
7938}
7939
7940void ScalarEvolution::forgetLoopDispositions(const Loop *L) {
7941 LoopDispositions.clear();
7942}
7943
7944/// Get the exact loop backedge taken count considering all loop exits. A
7945/// computable result can only be returned for loops with all exiting blocks
7946/// dominating the latch. howFarToZero assumes that the limit of each loop test
7947/// is never skipped. This is a valid assumption as long as the loop exits via
7948/// that test. For precise results, it is the caller's responsibility to specify
7949/// the relevant loop exiting block using getExact(ExitingBlock, SE).
7950const SCEV *
7951ScalarEvolution::BackedgeTakenInfo::getExact(const Loop *L, ScalarEvolution *SE,
7952 SCEVUnionPredicate *Preds) const {
7953 // If any exits were not computable, the loop is not computable.
7954 if (!isComplete() || ExitNotTaken.empty())
7955 return SE->getCouldNotCompute();
7956
7957 const BasicBlock *Latch = L->getLoopLatch();
7958 // All exiting blocks we have collected must dominate the only backedge.
7959 if (!Latch)
7960 return SE->getCouldNotCompute();
7961
7962 // All exiting blocks we have gathered dominate loop's latch, so exact trip
7963 // count is simply a minimum out of all these calculated exit counts.
7964 SmallVector<const SCEV *, 2> Ops;
7965 for (auto &ENT : ExitNotTaken) {
7966 const SCEV *BECount = ENT.ExactNotTaken;
7967 assert(BECount != SE->getCouldNotCompute() && "Bad exit SCEV!")(static_cast <bool> (BECount != SE->getCouldNotCompute
() && "Bad exit SCEV!") ? void (0) : __assert_fail ("BECount != SE->getCouldNotCompute() && \"Bad exit SCEV!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7967, __extension__
__PRETTY_FUNCTION__))
;
7968 assert(SE->DT.dominates(ENT.ExitingBlock, Latch) &&(static_cast <bool> (SE->DT.dominates(ENT.ExitingBlock
, Latch) && "We should only have known counts for exiting blocks that dominate "
"latch!") ? void (0) : __assert_fail ("SE->DT.dominates(ENT.ExitingBlock, Latch) && \"We should only have known counts for exiting blocks that dominate \" \"latch!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7970, __extension__
__PRETTY_FUNCTION__))
7969 "We should only have known counts for exiting blocks that dominate "(static_cast <bool> (SE->DT.dominates(ENT.ExitingBlock
, Latch) && "We should only have known counts for exiting blocks that dominate "
"latch!") ? void (0) : __assert_fail ("SE->DT.dominates(ENT.ExitingBlock, Latch) && \"We should only have known counts for exiting blocks that dominate \" \"latch!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7970, __extension__
__PRETTY_FUNCTION__))
7970 "latch!")(static_cast <bool> (SE->DT.dominates(ENT.ExitingBlock
, Latch) && "We should only have known counts for exiting blocks that dominate "
"latch!") ? void (0) : __assert_fail ("SE->DT.dominates(ENT.ExitingBlock, Latch) && \"We should only have known counts for exiting blocks that dominate \" \"latch!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7970, __extension__
__PRETTY_FUNCTION__))
;
7971
7972 Ops.push_back(BECount);
7973
7974 if (Preds && !ENT.hasAlwaysTruePredicate())
7975 Preds->add(ENT.Predicate.get());
7976
7977 assert((Preds || ENT.hasAlwaysTruePredicate()) &&(static_cast <bool> ((Preds || ENT.hasAlwaysTruePredicate
()) && "Predicate should be always true!") ? void (0)
: __assert_fail ("(Preds || ENT.hasAlwaysTruePredicate()) && \"Predicate should be always true!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7978, __extension__
__PRETTY_FUNCTION__))
7978 "Predicate should be always true!")(static_cast <bool> ((Preds || ENT.hasAlwaysTruePredicate
()) && "Predicate should be always true!") ? void (0)
: __assert_fail ("(Preds || ENT.hasAlwaysTruePredicate()) && \"Predicate should be always true!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 7978, __extension__
__PRETTY_FUNCTION__))
;
7979 }
7980
7981 return SE->getUMinFromMismatchedTypes(Ops);
7982}
7983
7984/// Get the exact not taken count for this loop exit.
7985const SCEV *
7986ScalarEvolution::BackedgeTakenInfo::getExact(const BasicBlock *ExitingBlock,
7987 ScalarEvolution *SE) const {
7988 for (auto &ENT : ExitNotTaken)
7989 if (ENT.ExitingBlock == ExitingBlock && ENT.hasAlwaysTruePredicate())
7990 return ENT.ExactNotTaken;
7991
7992 return SE->getCouldNotCompute();
7993}
7994
7995const SCEV *ScalarEvolution::BackedgeTakenInfo::getConstantMax(
7996 const BasicBlock *ExitingBlock, ScalarEvolution *SE) const {
7997 for (auto &ENT : ExitNotTaken)
7998 if (ENT.ExitingBlock == ExitingBlock && ENT.hasAlwaysTruePredicate())
7999 return ENT.MaxNotTaken;
8000
8001 return SE->getCouldNotCompute();
8002}
8003
8004/// getConstantMax - Get the constant max backedge taken count for the loop.
8005const SCEV *
8006ScalarEvolution::BackedgeTakenInfo::getConstantMax(ScalarEvolution *SE) const {
8007 auto PredicateNotAlwaysTrue = [](const ExitNotTakenInfo &ENT) {
8008 return !ENT.hasAlwaysTruePredicate();
8009 };
8010
8011 if (!getConstantMax() || any_of(ExitNotTaken, PredicateNotAlwaysTrue))
8012 return SE->getCouldNotCompute();
8013
8014 assert((isa<SCEVCouldNotCompute>(getConstantMax()) ||(static_cast <bool> ((isa<SCEVCouldNotCompute>(getConstantMax
()) || isa<SCEVConstant>(getConstantMax())) && "No point in having a non-constant max backedge taken count!"
) ? void (0) : __assert_fail ("(isa<SCEVCouldNotCompute>(getConstantMax()) || isa<SCEVConstant>(getConstantMax())) && \"No point in having a non-constant max backedge taken count!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8016, __extension__
__PRETTY_FUNCTION__))
8015 isa<SCEVConstant>(getConstantMax())) &&(static_cast <bool> ((isa<SCEVCouldNotCompute>(getConstantMax
()) || isa<SCEVConstant>(getConstantMax())) && "No point in having a non-constant max backedge taken count!"
) ? void (0) : __assert_fail ("(isa<SCEVCouldNotCompute>(getConstantMax()) || isa<SCEVConstant>(getConstantMax())) && \"No point in having a non-constant max backedge taken count!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8016, __extension__
__PRETTY_FUNCTION__))
8016 "No point in having a non-constant max backedge taken count!")(static_cast <bool> ((isa<SCEVCouldNotCompute>(getConstantMax
()) || isa<SCEVConstant>(getConstantMax())) && "No point in having a non-constant max backedge taken count!"
) ? void (0) : __assert_fail ("(isa<SCEVCouldNotCompute>(getConstantMax()) || isa<SCEVConstant>(getConstantMax())) && \"No point in having a non-constant max backedge taken count!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8016, __extension__
__PRETTY_FUNCTION__))
;
8017 return getConstantMax();
8018}
8019
8020const SCEV *
8021ScalarEvolution::BackedgeTakenInfo::getSymbolicMax(const Loop *L,
8022 ScalarEvolution *SE) {
8023 if (!SymbolicMax)
8024 SymbolicMax = SE->computeSymbolicMaxBackedgeTakenCount(L);
8025 return SymbolicMax;
8026}
8027
8028bool ScalarEvolution::BackedgeTakenInfo::isConstantMaxOrZero(
8029 ScalarEvolution *SE) const {
8030 auto PredicateNotAlwaysTrue = [](const ExitNotTakenInfo &ENT) {
8031 return !ENT.hasAlwaysTruePredicate();
8032 };
8033 return MaxOrZero && !any_of(ExitNotTaken, PredicateNotAlwaysTrue);
8034}
8035
8036ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E)
8037 : ExitLimit(E, E, false, None) {
8038}
8039
8040ScalarEvolution::ExitLimit::ExitLimit(
8041 const SCEV *E, const SCEV *M, bool MaxOrZero,
8042 ArrayRef<const SmallPtrSetImpl<const SCEVPredicate *> *> PredSetList)
8043 : ExactNotTaken(E), MaxNotTaken(M), MaxOrZero(MaxOrZero) {
8044 // If we prove the max count is zero, so is the symbolic bound. This happens
8045 // in practice due to differences in a) how context sensitive we've chosen
8046 // to be and b) how we reason about bounds impied by UB.
8047 if (MaxNotTaken->isZero())
8048 ExactNotTaken = MaxNotTaken;
8049
8050 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||(static_cast <bool> ((isa<SCEVCouldNotCompute>(ExactNotTaken
) || !isa<SCEVCouldNotCompute>(MaxNotTaken)) &&
"Exact is not allowed to be less precise than Max") ? void (
0) : __assert_fail ("(isa<SCEVCouldNotCompute>(ExactNotTaken) || !isa<SCEVCouldNotCompute>(MaxNotTaken)) && \"Exact is not allowed to be less precise than Max\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8052, __extension__
__PRETTY_FUNCTION__))
8051 !isa<SCEVCouldNotCompute>(MaxNotTaken)) &&(static_cast <bool> ((isa<SCEVCouldNotCompute>(ExactNotTaken
) || !isa<SCEVCouldNotCompute>(MaxNotTaken)) &&
"Exact is not allowed to be less precise than Max") ? void (
0) : __assert_fail ("(isa<SCEVCouldNotCompute>(ExactNotTaken) || !isa<SCEVCouldNotCompute>(MaxNotTaken)) && \"Exact is not allowed to be less precise than Max\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8052, __extension__
__PRETTY_FUNCTION__))
8052 "Exact is not allowed to be less precise than Max")(static_cast <bool> ((isa<SCEVCouldNotCompute>(ExactNotTaken
) || !isa<SCEVCouldNotCompute>(MaxNotTaken)) &&
"Exact is not allowed to be less precise than Max") ? void (
0) : __assert_fail ("(isa<SCEVCouldNotCompute>(ExactNotTaken) || !isa<SCEVCouldNotCompute>(MaxNotTaken)) && \"Exact is not allowed to be less precise than Max\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8052, __extension__
__PRETTY_FUNCTION__))
;
8053 assert((isa<SCEVCouldNotCompute>(MaxNotTaken) ||(static_cast <bool> ((isa<SCEVCouldNotCompute>(MaxNotTaken
) || isa<SCEVConstant>(MaxNotTaken)) && "No point in having a non-constant max backedge taken count!"
) ? void (0) : __assert_fail ("(isa<SCEVCouldNotCompute>(MaxNotTaken) || isa<SCEVConstant>(MaxNotTaken)) && \"No point in having a non-constant max backedge taken count!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8055, __extension__
__PRETTY_FUNCTION__))
8054 isa<SCEVConstant>(MaxNotTaken)) &&(static_cast <bool> ((isa<SCEVCouldNotCompute>(MaxNotTaken
) || isa<SCEVConstant>(MaxNotTaken)) && "No point in having a non-constant max backedge taken count!"
) ? void (0) : __assert_fail ("(isa<SCEVCouldNotCompute>(MaxNotTaken) || isa<SCEVConstant>(MaxNotTaken)) && \"No point in having a non-constant max backedge taken count!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8055, __extension__
__PRETTY_FUNCTION__))
8055 "No point in having a non-constant max backedge taken count!")(static_cast <bool> ((isa<SCEVCouldNotCompute>(MaxNotTaken
) || isa<SCEVConstant>(MaxNotTaken)) && "No point in having a non-constant max backedge taken count!"
) ? void (0) : __assert_fail ("(isa<SCEVCouldNotCompute>(MaxNotTaken) || isa<SCEVConstant>(MaxNotTaken)) && \"No point in having a non-constant max backedge taken count!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8055, __extension__
__PRETTY_FUNCTION__))
;
8056 for (auto *PredSet : PredSetList)
8057 for (auto *P : *PredSet)
8058 addPredicate(P);
8059 assert((isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) &&(static_cast <bool> ((isa<SCEVCouldNotCompute>(E)
|| !E->getType()->isPointerTy()) && "Backedge count should be int"
) ? void (0) : __assert_fail ("(isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) && \"Backedge count should be int\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8060, __extension__
__PRETTY_FUNCTION__))
8060 "Backedge count should be int")(static_cast <bool> ((isa<SCEVCouldNotCompute>(E)
|| !E->getType()->isPointerTy()) && "Backedge count should be int"
) ? void (0) : __assert_fail ("(isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) && \"Backedge count should be int\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8060, __extension__
__PRETTY_FUNCTION__))
;
8061 assert((isa<SCEVCouldNotCompute>(M) || !M->getType()->isPointerTy()) &&(static_cast <bool> ((isa<SCEVCouldNotCompute>(M)
|| !M->getType()->isPointerTy()) && "Max backedge count should be int"
) ? void (0) : __assert_fail ("(isa<SCEVCouldNotCompute>(M) || !M->getType()->isPointerTy()) && \"Max backedge count should be int\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8062, __extension__
__PRETTY_FUNCTION__))
8062 "Max backedge count should be int")(static_cast <bool> ((isa<SCEVCouldNotCompute>(M)
|| !M->getType()->isPointerTy()) && "Max backedge count should be int"
) ? void (0) : __assert_fail ("(isa<SCEVCouldNotCompute>(M) || !M->getType()->isPointerTy()) && \"Max backedge count should be int\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8062, __extension__
__PRETTY_FUNCTION__))
;
8063}
8064
8065ScalarEvolution::ExitLimit::ExitLimit(
8066 const SCEV *E, const SCEV *M, bool MaxOrZero,
8067 const SmallPtrSetImpl<const SCEVPredicate *> &PredSet)
8068 : ExitLimit(E, M, MaxOrZero, {&PredSet}) {
8069}
8070
8071ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E, const SCEV *M,
8072 bool MaxOrZero)
8073 : ExitLimit(E, M, MaxOrZero, None) {
8074}
8075
8076/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
8077/// computable exit into a persistent ExitNotTakenInfo array.
8078ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
8079 ArrayRef<ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo> ExitCounts,
8080 bool IsComplete, const SCEV *ConstantMax, bool MaxOrZero)
8081 : ConstantMax(ConstantMax), IsComplete(IsComplete), MaxOrZero(MaxOrZero) {
8082 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8083
8084 ExitNotTaken.reserve(ExitCounts.size());
8085 std::transform(
8086 ExitCounts.begin(), ExitCounts.end(), std::back_inserter(ExitNotTaken),
8087 [&](const EdgeExitInfo &EEI) {
8088 BasicBlock *ExitBB = EEI.first;
8089 const ExitLimit &EL = EEI.second;
8090 if (EL.Predicates.empty())
8091 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken, EL.MaxNotTaken,
8092 nullptr);
8093
8094 std::unique_ptr<SCEVUnionPredicate> Predicate(new SCEVUnionPredicate);
8095 for (auto *Pred : EL.Predicates)
8096 Predicate->add(Pred);
8097
8098 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken, EL.MaxNotTaken,
8099 std::move(Predicate));
8100 });
8101 assert((isa<SCEVCouldNotCompute>(ConstantMax) ||(static_cast <bool> ((isa<SCEVCouldNotCompute>(ConstantMax
) || isa<SCEVConstant>(ConstantMax)) && "No point in having a non-constant max backedge taken count!"
) ? void (0) : __assert_fail ("(isa<SCEVCouldNotCompute>(ConstantMax) || isa<SCEVConstant>(ConstantMax)) && \"No point in having a non-constant max backedge taken count!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8103, __extension__
__PRETTY_FUNCTION__))
8102 isa<SCEVConstant>(ConstantMax)) &&(static_cast <bool> ((isa<SCEVCouldNotCompute>(ConstantMax
) || isa<SCEVConstant>(ConstantMax)) && "No point in having a non-constant max backedge taken count!"
) ? void (0) : __assert_fail ("(isa<SCEVCouldNotCompute>(ConstantMax) || isa<SCEVConstant>(ConstantMax)) && \"No point in having a non-constant max backedge taken count!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8103, __extension__
__PRETTY_FUNCTION__))
8103 "No point in having a non-constant max backedge taken count!")(static_cast <bool> ((isa<SCEVCouldNotCompute>(ConstantMax
) || isa<SCEVConstant>(ConstantMax)) && "No point in having a non-constant max backedge taken count!"
) ? void (0) : __assert_fail ("(isa<SCEVCouldNotCompute>(ConstantMax) || isa<SCEVConstant>(ConstantMax)) && \"No point in having a non-constant max backedge taken count!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8103, __extension__
__PRETTY_FUNCTION__))
;
8104}
8105
8106/// Compute the number of times the backedge of the specified loop will execute.
8107ScalarEvolution::BackedgeTakenInfo
8108ScalarEvolution::computeBackedgeTakenCount(const Loop *L,
8109 bool AllowPredicates) {
8110 SmallVector<BasicBlock *, 8> ExitingBlocks;
8111 L->getExitingBlocks(ExitingBlocks);
8112
8113 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8114
8115 SmallVector<EdgeExitInfo, 4> ExitCounts;
8116 bool CouldComputeBECount = true;
8117 BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
8118 const SCEV *MustExitMaxBECount = nullptr;
8119 const SCEV *MayExitMaxBECount = nullptr;
8120 bool MustExitMaxOrZero = false;
8121
8122 // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts
8123 // and compute maxBECount.
8124 // Do a union of all the predicates here.
8125 for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) {
8126 BasicBlock *ExitBB = ExitingBlocks[i];
8127
8128 // We canonicalize untaken exits to br (constant), ignore them so that
8129 // proving an exit untaken doesn't negatively impact our ability to reason
8130 // about the loop as whole.
8131 if (auto *BI = dyn_cast<BranchInst>(ExitBB->getTerminator()))
8132 if (auto *CI = dyn_cast<ConstantInt>(BI->getCondition())) {
8133 bool ExitIfTrue = !L->contains(BI->getSuccessor(0));
8134 if (ExitIfTrue == CI->isZero())
8135 continue;
8136 }
8137
8138 ExitLimit EL = computeExitLimit(L, ExitBB, AllowPredicates);
8139
8140 assert((AllowPredicates || EL.Predicates.empty()) &&(static_cast <bool> ((AllowPredicates || EL.Predicates.
empty()) && "Predicated exit limit when predicates are not allowed!"
) ? void (0) : __assert_fail ("(AllowPredicates || EL.Predicates.empty()) && \"Predicated exit limit when predicates are not allowed!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8141, __extension__
__PRETTY_FUNCTION__))
8141 "Predicated exit limit when predicates are not allowed!")(static_cast <bool> ((AllowPredicates || EL.Predicates.
empty()) && "Predicated exit limit when predicates are not allowed!"
) ? void (0) : __assert_fail ("(AllowPredicates || EL.Predicates.empty()) && \"Predicated exit limit when predicates are not allowed!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8141, __extension__
__PRETTY_FUNCTION__))
;
8142
8143 // 1. For each exit that can be computed, add an entry to ExitCounts.
8144 // CouldComputeBECount is true only if all exits can be computed.
8145 if (EL.ExactNotTaken == getCouldNotCompute())
8146 // We couldn't compute an exact value for this exit, so
8147 // we won't be able to compute an exact value for the loop.
8148 CouldComputeBECount = false;
8149 else
8150 ExitCounts.emplace_back(ExitBB, EL);
8151
8152 // 2. Derive the loop's MaxBECount from each exit's max number of
8153 // non-exiting iterations. Partition the loop exits into two kinds:
8154 // LoopMustExits and LoopMayExits.
8155 //
8156 // If the exit dominates the loop latch, it is a LoopMustExit otherwise it
8157 // is a LoopMayExit. If any computable LoopMustExit is found, then
8158 // MaxBECount is the minimum EL.MaxNotTaken of computable
8159 // LoopMustExits. Otherwise, MaxBECount is conservatively the maximum
8160 // EL.MaxNotTaken, where CouldNotCompute is considered greater than any
8161 // computable EL.MaxNotTaken.
8162 if (EL.MaxNotTaken != getCouldNotCompute() && Latch &&
8163 DT.dominates(ExitBB, Latch)) {
8164 if (!MustExitMaxBECount) {
8165 MustExitMaxBECount = EL.MaxNotTaken;
8166 MustExitMaxOrZero = EL.MaxOrZero;
8167 } else {
8168 MustExitMaxBECount =
8169 getUMinFromMismatchedTypes(MustExitMaxBECount, EL.MaxNotTaken);
8170 }
8171 } else if (MayExitMaxBECount != getCouldNotCompute()) {
8172 if (!MayExitMaxBECount || EL.MaxNotTaken == getCouldNotCompute())
8173 MayExitMaxBECount = EL.MaxNotTaken;
8174 else {
8175 MayExitMaxBECount =
8176 getUMaxFromMismatchedTypes(MayExitMaxBECount, EL.MaxNotTaken);
8177 }
8178 }
8179 }
8180 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
8181 (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute());
8182 // The loop backedge will be taken the maximum or zero times if there's
8183 // a single exit that must be taken the maximum or zero times.
8184 bool MaxOrZero = (MustExitMaxOrZero && ExitingBlocks.size() == 1);
8185
8186 // Remember which SCEVs are used in exit limits for invalidation purposes.
8187 // We only care about non-constant SCEVs here, so we can ignore EL.MaxNotTaken
8188 // and MaxBECount, which must be SCEVConstant.
8189 for (const auto &Pair : ExitCounts)
8190 if (!isa<SCEVConstant>(Pair.second.ExactNotTaken))
8191 BECountUsers[Pair.second.ExactNotTaken].insert({L, AllowPredicates});
8192 return BackedgeTakenInfo(std::move(ExitCounts), CouldComputeBECount,
8193 MaxBECount, MaxOrZero);
8194}
8195
8196ScalarEvolution::ExitLimit
8197ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock,
8198 bool AllowPredicates) {
8199 assert(L->contains(ExitingBlock) && "Exit count for non-loop block?")(static_cast <bool> (L->contains(ExitingBlock) &&
"Exit count for non-loop block?") ? void (0) : __assert_fail
("L->contains(ExitingBlock) && \"Exit count for non-loop block?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8199, __extension__
__PRETTY_FUNCTION__))
;
8200 // If our exiting block does not dominate the latch, then its connection with
8201 // loop's exit limit may be far from trivial.
8202 const BasicBlock *Latch = L->getLoopLatch();
8203 if (!Latch || !DT.dominates(ExitingBlock, Latch))
8204 return getCouldNotCompute();
8205
8206 bool IsOnlyExit = (L->getExitingBlock() != nullptr);
8207 Instruction *Term = ExitingBlock->getTerminator();
8208 if (BranchInst *BI = dyn_cast<BranchInst>(Term)) {
8209 assert(BI->isConditional() && "If unconditional, it can't be in loop!")(static_cast <bool> (BI->isConditional() && "If unconditional, it can't be in loop!"
) ? void (0) : __assert_fail ("BI->isConditional() && \"If unconditional, it can't be in loop!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8209, __extension__
__PRETTY_FUNCTION__))
;
8210 bool ExitIfTrue = !L->contains(BI->getSuccessor(0));
8211 assert(ExitIfTrue == L->contains(BI->getSuccessor(1)) &&(static_cast <bool> (ExitIfTrue == L->contains(BI->
getSuccessor(1)) && "It should have one successor in loop and one exit block!"
) ? void (0) : __assert_fail ("ExitIfTrue == L->contains(BI->getSuccessor(1)) && \"It should have one successor in loop and one exit block!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8212, __extension__
__PRETTY_FUNCTION__))
8212 "It should have one successor in loop and one exit block!")(static_cast <bool> (ExitIfTrue == L->contains(BI->
getSuccessor(1)) && "It should have one successor in loop and one exit block!"
) ? void (0) : __assert_fail ("ExitIfTrue == L->contains(BI->getSuccessor(1)) && \"It should have one successor in loop and one exit block!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8212, __extension__
__PRETTY_FUNCTION__))
;
8213 // Proceed to the next level to examine the exit condition expression.
8214 return computeExitLimitFromCond(
8215 L, BI->getCondition(), ExitIfTrue,
8216 /*ControlsExit=*/IsOnlyExit, AllowPredicates);
8217 }
8218
8219 if (SwitchInst *SI = dyn_cast<SwitchInst>(Term)) {
8220 // For switch, make sure that there is a single exit from the loop.
8221 BasicBlock *Exit = nullptr;
8222 for (auto *SBB : successors(ExitingBlock))
8223 if (!L->contains(SBB)) {
8224 if (Exit) // Multiple exit successors.
8225 return getCouldNotCompute();
8226 Exit = SBB;
8227 }
8228 assert(Exit && "Exiting block must have at least one exit")(static_cast <bool> (Exit && "Exiting block must have at least one exit"
) ? void (0) : __assert_fail ("Exit && \"Exiting block must have at least one exit\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8228, __extension__
__PRETTY_FUNCTION__))
;
8229 return computeExitLimitFromSingleExitSwitch(L, SI, Exit,
8230 /*ControlsExit=*/IsOnlyExit);
8231 }
8232
8233 return getCouldNotCompute();
8234}
8235
8236ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCond(
8237 const Loop *L, Value *ExitCond, bool ExitIfTrue,
8238 bool ControlsExit, bool AllowPredicates) {
8239 ScalarEvolution::ExitLimitCacheTy Cache(L, ExitIfTrue, AllowPredicates);
8240 return computeExitLimitFromCondCached(Cache, L, ExitCond, ExitIfTrue,
8241 ControlsExit, AllowPredicates);
8242}
8243
8244Optional<ScalarEvolution::ExitLimit>
8245ScalarEvolution::ExitLimitCache::find(const Loop *L, Value *ExitCond,
8246 bool ExitIfTrue, bool ControlsExit,
8247 bool AllowPredicates) {
8248 (void)this->L;
8249 (void)this->ExitIfTrue;
8250 (void)this->AllowPredicates;
8251
8252 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&(static_cast <bool> (this->L == L && this->
ExitIfTrue == ExitIfTrue && this->AllowPredicates ==
AllowPredicates && "Variance in assumed invariant key components!"
) ? void (0) : __assert_fail ("this->L == L && this->ExitIfTrue == ExitIfTrue && this->AllowPredicates == AllowPredicates && \"Variance in assumed invariant key components!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8254, __extension__
__PRETTY_FUNCTION__))
8253 this->AllowPredicates == AllowPredicates &&(static_cast <bool> (this->L == L && this->
ExitIfTrue == ExitIfTrue && this->AllowPredicates ==
AllowPredicates && "Variance in assumed invariant key components!"
) ? void (0) : __assert_fail ("this->L == L && this->ExitIfTrue == ExitIfTrue && this->AllowPredicates == AllowPredicates && \"Variance in assumed invariant key components!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8254, __extension__
__PRETTY_FUNCTION__))
8254 "Variance in assumed invariant key components!")(static_cast <bool> (this->L == L && this->
ExitIfTrue == ExitIfTrue && this->AllowPredicates ==
AllowPredicates && "Variance in assumed invariant key components!"
) ? void (0) : __assert_fail ("this->L == L && this->ExitIfTrue == ExitIfTrue && this->AllowPredicates == AllowPredicates && \"Variance in assumed invariant key components!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8254, __extension__
__PRETTY_FUNCTION__))
;
8255 auto Itr = TripCountMap.find({ExitCond, ControlsExit});
8256 if (Itr == TripCountMap.end())
8257 return None;
8258 return Itr->second;
8259}
8260
8261void ScalarEvolution::ExitLimitCache::insert(const Loop *L, Value *ExitCond,
8262 bool ExitIfTrue,
8263 bool ControlsExit,
8264 bool AllowPredicates,
8265 const ExitLimit &EL) {
8266 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&(static_cast <bool> (this->L == L && this->
ExitIfTrue == ExitIfTrue && this->AllowPredicates ==
AllowPredicates && "Variance in assumed invariant key components!"
) ? void (0) : __assert_fail ("this->L == L && this->ExitIfTrue == ExitIfTrue && this->AllowPredicates == AllowPredicates && \"Variance in assumed invariant key components!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8268, __extension__
__PRETTY_FUNCTION__))
8267 this->AllowPredicates == AllowPredicates &&(static_cast <bool> (this->L == L && this->
ExitIfTrue == ExitIfTrue && this->AllowPredicates ==
AllowPredicates && "Variance in assumed invariant key components!"
) ? void (0) : __assert_fail ("this->L == L && this->ExitIfTrue == ExitIfTrue && this->AllowPredicates == AllowPredicates && \"Variance in assumed invariant key components!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8268, __extension__
__PRETTY_FUNCTION__))
8268 "Variance in assumed invariant key components!")(static_cast <bool> (this->L == L && this->
ExitIfTrue == ExitIfTrue && this->AllowPredicates ==
AllowPredicates && "Variance in assumed invariant key components!"
) ? void (0) : __assert_fail ("this->L == L && this->ExitIfTrue == ExitIfTrue && this->AllowPredicates == AllowPredicates && \"Variance in assumed invariant key components!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8268, __extension__
__PRETTY_FUNCTION__))
;
8269
8270 auto InsertResult = TripCountMap.insert({{ExitCond, ControlsExit}, EL});
8271 assert(InsertResult.second && "Expected successful insertion!")(static_cast <bool> (InsertResult.second && "Expected successful insertion!"
) ? void (0) : __assert_fail ("InsertResult.second && \"Expected successful insertion!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8271, __extension__
__PRETTY_FUNCTION__))
;
8272 (void)InsertResult;
8273 (void)ExitIfTrue;
8274}
8275
8276ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondCached(
8277 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
8278 bool ControlsExit, bool AllowPredicates) {
8279
8280 if (auto MaybeEL =
8281 Cache.find(L, ExitCond, ExitIfTrue, ControlsExit, AllowPredicates))
8282 return *MaybeEL;
8283
8284 ExitLimit EL = computeExitLimitFromCondImpl(Cache, L, ExitCond, ExitIfTrue,
8285 ControlsExit, AllowPredicates);
8286 Cache.insert(L, ExitCond, ExitIfTrue, ControlsExit, AllowPredicates, EL);
8287 return EL;
8288}
8289
8290ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl(
8291 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
8292 bool ControlsExit, bool AllowPredicates) {
8293 // Handle BinOp conditions (And, Or).
8294 if (auto LimitFromBinOp = computeExitLimitFromCondFromBinOp(
8295 Cache, L, ExitCond, ExitIfTrue, ControlsExit, AllowPredicates))
8296 return *LimitFromBinOp;
8297
8298 // With an icmp, it may be feasible to compute an exact backedge-taken count.
8299 // Proceed to the next level to examine the icmp.
8300 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond)) {
8301 ExitLimit EL =
8302 computeExitLimitFromICmp(L, ExitCondICmp, ExitIfTrue, ControlsExit);
8303 if (EL.hasFullInfo() || !AllowPredicates)
8304 return EL;
8305
8306 // Try again, but use SCEV predicates this time.
8307 return computeExitLimitFromICmp(L, ExitCondICmp, ExitIfTrue, ControlsExit,
8308 /*AllowPredicates=*/true);
8309 }
8310
8311 // Check for a constant condition. These are normally stripped out by
8312 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
8313 // preserve the CFG and is temporarily leaving constant conditions
8314 // in place.
8315 if (ConstantInt *CI = dyn_cast<ConstantInt>(ExitCond)) {
8316 if (ExitIfTrue == !CI->getZExtValue())
8317 // The backedge is always taken.
8318 return getCouldNotCompute();
8319 else
8320 // The backedge is never taken.
8321 return getZero(CI->getType());
8322 }
8323
8324 // If we're exiting based on the overflow flag of an x.with.overflow intrinsic
8325 // with a constant step, we can form an equivalent icmp predicate and figure
8326 // out how many iterations will be taken before we exit.
8327 const WithOverflowInst *WO;
8328 const APInt *C;
8329 if (match(ExitCond, m_ExtractValue<1>(m_WithOverflowInst(WO))) &&
8330 match(WO->getRHS(), m_APInt(C))) {
8331 ConstantRange NWR =
8332 ConstantRange::makeExactNoWrapRegion(WO->getBinaryOp(), *C,
8333 WO->getNoWrapKind());
8334 CmpInst::Predicate Pred;
8335 APInt NewRHSC, Offset;
8336 NWR.getEquivalentICmp(Pred, NewRHSC, Offset);
8337 if (!ExitIfTrue)
8338 Pred = ICmpInst::getInversePredicate(Pred);
8339 auto *LHS = getSCEV(WO->getLHS());
8340 if (Offset != 0)
8341 LHS = getAddExpr(LHS, getConstant(Offset));
8342 auto EL = computeExitLimitFromICmp(L, Pred, LHS, getConstant(NewRHSC),
8343 ControlsExit, AllowPredicates);
8344 if (EL.hasAnyInfo()) return EL;
8345 }
8346
8347 // If it's not an integer or pointer comparison then compute it the hard way.
8348 return computeExitCountExhaustively(L, ExitCond, ExitIfTrue);
8349}
8350
8351Optional<ScalarEvolution::ExitLimit>
8352ScalarEvolution::computeExitLimitFromCondFromBinOp(
8353 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
8354 bool ControlsExit, bool AllowPredicates) {
8355 // Check if the controlling expression for this loop is an And or Or.
8356 Value *Op0, *Op1;
8357 bool IsAnd = false;
8358 if (match(ExitCond, m_LogicalAnd(m_Value(Op0), m_Value(Op1))))
8359 IsAnd = true;
8360 else if (match(ExitCond, m_LogicalOr(m_Value(Op0), m_Value(Op1))))
8361 IsAnd = false;
8362 else
8363 return None;
8364
8365 // EitherMayExit is true in these two cases:
8366 // br (and Op0 Op1), loop, exit
8367 // br (or Op0 Op1), exit, loop
8368 bool EitherMayExit = IsAnd ^ ExitIfTrue;
8369 ExitLimit EL0 = computeExitLimitFromCondCached(Cache, L, Op0, ExitIfTrue,
8370 ControlsExit && !EitherMayExit,
8371 AllowPredicates);
8372 ExitLimit EL1 = computeExitLimitFromCondCached(Cache, L, Op1, ExitIfTrue,
8373 ControlsExit && !EitherMayExit,
8374 AllowPredicates);
8375
8376 // Be robust against unsimplified IR for the form "op i1 X, NeutralElement"
8377 const Constant *NeutralElement = ConstantInt::get(ExitCond->getType(), IsAnd);
8378 if (isa<ConstantInt>(Op1))
8379 return Op1 == NeutralElement ? EL0 : EL1;
8380 if (isa<ConstantInt>(Op0))
8381 return Op0 == NeutralElement ? EL1 : EL0;
8382
8383 const SCEV *BECount = getCouldNotCompute();
8384 const SCEV *MaxBECount = getCouldNotCompute();
8385 if (EitherMayExit) {
8386 // Both conditions must be same for the loop to continue executing.
8387 // Choose the less conservative count.
8388 if (EL0.ExactNotTaken != getCouldNotCompute() &&
8389 EL1.ExactNotTaken != getCouldNotCompute()) {
8390 BECount = getUMinFromMismatchedTypes(
8391 EL0.ExactNotTaken, EL1.ExactNotTaken,
8392 /*Sequential=*/!isa<BinaryOperator>(ExitCond));
8393
8394 // If EL0.ExactNotTaken was zero and ExitCond was a short-circuit form,
8395 // it should have been simplified to zero (see the condition (3) above)
8396 assert(!isa<BinaryOperator>(ExitCond) || !EL0.ExactNotTaken->isZero() ||(static_cast <bool> (!isa<BinaryOperator>(ExitCond
) || !EL0.ExactNotTaken->isZero() || BECount->isZero())
? void (0) : __assert_fail ("!isa<BinaryOperator>(ExitCond) || !EL0.ExactNotTaken->isZero() || BECount->isZero()"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8397, __extension__
__PRETTY_FUNCTION__))
8397 BECount->isZero())(static_cast <bool> (!isa<BinaryOperator>(ExitCond
) || !EL0.ExactNotTaken->isZero() || BECount->isZero())
? void (0) : __assert_fail ("!isa<BinaryOperator>(ExitCond) || !EL0.ExactNotTaken->isZero() || BECount->isZero()"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8397, __extension__
__PRETTY_FUNCTION__))
;
8398 }
8399 if (EL0.MaxNotTaken == getCouldNotCompute())
8400 MaxBECount = EL1.MaxNotTaken;
8401 else if (EL1.MaxNotTaken == getCouldNotCompute())
8402 MaxBECount = EL0.MaxNotTaken;
8403 else
8404 MaxBECount = getUMinFromMismatchedTypes(EL0.MaxNotTaken, EL1.MaxNotTaken);
8405 } else {
8406 // Both conditions must be same at the same time for the loop to exit.
8407 // For now, be conservative.
8408 if (EL0.ExactNotTaken == EL1.ExactNotTaken)
8409 BECount = EL0.ExactNotTaken;
8410 }
8411
8412 // There are cases (e.g. PR26207) where computeExitLimitFromCond is able
8413 // to be more aggressive when computing BECount than when computing
8414 // MaxBECount. In these cases it is possible for EL0.ExactNotTaken and
8415 // EL1.ExactNotTaken to match, but for EL0.MaxNotTaken and EL1.MaxNotTaken
8416 // to not.
8417 if (isa<SCEVCouldNotCompute>(MaxBECount) &&
8418 !isa<SCEVCouldNotCompute>(BECount))
8419 MaxBECount = getConstant(getUnsignedRangeMax(BECount));
8420
8421 return ExitLimit(BECount, MaxBECount, false,
8422 { &EL0.Predicates, &EL1.Predicates });
8423}
8424
8425ScalarEvolution::ExitLimit
8426ScalarEvolution::computeExitLimitFromICmp(const Loop *L,
8427 ICmpInst *ExitCond,
8428 bool ExitIfTrue,
8429 bool ControlsExit,
8430 bool AllowPredicates) {
8431 // If the condition was exit on true, convert the condition to exit on false
8432 ICmpInst::Predicate Pred;
8433 if (!ExitIfTrue)
8434 Pred = ExitCond->getPredicate();
8435 else
8436 Pred = ExitCond->getInversePredicate();
8437 const ICmpInst::Predicate OriginalPred = Pred;
8438
8439 const SCEV *LHS = getSCEV(ExitCond->getOperand(0));
8440 const SCEV *RHS = getSCEV(ExitCond->getOperand(1));
8441
8442 ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, ControlsExit,
8443 AllowPredicates);
8444 if (EL.hasAnyInfo()) return EL;
8445
8446 auto *ExhaustiveCount =
8447 computeExitCountExhaustively(L, ExitCond, ExitIfTrue);
8448
8449 if (!isa<SCEVCouldNotCompute>(ExhaustiveCount))
8450 return ExhaustiveCount;
8451
8452 return computeShiftCompareExitLimit(ExitCond->getOperand(0),
8453 ExitCond->getOperand(1), L, OriginalPred);
8454}
8455ScalarEvolution::ExitLimit
8456ScalarEvolution::computeExitLimitFromICmp(const Loop *L,
8457 ICmpInst::Predicate Pred,
8458 const SCEV *LHS, const SCEV *RHS,
8459 bool ControlsExit,
8460 bool AllowPredicates) {
8461
8462 // Try to evaluate any dependencies out of the loop.
8463 LHS = getSCEVAtScope(LHS, L);
8464 RHS = getSCEVAtScope(RHS, L);
8465
8466 // At this point, we would like to compute how many iterations of the
8467 // loop the predicate will return true for these inputs.
8468 if (isLoopInvariant(LHS, L) && !isLoopInvariant(RHS, L)) {
8469 // If there is a loop-invariant, force it into the RHS.
8470 std::swap(LHS, RHS);
8471 Pred = ICmpInst::getSwappedPredicate(Pred);
8472 }
8473
8474 // Simplify the operands before analyzing them.
8475 (void)SimplifyICmpOperands(Pred, LHS, RHS);
8476
8477 // If we have a comparison of a chrec against a constant, try to use value
8478 // ranges to answer this query.
8479 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS))
8480 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS))
8481 if (AddRec->getLoop() == L) {
8482 // Form the constant range.
8483 ConstantRange CompRange =
8484 ConstantRange::makeExactICmpRegion(Pred, RHSC->getAPInt());
8485
8486 const SCEV *Ret = AddRec->getNumIterationsInRange(CompRange, *this);
8487 if (!isa<SCEVCouldNotCompute>(Ret)) return Ret;
8488 }
8489
8490 // If this loop must exit based on this condition (or execute undefined
8491 // behaviour), and we can prove the test sequence produced must repeat
8492 // the same values on self-wrap of the IV, then we can infer that IV
8493 // doesn't self wrap because if it did, we'd have an infinite (undefined)
8494 // loop.
8495 if (ControlsExit && isLoopInvariant(RHS, L) && loopHasNoAbnormalExits(L) &&
8496 loopIsFiniteByAssumption(L)) {
8497
8498 // TODO: We can peel off any functions which are invertible *in L*. Loop
8499 // invariant terms are effectively constants for our purposes here.
8500 auto *InnerLHS = LHS;
8501 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(LHS))
8502 InnerLHS = ZExt->getOperand();
8503 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(InnerLHS)) {
8504 auto *StrideC = dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this));
8505 if (!AR->hasNoSelfWrap() && AR->getLoop() == L && AR->isAffine() &&
8506 StrideC && StrideC->getAPInt().isPowerOf2()) {
8507 auto Flags = AR->getNoWrapFlags();
8508 Flags = setFlags(Flags, SCEV::FlagNW);
8509 SmallVector<const SCEV*> Operands{AR->operands()};
8510 Flags = StrengthenNoWrapFlags(this, scAddRecExpr, Operands, Flags);
8511 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), Flags);
8512 }
8513 }
8514 }
8515
8516 switch (Pred) {
8517 case ICmpInst::ICMP_NE: { // while (X != Y)
8518 // Convert to: while (X-Y != 0)
8519 if (LHS->getType()->isPointerTy()) {
8520 LHS = getLosslessPtrToIntExpr(LHS);
8521 if (isa<SCEVCouldNotCompute>(LHS))
8522 return LHS;
8523 }
8524 if (RHS->getType()->isPointerTy()) {
8525 RHS = getLosslessPtrToIntExpr(RHS);
8526 if (isa<SCEVCouldNotCompute>(RHS))
8527 return RHS;
8528 }
8529 ExitLimit EL = howFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit,
8530 AllowPredicates);
8531 if (EL.hasAnyInfo()) return EL;
8532 break;
8533 }
8534 case ICmpInst::ICMP_EQ: { // while (X == Y)
8535 // Convert to: while (X-Y == 0)
8536 if (LHS->getType()->isPointerTy()) {
8537 LHS = getLosslessPtrToIntExpr(LHS);
8538 if (isa<SCEVCouldNotCompute>(LHS))
8539 return LHS;
8540 }
8541 if (RHS->getType()->isPointerTy()) {
8542 RHS = getLosslessPtrToIntExpr(RHS);
8543 if (isa<SCEVCouldNotCompute>(RHS))
8544 return RHS;
8545 }
8546 ExitLimit EL = howFarToNonZero(getMinusSCEV(LHS, RHS), L);
8547 if (EL.hasAnyInfo()) return EL;
8548 break;
8549 }
8550 case ICmpInst::ICMP_SLT:
8551 case ICmpInst::ICMP_ULT: { // while (X < Y)
8552 bool IsSigned = Pred == ICmpInst::ICMP_SLT;
8553 ExitLimit EL = howManyLessThans(LHS, RHS, L, IsSigned, ControlsExit,
8554 AllowPredicates);
8555 if (EL.hasAnyInfo()) return EL;
8556 break;
8557 }
8558 case ICmpInst::ICMP_SGT:
8559 case ICmpInst::ICMP_UGT: { // while (X > Y)
8560 bool IsSigned = Pred == ICmpInst::ICMP_SGT;
8561 ExitLimit EL =
8562 howManyGreaterThans(LHS, RHS, L, IsSigned, ControlsExit,
8563 AllowPredicates);
8564 if (EL.hasAnyInfo()) return EL;
8565 break;
8566 }
8567 default:
8568 break;
8569 }
8570
8571 return getCouldNotCompute();
8572}
8573
8574ScalarEvolution::ExitLimit
8575ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L,
8576 SwitchInst *Switch,
8577 BasicBlock *ExitingBlock,
8578 bool ControlsExit) {
8579 assert(!L->contains(ExitingBlock) && "Not an exiting block!")(static_cast <bool> (!L->contains(ExitingBlock) &&
"Not an exiting block!") ? void (0) : __assert_fail ("!L->contains(ExitingBlock) && \"Not an exiting block!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8579, __extension__
__PRETTY_FUNCTION__))
;
8580
8581 // Give up if the exit is the default dest of a switch.
8582 if (Switch->getDefaultDest() == ExitingBlock)
8583 return getCouldNotCompute();
8584
8585 assert(L->contains(Switch->getDefaultDest()) &&(static_cast <bool> (L->contains(Switch->getDefaultDest
()) && "Default case must not exit the loop!") ? void
(0) : __assert_fail ("L->contains(Switch->getDefaultDest()) && \"Default case must not exit the loop!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8586, __extension__
__PRETTY_FUNCTION__))
8586 "Default case must not exit the loop!")(static_cast <bool> (L->contains(Switch->getDefaultDest
()) && "Default case must not exit the loop!") ? void
(0) : __assert_fail ("L->contains(Switch->getDefaultDest()) && \"Default case must not exit the loop!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8586, __extension__
__PRETTY_FUNCTION__))
;
8587 const SCEV *LHS = getSCEVAtScope(Switch->getCondition(), L);
8588 const SCEV *RHS = getConstant(Switch->findCaseDest(ExitingBlock));
8589
8590 // while (X != Y) --> while (X-Y != 0)
8591 ExitLimit EL = howFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit);
8592 if (EL.hasAnyInfo())
8593 return EL;
8594
8595 return getCouldNotCompute();
8596}
8597
8598static ConstantInt *
8599EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C,
8600 ScalarEvolution &SE) {
8601 const SCEV *InVal = SE.getConstant(C);
8602 const SCEV *Val = AddRec->evaluateAtIteration(InVal, SE);
8603 assert(isa<SCEVConstant>(Val) &&(static_cast <bool> (isa<SCEVConstant>(Val) &&
"Evaluation of SCEV at constant didn't fold correctly?") ? void
(0) : __assert_fail ("isa<SCEVConstant>(Val) && \"Evaluation of SCEV at constant didn't fold correctly?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8604, __extension__
__PRETTY_FUNCTION__))
8604 "Evaluation of SCEV at constant didn't fold correctly?")(static_cast <bool> (isa<SCEVConstant>(Val) &&
"Evaluation of SCEV at constant didn't fold correctly?") ? void
(0) : __assert_fail ("isa<SCEVConstant>(Val) && \"Evaluation of SCEV at constant didn't fold correctly?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8604, __extension__
__PRETTY_FUNCTION__))
;
8605 return cast<SCEVConstant>(Val)->getValue();
8606}
8607
8608ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit(
8609 Value *LHS, Value *RHSV, const Loop *L, ICmpInst::Predicate Pred) {
8610 ConstantInt *RHS = dyn_cast<ConstantInt>(RHSV);
8611 if (!RHS)
8612 return getCouldNotCompute();
8613
8614 const BasicBlock *Latch = L->getLoopLatch();
8615 if (!Latch)
8616 return getCouldNotCompute();
8617
8618 const BasicBlock *Predecessor = L->getLoopPredecessor();
8619 if (!Predecessor)
8620 return getCouldNotCompute();
8621
8622 // Return true if V is of the form "LHS `shift_op` <positive constant>".
8623 // Return LHS in OutLHS and shift_opt in OutOpCode.
8624 auto MatchPositiveShift =
8625 [](Value *V, Value *&OutLHS, Instruction::BinaryOps &OutOpCode) {
8626
8627 using namespace PatternMatch;
8628
8629 ConstantInt *ShiftAmt;
8630 if (match(V, m_LShr(m_Value(OutLHS), m_ConstantInt(ShiftAmt))))
8631 OutOpCode = Instruction::LShr;
8632 else if (match(V, m_AShr(m_Value(OutLHS), m_ConstantInt(ShiftAmt))))
8633 OutOpCode = Instruction::AShr;
8634 else if (match(V, m_Shl(m_Value(OutLHS), m_ConstantInt(ShiftAmt))))
8635 OutOpCode = Instruction::Shl;
8636 else
8637 return false;
8638
8639 return ShiftAmt->getValue().isStrictlyPositive();
8640 };
8641
8642 // Recognize a "shift recurrence" either of the form %iv or of %iv.shifted in
8643 //
8644 // loop:
8645 // %iv = phi i32 [ %iv.shifted, %loop ], [ %val, %preheader ]
8646 // %iv.shifted = lshr i32 %iv, <positive constant>
8647 //
8648 // Return true on a successful match. Return the corresponding PHI node (%iv
8649 // above) in PNOut and the opcode of the shift operation in OpCodeOut.
8650 auto MatchShiftRecurrence =
8651 [&](Value *V, PHINode *&PNOut, Instruction::BinaryOps &OpCodeOut) {
8652 Optional<Instruction::BinaryOps> PostShiftOpCode;
8653
8654 {
8655 Instruction::BinaryOps OpC;
8656 Value *V;
8657
8658 // If we encounter a shift instruction, "peel off" the shift operation,
8659 // and remember that we did so. Later when we inspect %iv's backedge
8660 // value, we will make sure that the backedge value uses the same
8661 // operation.
8662 //
8663 // Note: the peeled shift operation does not have to be the same
8664 // instruction as the one feeding into the PHI's backedge value. We only
8665 // really care about it being the same *kind* of shift instruction --
8666 // that's all that is required for our later inferences to hold.
8667 if (MatchPositiveShift(LHS, V, OpC)) {
8668 PostShiftOpCode = OpC;
8669 LHS = V;
8670 }
8671 }
8672
8673 PNOut = dyn_cast<PHINode>(LHS);
8674 if (!PNOut || PNOut->getParent() != L->getHeader())
8675 return false;
8676
8677 Value *BEValue = PNOut->getIncomingValueForBlock(Latch);
8678 Value *OpLHS;
8679
8680 return
8681 // The backedge value for the PHI node must be a shift by a positive
8682 // amount
8683 MatchPositiveShift(BEValue, OpLHS, OpCodeOut) &&
8684
8685 // of the PHI node itself
8686 OpLHS == PNOut &&
8687
8688 // and the kind of shift should be match the kind of shift we peeled
8689 // off, if any.
8690 (!PostShiftOpCode.hasValue() || *PostShiftOpCode == OpCodeOut);
8691 };
8692
8693 PHINode *PN;
8694 Instruction::BinaryOps OpCode;
8695 if (!MatchShiftRecurrence(LHS, PN, OpCode))
8696 return getCouldNotCompute();
8697
8698 const DataLayout &DL = getDataLayout();
8699
8700 // The key rationale for this optimization is that for some kinds of shift
8701 // recurrences, the value of the recurrence "stabilizes" to either 0 or -1
8702 // within a finite number of iterations. If the condition guarding the
8703 // backedge (in the sense that the backedge is taken if the condition is true)
8704 // is false for the value the shift recurrence stabilizes to, then we know
8705 // that the backedge is taken only a finite number of times.
8706
8707 ConstantInt *StableValue = nullptr;
8708 switch (OpCode) {
8709 default:
8710 llvm_unreachable("Impossible case!")::llvm::llvm_unreachable_internal("Impossible case!", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 8710)
;
8711
8712 case Instruction::AShr: {
8713 // {K,ashr,<positive-constant>} stabilizes to signum(K) in at most
8714 // bitwidth(K) iterations.
8715 Value *FirstValue = PN->getIncomingValueForBlock(Predecessor);
8716 KnownBits Known = computeKnownBits(FirstValue, DL, 0, &AC,
8717 Predecessor->getTerminator(), &DT);
8718 auto *Ty = cast<IntegerType>(RHS->getType());
8719 if (Known.isNonNegative())
8720 StableValue = ConstantInt::get(Ty, 0);
8721 else if (Known.isNegative())
8722 StableValue = ConstantInt::get(Ty, -1, true);
8723 else
8724 return getCouldNotCompute();
8725
8726 break;
8727 }
8728 case Instruction::LShr:
8729 case Instruction::Shl:
8730 // Both {K,lshr,<positive-constant>} and {K,shl,<positive-constant>}
8731 // stabilize to 0 in at most bitwidth(K) iterations.
8732 StableValue = ConstantInt::get(cast<IntegerType>(RHS->getType()), 0);
8733 break;
8734 }
8735
8736 auto *Result =
8737 ConstantFoldCompareInstOperands(Pred, StableValue, RHS, DL, &TLI);
8738 assert(Result->getType()->isIntegerTy(1) &&(static_cast <bool> (Result->getType()->isIntegerTy
(1) && "Otherwise cannot be an operand to a branch instruction"
) ? void (0) : __assert_fail ("Result->getType()->isIntegerTy(1) && \"Otherwise cannot be an operand to a branch instruction\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8739, __extension__
__PRETTY_FUNCTION__))
8739 "Otherwise cannot be an operand to a branch instruction")(static_cast <bool> (Result->getType()->isIntegerTy
(1) && "Otherwise cannot be an operand to a branch instruction"
) ? void (0) : __assert_fail ("Result->getType()->isIntegerTy(1) && \"Otherwise cannot be an operand to a branch instruction\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8739, __extension__
__PRETTY_FUNCTION__))
;
8740
8741 if (Result->isZeroValue()) {
8742 unsigned BitWidth = getTypeSizeInBits(RHS->getType());
8743 const SCEV *UpperBound =
8744 getConstant(getEffectiveSCEVType(RHS->getType()), BitWidth);
8745 return ExitLimit(getCouldNotCompute(), UpperBound, false);
8746 }
8747
8748 return getCouldNotCompute();
8749}
8750
8751/// Return true if we can constant fold an instruction of the specified type,
8752/// assuming that all operands were constants.
8753static bool CanConstantFold(const Instruction *I) {
8754 if (isa<BinaryOperator>(I) || isa<CmpInst>(I) ||
8755 isa<SelectInst>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
8756 isa<LoadInst>(I) || isa<ExtractValueInst>(I))
8757 return true;
8758
8759 if (const CallInst *CI = dyn_cast<CallInst>(I))
8760 if (const Function *F = CI->getCalledFunction())
8761 return canConstantFoldCallTo(CI, F);
8762 return false;
8763}
8764
8765/// Determine whether this instruction can constant evolve within this loop
8766/// assuming its operands can all constant evolve.
8767static bool canConstantEvolve(Instruction *I, const Loop *L) {
8768 // An instruction outside of the loop can't be derived from a loop PHI.
8769 if (!L->contains(I)) return false;
8770
8771 if (isa<PHINode>(I)) {
8772 // We don't currently keep track of the control flow needed to evaluate
8773 // PHIs, so we cannot handle PHIs inside of loops.
8774 return L->getHeader() == I->getParent();
8775 }
8776
8777 // If we won't be able to constant fold this expression even if the operands
8778 // are constants, bail early.
8779 return CanConstantFold(I);
8780}
8781
8782/// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
8783/// recursing through each instruction operand until reaching a loop header phi.
8784static PHINode *
8785getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L,
8786 DenseMap<Instruction *, PHINode *> &PHIMap,
8787 unsigned Depth) {
8788 if (Depth > MaxConstantEvolvingDepth)
8789 return nullptr;
8790
8791 // Otherwise, we can evaluate this instruction if all of its operands are
8792 // constant or derived from a PHI node themselves.
8793 PHINode *PHI = nullptr;
8794 for (Value *Op : UseInst->operands()) {
8795 if (isa<Constant>(Op)) continue;
8796
8797 Instruction *OpInst = dyn_cast<Instruction>(Op);
8798 if (!OpInst || !canConstantEvolve(OpInst, L)) return nullptr;
8799
8800 PHINode *P = dyn_cast<PHINode>(OpInst);
8801 if (!P)
8802 // If this operand is already visited, reuse the prior result.
8803 // We may have P != PHI if this is the deepest point at which the
8804 // inconsistent paths meet.
8805 P = PHIMap.lookup(OpInst);
8806 if (!P) {
8807 // Recurse and memoize the results, whether a phi is found or not.
8808 // This recursive call invalidates pointers into PHIMap.
8809 P = getConstantEvolvingPHIOperands(OpInst, L, PHIMap, Depth + 1);
8810 PHIMap[OpInst] = P;
8811 }
8812 if (!P)
8813 return nullptr; // Not evolving from PHI
8814 if (PHI && PHI != P)
8815 return nullptr; // Evolving from multiple different PHIs.
8816 PHI = P;
8817 }
8818 // This is a expression evolving from a constant PHI!
8819 return PHI;
8820}
8821
8822/// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
8823/// in the loop that V is derived from. We allow arbitrary operations along the
8824/// way, but the operands of an operation must either be constants or a value
8825/// derived from a constant PHI. If this expression does not fit with these
8826/// constraints, return null.
8827static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) {
8828 Instruction *I = dyn_cast<Instruction>(V);
8829 if (!I || !canConstantEvolve(I, L)) return nullptr;
8830
8831 if (PHINode *PN = dyn_cast<PHINode>(I))
8832 return PN;
8833
8834 // Record non-constant instructions contained by the loop.
8835 DenseMap<Instruction *, PHINode *> PHIMap;
8836 return getConstantEvolvingPHIOperands(I, L, PHIMap, 0);
8837}
8838
8839/// EvaluateExpression - Given an expression that passes the
8840/// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
8841/// in the loop has the value PHIVal. If we can't fold this expression for some
8842/// reason, return null.
8843static Constant *EvaluateExpression(Value *V, const Loop *L,
8844 DenseMap<Instruction *, Constant *> &Vals,
8845 const DataLayout &DL,
8846 const TargetLibraryInfo *TLI) {
8847 // Convenient constant check, but redundant for recursive calls.
8848 if (Constant *C = dyn_cast<Constant>(V)) return C;
8849 Instruction *I = dyn_cast<Instruction>(V);
8850 if (!I) return nullptr;
8851
8852 if (Constant *C = Vals.lookup(I)) return C;
8853
8854 // An instruction inside the loop depends on a value outside the loop that we
8855 // weren't given a mapping for, or a value such as a call inside the loop.
8856 if (!canConstantEvolve(I, L)) return nullptr;
8857
8858 // An unmapped PHI can be due to a branch or another loop inside this loop,
8859 // or due to this not being the initial iteration through a loop where we
8860 // couldn't compute the evolution of this particular PHI last time.
8861 if (isa<PHINode>(I)) return nullptr;
8862
8863 std::vector<Constant*> Operands(I->getNumOperands());
8864
8865 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
8866 Instruction *Operand = dyn_cast<Instruction>(I->getOperand(i));
8867 if (!Operand) {
8868 Operands[i] = dyn_cast<Constant>(I->getOperand(i));
8869 if (!Operands[i]) return nullptr;
8870 continue;
8871 }
8872 Constant *C = EvaluateExpression(Operand, L, Vals, DL, TLI);
8873 Vals[Operand] = C;
8874 if (!C) return nullptr;
8875 Operands[i] = C;
8876 }
8877
8878 if (CmpInst *CI = dyn_cast<CmpInst>(I))
8879 return ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0],
8880 Operands[1], DL, TLI);
8881 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
8882 if (!LI->isVolatile())
8883 return ConstantFoldLoadFromConstPtr(Operands[0], LI->getType(), DL);
8884 }
8885 return ConstantFoldInstOperands(I, Operands, DL, TLI);
8886}
8887
8888
8889// If every incoming value to PN except the one for BB is a specific Constant,
8890// return that, else return nullptr.
8891static Constant *getOtherIncomingValue(PHINode *PN, BasicBlock *BB) {
8892 Constant *IncomingVal = nullptr;
8893
8894 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
8895 if (PN->getIncomingBlock(i) == BB)
8896 continue;
8897
8898 auto *CurrentVal = dyn_cast<Constant>(PN->getIncomingValue(i));
8899 if (!CurrentVal)
8900 return nullptr;
8901
8902 if (IncomingVal != CurrentVal) {
8903 if (IncomingVal)
8904 return nullptr;
8905 IncomingVal = CurrentVal;
8906 }
8907 }
8908
8909 return IncomingVal;
8910}
8911
8912/// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
8913/// in the header of its containing loop, we know the loop executes a
8914/// constant number of times, and the PHI node is just a recurrence
8915/// involving constants, fold it.
8916Constant *
8917ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
8918 const APInt &BEs,
8919 const Loop *L) {
8920 auto I = ConstantEvolutionLoopExitValue.find(PN);
8921 if (I != ConstantEvolutionLoopExitValue.end())
8922 return I->second;
8923
8924 if (BEs.ugt(MaxBruteForceIterations))
8925 return ConstantEvolutionLoopExitValue[PN] = nullptr; // Not going to evaluate it.
8926
8927 Constant *&RetVal = ConstantEvolutionLoopExitValue[PN];
8928
8929 DenseMap<Instruction *, Constant *> CurrentIterVals;
8930 BasicBlock *Header = L->getHeader();
8931 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!")(static_cast <bool> (PN->getParent() == Header &&
"Can't evaluate PHI not in loop header!") ? void (0) : __assert_fail
("PN->getParent() == Header && \"Can't evaluate PHI not in loop header!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8931, __extension__
__PRETTY_FUNCTION__))
;
8932
8933 BasicBlock *Latch = L->getLoopLatch();
8934 if (!Latch)
8935 return nullptr;
8936
8937 for (PHINode &PHI : Header->phis()) {
8938 if (auto *StartCST = getOtherIncomingValue(&PHI, Latch))
8939 CurrentIterVals[&PHI] = StartCST;
8940 }
8941 if (!CurrentIterVals.count(PN))
8942 return RetVal = nullptr;
8943
8944 Value *BEValue = PN->getIncomingValueForBlock(Latch);
8945
8946 // Execute the loop symbolically to determine the exit value.
8947 assert(BEs.getActiveBits() < CHAR_BIT * sizeof(unsigned) &&(static_cast <bool> (BEs.getActiveBits() < 8 * sizeof
(unsigned) && "BEs is <= MaxBruteForceIterations which is an 'unsigned'!"
) ? void (0) : __assert_fail ("BEs.getActiveBits() < CHAR_BIT * sizeof(unsigned) && \"BEs is <= MaxBruteForceIterations which is an 'unsigned'!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8948, __extension__
__PRETTY_FUNCTION__))
8948 "BEs is <= MaxBruteForceIterations which is an 'unsigned'!")(static_cast <bool> (BEs.getActiveBits() < 8 * sizeof
(unsigned) && "BEs is <= MaxBruteForceIterations which is an 'unsigned'!"
) ? void (0) : __assert_fail ("BEs.getActiveBits() < CHAR_BIT * sizeof(unsigned) && \"BEs is <= MaxBruteForceIterations which is an 'unsigned'!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 8948, __extension__
__PRETTY_FUNCTION__))
;
8949
8950 unsigned NumIterations = BEs.getZExtValue(); // must be in range
8951 unsigned IterationNum = 0;
8952 const DataLayout &DL = getDataLayout();
8953 for (; ; ++IterationNum) {
8954 if (IterationNum == NumIterations)
8955 return RetVal = CurrentIterVals[PN]; // Got exit value!
8956
8957 // Compute the value of the PHIs for the next iteration.
8958 // EvaluateExpression adds non-phi values to the CurrentIterVals map.
8959 DenseMap<Instruction *, Constant *> NextIterVals;
8960 Constant *NextPHI =
8961 EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI);
8962 if (!NextPHI)
8963 return nullptr; // Couldn't evaluate!
8964 NextIterVals[PN] = NextPHI;
8965
8966 bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
8967
8968 // Also evaluate the other PHI nodes. However, we don't get to stop if we
8969 // cease to be able to evaluate one of them or if they stop evolving,
8970 // because that doesn't necessarily prevent us from computing PN.
8971 SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute;
8972 for (const auto &I : CurrentIterVals) {
8973 PHINode *PHI = dyn_cast<PHINode>(I.first);
8974 if (!PHI || PHI == PN || PHI->getParent() != Header) continue;
8975 PHIsToCompute.emplace_back(PHI, I.second);
8976 }
8977 // We use two distinct loops because EvaluateExpression may invalidate any
8978 // iterators into CurrentIterVals.
8979 for (const auto &I : PHIsToCompute) {
8980 PHINode *PHI = I.first;
8981 Constant *&NextPHI = NextIterVals[PHI];
8982 if (!NextPHI) { // Not already computed.
8983 Value *BEValue = PHI->getIncomingValueForBlock(Latch);
8984 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI);
8985 }
8986 if (NextPHI != I.second)
8987 StoppedEvolving = false;
8988 }
8989
8990 // If all entries in CurrentIterVals == NextIterVals then we can stop
8991 // iterating, the loop can't continue to change.
8992 if (StoppedEvolving)
8993 return RetVal = CurrentIterVals[PN];
8994
8995 CurrentIterVals.swap(NextIterVals);
8996 }
8997}
8998
8999const SCEV *ScalarEvolution::computeExitCountExhaustively(const Loop *L,
9000 Value *Cond,
9001 bool ExitWhen) {
9002 PHINode *PN = getConstantEvolvingPHI(Cond, L);
9003 if (!PN) return getCouldNotCompute();
9004
9005 // If the loop is canonicalized, the PHI will have exactly two entries.
9006 // That's the only form we support here.
9007 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
9008
9009 DenseMap<Instruction *, Constant *> CurrentIterVals;
9010 BasicBlock *Header = L->getHeader();
9011 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!")(static_cast <bool> (PN->getParent() == Header &&
"Can't evaluate PHI not in loop header!") ? void (0) : __assert_fail
("PN->getParent() == Header && \"Can't evaluate PHI not in loop header!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 9011, __extension__
__PRETTY_FUNCTION__))
;
9012
9013 BasicBlock *Latch = L->getLoopLatch();
9014 assert(Latch && "Should follow from NumIncomingValues == 2!")(static_cast <bool> (Latch && "Should follow from NumIncomingValues == 2!"
) ? void (0) : __assert_fail ("Latch && \"Should follow from NumIncomingValues == 2!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 9014, __extension__
__PRETTY_FUNCTION__))
;
9015
9016 for (PHINode &PHI : Header->phis()) {
9017 if (auto *StartCST = getOtherIncomingValue(&PHI, Latch))
9018 CurrentIterVals[&PHI] = StartCST;
9019 }
9020 if (!CurrentIterVals.count(PN))
9021 return getCouldNotCompute();
9022
9023 // Okay, we find a PHI node that defines the trip count of this loop. Execute
9024 // the loop symbolically to determine when the condition gets a value of
9025 // "ExitWhen".
9026 unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis.
9027 const DataLayout &DL = getDataLayout();
9028 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
9029 auto *CondVal = dyn_cast_or_null<ConstantInt>(
9030 EvaluateExpression(Cond, L, CurrentIterVals, DL, &TLI));
9031
9032 // Couldn't symbolically evaluate.
9033 if (!CondVal) return getCouldNotCompute();
9034
9035 if (CondVal->getValue() == uint64_t(ExitWhen)) {
9036 ++NumBruteForceTripCountsComputed;
9037 return getConstant(Type::getInt32Ty(getContext()), IterationNum);
9038 }
9039
9040 // Update all the PHI nodes for the next iteration.
9041 DenseMap<Instruction *, Constant *> NextIterVals;
9042
9043 // Create a list of which PHIs we need to compute. We want to do this before
9044 // calling EvaluateExpression on them because that may invalidate iterators
9045 // into CurrentIterVals.
9046 SmallVector<PHINode *, 8> PHIsToCompute;
9047 for (const auto &I : CurrentIterVals) {
9048 PHINode *PHI = dyn_cast<PHINode>(I.first);
9049 if (!PHI || PHI->getParent() != Header) continue;
9050 PHIsToCompute.push_back(PHI);
9051 }
9052 for (PHINode *PHI : PHIsToCompute) {
9053 Constant *&NextPHI = NextIterVals[PHI];
9054 if (NextPHI) continue; // Already computed!
9055
9056 Value *BEValue = PHI->getIncomingValueForBlock(Latch);
9057 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI);
9058 }
9059 CurrentIterVals.swap(NextIterVals);
9060 }
9061
9062 // Too many iterations were needed to evaluate.
9063 return getCouldNotCompute();
9064}
9065
9066const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
9067 SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values =
9068 ValuesAtScopes[V];
9069 // Check to see if we've folded this expression at this loop before.
9070 for (auto &LS : Values)
9071 if (LS.first == L)
9072 return LS.second ? LS.second : V;
9073
9074 Values.emplace_back(L, nullptr);
9075
9076 // Otherwise compute it.
9077 const SCEV *C = computeSCEVAtScope(V, L);
9078 for (auto &LS : reverse(ValuesAtScopes[V]))
9079 if (LS.first == L) {
9080 LS.second = C;
9081 if (!isa<SCEVConstant>(C))
9082 ValuesAtScopesUsers[C].push_back({L, V});
9083 break;
9084 }
9085 return C;
9086}
9087
9088/// This builds up a Constant using the ConstantExpr interface. That way, we
9089/// will return Constants for objects which aren't represented by a
9090/// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
9091/// Returns NULL if the SCEV isn't representable as a Constant.
9092static Constant *BuildConstantFromSCEV(const SCEV *V) {
9093 switch (V->getSCEVType()) {
9094 case scCouldNotCompute:
9095 case scAddRecExpr:
9096 return nullptr;
9097 case scConstant:
9098 return cast<SCEVConstant>(V)->getValue();
9099 case scUnknown:
9100 return dyn_cast<Constant>(cast<SCEVUnknown>(V)->getValue());
9101 case scSignExtend: {
9102 const SCEVSignExtendExpr *SS = cast<SCEVSignExtendExpr>(V);
9103 if (Constant *CastOp = BuildConstantFromSCEV(SS->getOperand()))
9104 return ConstantExpr::getSExt(CastOp, SS->getType());
9105 return nullptr;
9106 }
9107 case scZeroExtend: {
9108 const SCEVZeroExtendExpr *SZ = cast<SCEVZeroExtendExpr>(V);
9109 if (Constant *CastOp = BuildConstantFromSCEV(SZ->getOperand()))
9110 return ConstantExpr::getZExt(CastOp, SZ->getType());
9111 return nullptr;
9112 }
9113 case scPtrToInt: {
9114 const SCEVPtrToIntExpr *P2I = cast<SCEVPtrToIntExpr>(V);
9115 if (Constant *CastOp = BuildConstantFromSCEV(P2I->getOperand()))
9116 return ConstantExpr::getPtrToInt(CastOp, P2I->getType());
9117
9118 return nullptr;
9119 }
9120 case scTruncate: {
9121 const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(V);
9122 if (Constant *CastOp = BuildConstantFromSCEV(ST->getOperand()))
9123 return ConstantExpr::getTrunc(CastOp, ST->getType());
9124 return nullptr;
9125 }
9126 case scAddExpr: {
9127 const SCEVAddExpr *SA = cast<SCEVAddExpr>(V);
9128 if (Constant *C = BuildConstantFromSCEV(SA->getOperand(0))) {
9129 if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) {
9130 unsigned AS = PTy->getAddressSpace();
9131 Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS);
9132 C = ConstantExpr::getBitCast(C, DestPtrTy);
9133 }
9134 for (unsigned i = 1, e = SA->getNumOperands(); i != e; ++i) {
9135 Constant *C2 = BuildConstantFromSCEV(SA->getOperand(i));
9136 if (!C2)
9137 return nullptr;
9138
9139 // First pointer!
9140 if (!C->getType()->isPointerTy() && C2->getType()->isPointerTy()) {
9141 unsigned AS = C2->getType()->getPointerAddressSpace();
9142 std::swap(C, C2);
9143 Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS);
9144 // The offsets have been converted to bytes. We can add bytes to an
9145 // i8* by GEP with the byte count in the first index.
9146 C = ConstantExpr::getBitCast(C, DestPtrTy);
9147 }
9148
9149 // Don't bother trying to sum two pointers. We probably can't
9150 // statically compute a load that results from it anyway.
9151 if (C2->getType()->isPointerTy())
9152 return nullptr;
9153
9154 if (C->getType()->isPointerTy()) {
9155 C = ConstantExpr::getGetElementPtr(Type::getInt8Ty(C->getContext()),
9156 C, C2);
9157 } else {
9158 C = ConstantExpr::getAdd(C, C2);
9159 }
9160 }
9161 return C;
9162 }
9163 return nullptr;
9164 }
9165 case scMulExpr: {
9166 const SCEVMulExpr *SM = cast<SCEVMulExpr>(V);
9167 if (Constant *C = BuildConstantFromSCEV(SM->getOperand(0))) {
9168 // Don't bother with pointers at all.
9169 if (C->getType()->isPointerTy())
9170 return nullptr;
9171 for (unsigned i = 1, e = SM->getNumOperands(); i != e; ++i) {
9172 Constant *C2 = BuildConstantFromSCEV(SM->getOperand(i));
9173 if (!C2 || C2->getType()->isPointerTy())
9174 return nullptr;
9175 C = ConstantExpr::getMul(C, C2);
9176 }
9177 return C;
9178 }
9179 return nullptr;
9180 }
9181 case scUDivExpr: {
9182 const SCEVUDivExpr *SU = cast<SCEVUDivExpr>(V);
9183 if (Constant *LHS = BuildConstantFromSCEV(SU->getLHS()))
9184 if (Constant *RHS = BuildConstantFromSCEV(SU->getRHS()))
9185 if (LHS->getType() == RHS->getType())
9186 return ConstantExpr::getUDiv(LHS, RHS);
9187 return nullptr;
9188 }
9189 case scSMaxExpr:
9190 case scUMaxExpr:
9191 case scSMinExpr:
9192 case scUMinExpr:
9193 case scSequentialUMinExpr:
9194 return nullptr; // TODO: smax, umax, smin, umax, umin_seq.
9195 }
9196 llvm_unreachable("Unknown SCEV kind!")::llvm::llvm_unreachable_internal("Unknown SCEV kind!", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 9196)
;
9197}
9198
9199const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
9200 if (isa<SCEVConstant>(V)) return V;
9201
9202 // If this instruction is evolved from a constant-evolving PHI, compute the
9203 // exit value from the loop without using SCEVs.
9204 if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) {
9205 if (Instruction *I = dyn_cast<Instruction>(SU->getValue())) {
9206 if (PHINode *PN = dyn_cast<PHINode>(I)) {
9207 const Loop *CurrLoop = this->LI[I->getParent()];
9208 // Looking for loop exit value.
9209 if (CurrLoop && CurrLoop->getParentLoop() == L &&
9210 PN->getParent() == CurrLoop->getHeader()) {
9211 // Okay, there is no closed form solution for the PHI node. Check
9212 // to see if the loop that contains it has a known backedge-taken
9213 // count. If so, we may be able to force computation of the exit
9214 // value.
9215 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(CurrLoop);
9216 // This trivial case can show up in some degenerate cases where
9217 // the incoming IR has not yet been fully simplified.
9218 if (BackedgeTakenCount->isZero()) {
9219 Value *InitValue = nullptr;
9220 bool MultipleInitValues = false;
9221 for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) {
9222 if (!CurrLoop->contains(PN->getIncomingBlock(i))) {
9223 if (!InitValue)
9224 InitValue = PN->getIncomingValue(i);
9225 else if (InitValue != PN->getIncomingValue(i)) {
9226 MultipleInitValues = true;
9227 break;
9228 }
9229 }
9230 }
9231 if (!MultipleInitValues && InitValue)
9232 return getSCEV(InitValue);
9233 }
9234 // Do we have a loop invariant value flowing around the backedge
9235 // for a loop which must execute the backedge?
9236 if (!isa<SCEVCouldNotCompute>(BackedgeTakenCount) &&
9237 isKnownPositive(BackedgeTakenCount) &&
9238 PN->getNumIncomingValues() == 2) {
9239
9240 unsigned InLoopPred =
9241 CurrLoop->contains(PN->getIncomingBlock(0)) ? 0 : 1;
9242 Value *BackedgeVal = PN->getIncomingValue(InLoopPred);
9243 if (CurrLoop->isLoopInvariant(BackedgeVal))
9244 return getSCEV(BackedgeVal);
9245 }
9246 if (auto *BTCC = dyn_cast<SCEVConstant>(BackedgeTakenCount)) {
9247 // Okay, we know how many times the containing loop executes. If
9248 // this is a constant evolving PHI node, get the final value at
9249 // the specified iteration number.
9250 Constant *RV = getConstantEvolutionLoopExitValue(
9251 PN, BTCC->getAPInt(), CurrLoop);
9252 if (RV) return getSCEV(RV);
9253 }
9254 }
9255
9256 // If there is a single-input Phi, evaluate it at our scope. If we can
9257 // prove that this replacement does not break LCSSA form, use new value.
9258 if (PN->getNumOperands() == 1) {
9259 const SCEV *Input = getSCEV(PN->getOperand(0));
9260 const SCEV *InputAtScope = getSCEVAtScope(Input, L);
9261 // TODO: We can generalize it using LI.replacementPreservesLCSSAForm,
9262 // for the simplest case just support constants.
9263 if (isa<SCEVConstant>(InputAtScope)) return InputAtScope;
9264 }
9265 }
9266
9267 // Okay, this is an expression that we cannot symbolically evaluate
9268 // into a SCEV. Check to see if it's possible to symbolically evaluate
9269 // the arguments into constants, and if so, try to constant propagate the
9270 // result. This is particularly useful for computing loop exit values.
9271 if (CanConstantFold(I)) {
9272 SmallVector<Constant *, 4> Operands;
9273 bool MadeImprovement = false;
9274 for (Value *Op : I->operands()) {
9275 if (Constant *C = dyn_cast<Constant>(Op)) {
9276 Operands.push_back(C);
9277 continue;
9278 }
9279
9280 // If any of the operands is non-constant and if they are
9281 // non-integer and non-pointer, don't even try to analyze them
9282 // with scev techniques.
9283 if (!isSCEVable(Op->getType()))
9284 return V;
9285
9286 const SCEV *OrigV = getSCEV(Op);
9287 const SCEV *OpV = getSCEVAtScope(OrigV, L);
9288 MadeImprovement |= OrigV != OpV;
9289
9290 Constant *C = BuildConstantFromSCEV(OpV);
9291 if (!C) return V;
9292 if (C->getType() != Op->getType())
9293 C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false,
9294 Op->getType(),
9295 false),
9296 C, Op->getType());
9297 Operands.push_back(C);
9298 }
9299
9300 // Check to see if getSCEVAtScope actually made an improvement.
9301 if (MadeImprovement) {
9302 Constant *C = nullptr;
9303 const DataLayout &DL = getDataLayout();
9304 if (const CmpInst *CI = dyn_cast<CmpInst>(I))
9305 C = ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0],
9306 Operands[1], DL, &TLI);
9307 else if (const LoadInst *Load = dyn_cast<LoadInst>(I)) {
9308 if (!Load->isVolatile())
9309 C = ConstantFoldLoadFromConstPtr(Operands[0], Load->getType(),
9310 DL);
9311 } else
9312 C = ConstantFoldInstOperands(I, Operands, DL, &TLI);
9313 if (!C) return V;
9314 return getSCEV(C);
9315 }
9316 }
9317 }
9318
9319 // This is some other type of SCEVUnknown, just return it.
9320 return V;
9321 }
9322
9323 if (isa<SCEVCommutativeExpr>(V) || isa<SCEVSequentialMinMaxExpr>(V)) {
9324 const auto *Comm = cast<SCEVNAryExpr>(V);
9325 // Avoid performing the look-up in the common case where the specified
9326 // expression has no loop-variant portions.
9327 for (unsigned i = 0, e = Comm->getNumOperands(); i != e; ++i) {
9328 const SCEV *OpAtScope = getSCEVAtScope(Comm->getOperand(i), L);
9329 if (OpAtScope != Comm->getOperand(i)) {
9330 // Okay, at least one of these operands is loop variant but might be
9331 // foldable. Build a new instance of the folded commutative expression.
9332 SmallVector<const SCEV *, 8> NewOps(Comm->op_begin(),
9333 Comm->op_begin()+i);
9334 NewOps.push_back(OpAtScope);
9335
9336 for (++i; i != e; ++i) {
9337 OpAtScope = getSCEVAtScope(Comm->getOperand(i), L);
9338 NewOps.push_back(OpAtScope);
9339 }
9340 if (isa<SCEVAddExpr>(Comm))
9341 return getAddExpr(NewOps, Comm->getNoWrapFlags());
9342 if (isa<SCEVMulExpr>(Comm))
9343 return getMulExpr(NewOps, Comm->getNoWrapFlags());
9344 if (isa<SCEVMinMaxExpr>(Comm))
9345 return getMinMaxExpr(Comm->getSCEVType(), NewOps);
9346 if (isa<SCEVSequentialMinMaxExpr>(Comm))
9347 return getSequentialMinMaxExpr(Comm->getSCEVType(), NewOps);
9348 llvm_unreachable("Unknown commutative / sequential min/max SCEV type!")::llvm::llvm_unreachable_internal("Unknown commutative / sequential min/max SCEV type!"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 9348)
;
9349 }
9350 }
9351 // If we got here, all operands are loop invariant.
9352 return Comm;
9353 }
9354
9355 if (const SCEVUDivExpr *Div = dyn_cast<SCEVUDivExpr>(V)) {
9356 const SCEV *LHS = getSCEVAtScope(Div->getLHS(), L);
9357 const SCEV *RHS = getSCEVAtScope(Div->getRHS(), L);
9358 if (LHS == Div->getLHS() && RHS == Div->getRHS())
9359 return Div; // must be loop invariant
9360 return getUDivExpr(LHS, RHS);
9361 }
9362
9363 // If this is a loop recurrence for a loop that does not contain L, then we
9364 // are dealing with the final value computed by the loop.
9365 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V)) {
9366 // First, attempt to evaluate each operand.
9367 // Avoid performing the look-up in the common case where the specified
9368 // expression has no loop-variant portions.
9369 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
9370 const SCEV *OpAtScope = getSCEVAtScope(AddRec->getOperand(i), L);
9371 if (OpAtScope == AddRec->getOperand(i))
9372 continue;
9373
9374 // Okay, at least one of these operands is loop variant but might be
9375 // foldable. Build a new instance of the folded commutative expression.
9376 SmallVector<const SCEV *, 8> NewOps(AddRec->op_begin(),
9377 AddRec->op_begin()+i);
9378 NewOps.push_back(OpAtScope);
9379 for (++i; i != e; ++i)
9380 NewOps.push_back(getSCEVAtScope(AddRec->getOperand(i), L));
9381
9382 const SCEV *FoldedRec =
9383 getAddRecExpr(NewOps, AddRec->getLoop(),
9384 AddRec->getNoWrapFlags(SCEV::FlagNW));
9385 AddRec = dyn_cast<SCEVAddRecExpr>(FoldedRec);
9386 // The addrec may be folded to a nonrecurrence, for example, if the
9387 // induction variable is multiplied by zero after constant folding. Go
9388 // ahead and return the folded value.
9389 if (!AddRec)
9390 return FoldedRec;
9391 break;
9392 }
9393
9394 // If the scope is outside the addrec's loop, evaluate it by using the
9395 // loop exit value of the addrec.
9396 if (!AddRec->getLoop()->contains(L)) {
9397 // To evaluate this recurrence, we need to know how many times the AddRec
9398 // loop iterates. Compute this now.
9399 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(AddRec->getLoop());
9400 if (BackedgeTakenCount == getCouldNotCompute()) return AddRec;
9401
9402 // Then, evaluate the AddRec.
9403 return AddRec->evaluateAtIteration(BackedgeTakenCount, *this);
9404 }
9405
9406 return AddRec;
9407 }
9408
9409 if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(V)) {
9410 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L);
9411 if (Op == Cast->getOperand())
9412 return Cast; // must be loop invariant
9413 return getCastExpr(Cast->getSCEVType(), Op, Cast->getType());
9414 }
9415
9416 llvm_unreachable("Unknown SCEV type!")::llvm::llvm_unreachable_internal("Unknown SCEV type!", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 9416)
;
9417}
9418
9419const SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) {
9420 return getSCEVAtScope(getSCEV(V), L);
9421}
9422
9423const SCEV *ScalarEvolution::stripInjectiveFunctions(const SCEV *S) const {
9424 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S))
9425 return stripInjectiveFunctions(ZExt->getOperand());
9426 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S))
9427 return stripInjectiveFunctions(SExt->getOperand());
9428 return S;
9429}
9430
9431/// Finds the minimum unsigned root of the following equation:
9432///
9433/// A * X = B (mod N)
9434///
9435/// where N = 2^BW and BW is the common bit width of A and B. The signedness of
9436/// A and B isn't important.
9437///
9438/// If the equation does not have a solution, SCEVCouldNotCompute is returned.
9439static const SCEV *SolveLinEquationWithOverflow(const APInt &A, const SCEV *B,
9440 ScalarEvolution &SE) {
9441 uint32_t BW = A.getBitWidth();
9442 assert(BW == SE.getTypeSizeInBits(B->getType()))(static_cast <bool> (BW == SE.getTypeSizeInBits(B->getType
())) ? void (0) : __assert_fail ("BW == SE.getTypeSizeInBits(B->getType())"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 9442, __extension__
__PRETTY_FUNCTION__))
;
9443 assert(A != 0 && "A must be non-zero.")(static_cast <bool> (A != 0 && "A must be non-zero."
) ? void (0) : __assert_fail ("A != 0 && \"A must be non-zero.\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 9443, __extension__
__PRETTY_FUNCTION__))
;
9444
9445 // 1. D = gcd(A, N)
9446 //
9447 // The gcd of A and N may have only one prime factor: 2. The number of
9448 // trailing zeros in A is its multiplicity
9449 uint32_t Mult2 = A.countTrailingZeros();
9450 // D = 2^Mult2
9451
9452 // 2. Check if B is divisible by D.
9453 //
9454 // B is divisible by D if and only if the multiplicity of prime factor 2 for B
9455 // is not less than multiplicity of this prime factor for D.
9456 if (SE.GetMinTrailingZeros(B) < Mult2)
9457 return SE.getCouldNotCompute();
9458
9459 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
9460 // modulo (N / D).
9461 //
9462 // If D == 1, (N / D) == N == 2^BW, so we need one extra bit to represent
9463 // (N / D) in general. The inverse itself always fits into BW bits, though,
9464 // so we immediately truncate it.
9465 APInt AD = A.lshr(Mult2).zext(BW + 1); // AD = A / D
9466 APInt Mod(BW + 1, 0);
9467 Mod.setBit(BW - Mult2); // Mod = N / D
9468 APInt I = AD.multiplicativeInverse(Mod).trunc(BW);
9469
9470 // 4. Compute the minimum unsigned root of the equation:
9471 // I * (B / D) mod (N / D)
9472 // To simplify the computation, we factor out the divide by D:
9473 // (I * B mod N) / D
9474 const SCEV *D = SE.getConstant(APInt::getOneBitSet(BW, Mult2));
9475 return SE.getUDivExactExpr(SE.getMulExpr(B, SE.getConstant(I)), D);
9476}
9477
9478/// For a given quadratic addrec, generate coefficients of the corresponding
9479/// quadratic equation, multiplied by a common value to ensure that they are
9480/// integers.
9481/// The returned value is a tuple { A, B, C, M, BitWidth }, where
9482/// Ax^2 + Bx + C is the quadratic function, M is the value that A, B and C
9483/// were multiplied by, and BitWidth is the bit width of the original addrec
9484/// coefficients.
9485/// This function returns None if the addrec coefficients are not compile-
9486/// time constants.
9487static Optional<std::tuple<APInt, APInt, APInt, APInt, unsigned>>
9488GetQuadraticEquation(const SCEVAddRecExpr *AddRec) {
9489 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!")(static_cast <bool> (AddRec->getNumOperands() == 3 &&
"This is not a quadratic chrec!") ? void (0) : __assert_fail
("AddRec->getNumOperands() == 3 && \"This is not a quadratic chrec!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 9489, __extension__
__PRETTY_FUNCTION__))
;
9490 const SCEVConstant *LC = dyn_cast<SCEVConstant>(AddRec->getOperand(0));
9491 const SCEVConstant *MC = dyn_cast<SCEVConstant>(AddRec->getOperand(1));
9492 const SCEVConstant *NC = dyn_cast<SCEVConstant>(AddRec->getOperand(2));
9493 LLVM_DEBUG(dbgs() << __func__ << ": analyzing quadratic addrec: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << __func__ << ": analyzing quadratic addrec: "
<< *AddRec << '\n'; } } while (false)
9494 << *AddRec << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << __func__ << ": analyzing quadratic addrec: "
<< *AddRec << '\n'; } } while (false)
;
9495
9496 // We currently can only solve this if the coefficients are constants.
9497 if (!LC || !MC || !NC) {
9498 LLVM_DEBUG(dbgs() << __func__ << ": coefficients are not constant\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << __func__ << ": coefficients are not constant\n"
; } } while (false)
;
9499 return None;
9500 }
9501
9502 APInt L = LC->getAPInt();
9503 APInt M = MC->getAPInt();
9504 APInt N = NC->getAPInt();
9505 assert(!N.isZero() && "This is not a quadratic addrec")(static_cast <bool> (!N.isZero() && "This is not a quadratic addrec"
) ? void (0) : __assert_fail ("!N.isZero() && \"This is not a quadratic addrec\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 9505, __extension__
__PRETTY_FUNCTION__))
;
9506
9507 unsigned BitWidth = LC->getAPInt().getBitWidth();
9508 unsigned NewWidth = BitWidth + 1;
9509 LLVM_DEBUG(dbgs() << __func__ << ": addrec coeff bw: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << __func__ << ": addrec coeff bw: "
<< BitWidth << '\n'; } } while (false)
9510 << BitWidth << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << __func__ << ": addrec coeff bw: "
<< BitWidth << '\n'; } } while (false)
;
9511 // The sign-extension (as opposed to a zero-extension) here matches the
9512 // extension used in SolveQuadraticEquationWrap (with the same motivation).
9513 N = N.sext(NewWidth);
9514 M = M.sext(NewWidth);
9515 L = L.sext(NewWidth);
9516
9517 // The increments are M, M+N, M+2N, ..., so the accumulated values are
9518 // L+M, (L+M)+(M+N), (L+M)+(M+N)+(M+2N), ..., that is,
9519 // L+M, L+2M+N, L+3M+3N, ...
9520 // After n iterations the accumulated value Acc is L + nM + n(n-1)/2 N.
9521 //
9522 // The equation Acc = 0 is then
9523 // L + nM + n(n-1)/2 N = 0, or 2L + 2M n + n(n-1) N = 0.
9524 // In a quadratic form it becomes:
9525 // N n^2 + (2M-N) n + 2L = 0.
9526
9527 APInt A = N;
9528 APInt B = 2 * M - A;
9529 APInt C = 2 * L;
9530 APInt T = APInt(NewWidth, 2);
9531 LLVM_DEBUG(dbgs() << __func__ << ": equation " << A << "x^2 + " << Bdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << __func__ << ": equation "
<< A << "x^2 + " << B << "x + " <<
C << ", coeff bw: " << NewWidth << ", multiplied by "
<< T << '\n'; } } while (false)
9532 << "x + " << C << ", coeff bw: " << NewWidthdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << __func__ << ": equation "
<< A << "x^2 + " << B << "x + " <<
C << ", coeff bw: " << NewWidth << ", multiplied by "
<< T << '\n'; } } while (false)
9533 << ", multiplied by " << T << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << __func__ << ": equation "
<< A << "x^2 + " << B << "x + " <<
C << ", coeff bw: " << NewWidth << ", multiplied by "
<< T << '\n'; } } while (false)
;
9534 return std::make_tuple(A, B, C, T, BitWidth);
9535}
9536
9537/// Helper function to compare optional APInts:
9538/// (a) if X and Y both exist, return min(X, Y),
9539/// (b) if neither X nor Y exist, return None,
9540/// (c) if exactly one of X and Y exists, return that value.
9541static Optional<APInt> MinOptional(Optional<APInt> X, Optional<APInt> Y) {
9542 if (X.hasValue() && Y.hasValue()) {
9543 unsigned W = std::max(X->getBitWidth(), Y->getBitWidth());
9544 APInt XW = X->sextOrSelf(W);
9545 APInt YW = Y->sextOrSelf(W);
9546 return XW.slt(YW) ? *X : *Y;
9547 }
9548 if (!X.hasValue() && !Y.hasValue())
9549 return None;
9550 return X.hasValue() ? *X : *Y;
9551}
9552
9553/// Helper function to truncate an optional APInt to a given BitWidth.
9554/// When solving addrec-related equations, it is preferable to return a value
9555/// that has the same bit width as the original addrec's coefficients. If the
9556/// solution fits in the original bit width, truncate it (except for i1).
9557/// Returning a value of a different bit width may inhibit some optimizations.
9558///
9559/// In general, a solution to a quadratic equation generated from an addrec
9560/// may require BW+1 bits, where BW is the bit width of the addrec's
9561/// coefficients. The reason is that the coefficients of the quadratic
9562/// equation are BW+1 bits wide (to avoid truncation when converting from
9563/// the addrec to the equation).
9564static Optional<APInt> TruncIfPossible(Optional<APInt> X, unsigned BitWidth) {
9565 if (!X.hasValue())
9566 return None;
9567 unsigned W = X->getBitWidth();
9568 if (BitWidth > 1 && BitWidth < W && X->isIntN(BitWidth))
9569 return X->trunc(BitWidth);
9570 return X;
9571}
9572
9573/// Let c(n) be the value of the quadratic chrec {L,+,M,+,N} after n
9574/// iterations. The values L, M, N are assumed to be signed, and they
9575/// should all have the same bit widths.
9576/// Find the least n >= 0 such that c(n) = 0 in the arithmetic modulo 2^BW,
9577/// where BW is the bit width of the addrec's coefficients.
9578/// If the calculated value is a BW-bit integer (for BW > 1), it will be
9579/// returned as such, otherwise the bit width of the returned value may
9580/// be greater than BW.
9581///
9582/// This function returns None if
9583/// (a) the addrec coefficients are not constant, or
9584/// (b) SolveQuadraticEquationWrap was unable to find a solution. For cases
9585/// like x^2 = 5, no integer solutions exist, in other cases an integer
9586/// solution may exist, but SolveQuadraticEquationWrap may fail to find it.
9587static Optional<APInt>
9588SolveQuadraticAddRecExact(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
9589 APInt A, B, C, M;
9590 unsigned BitWidth;
9591 auto T = GetQuadraticEquation(AddRec);
9592 if (!T.hasValue())
9593 return None;
9594
9595 std::tie(A, B, C, M, BitWidth) = *T;
9596 LLVM_DEBUG(dbgs() << __func__ << ": solving for unsigned overflow\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << __func__ << ": solving for unsigned overflow\n"
; } } while (false)
;
9597 Optional<APInt> X = APIntOps::SolveQuadraticEquationWrap(A, B, C, BitWidth+1);
9598 if (!X.hasValue())
9599 return None;
9600
9601 ConstantInt *CX = ConstantInt::get(SE.getContext(), *X);
9602 ConstantInt *V = EvaluateConstantChrecAtConstant(AddRec, CX, SE);
9603 if (!V->isZero())
9604 return None;
9605
9606 return TruncIfPossible(X, BitWidth);
9607}
9608
9609/// Let c(n) be the value of the quadratic chrec {0,+,M,+,N} after n
9610/// iterations. The values M, N are assumed to be signed, and they
9611/// should all have the same bit widths.
9612/// Find the least n such that c(n) does not belong to the given range,
9613/// while c(n-1) does.
9614///
9615/// This function returns None if
9616/// (a) the addrec coefficients are not constant, or
9617/// (b) SolveQuadraticEquationWrap was unable to find a solution for the
9618/// bounds of the range.
9619static Optional<APInt>
9620SolveQuadraticAddRecRange(const SCEVAddRecExpr *AddRec,
9621 const ConstantRange &Range, ScalarEvolution &SE) {
9622 assert(AddRec->getOperand(0)->isZero() &&(static_cast <bool> (AddRec->getOperand(0)->isZero
() && "Starting value of addrec should be 0") ? void (
0) : __assert_fail ("AddRec->getOperand(0)->isZero() && \"Starting value of addrec should be 0\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 9623, __extension__
__PRETTY_FUNCTION__))
9623 "Starting value of addrec should be 0")(static_cast <bool> (AddRec->getOperand(0)->isZero
() && "Starting value of addrec should be 0") ? void (
0) : __assert_fail ("AddRec->getOperand(0)->isZero() && \"Starting value of addrec should be 0\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 9623, __extension__
__PRETTY_FUNCTION__))
;
9624 LLVM_DEBUG(dbgs() << __func__ << ": solving boundary crossing for range "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << __func__ << ": solving boundary crossing for range "
<< Range << ", addrec " << *AddRec <<
'\n'; } } while (false)
9625 << Range << ", addrec " << *AddRec << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << __func__ << ": solving boundary crossing for range "
<< Range << ", addrec " << *AddRec <<
'\n'; } } while (false)
;
9626 // This case is handled in getNumIterationsInRange. Here we can assume that
9627 // we start in the range.
9628 assert(Range.contains(APInt(SE.getTypeSizeInBits(AddRec->getType()), 0)) &&(static_cast <bool> (Range.contains(APInt(SE.getTypeSizeInBits
(AddRec->getType()), 0)) && "Addrec's initial value should be in range"
) ? void (0) : __assert_fail ("Range.contains(APInt(SE.getTypeSizeInBits(AddRec->getType()), 0)) && \"Addrec's initial value should be in range\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 9629, __extension__
__PRETTY_FUNCTION__))
9629 "Addrec's initial value should be in range")(static_cast <bool> (Range.contains(APInt(SE.getTypeSizeInBits
(AddRec->getType()), 0)) && "Addrec's initial value should be in range"
) ? void (0) : __assert_fail ("Range.contains(APInt(SE.getTypeSizeInBits(AddRec->getType()), 0)) && \"Addrec's initial value should be in range\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 9629, __extension__
__PRETTY_FUNCTION__))
;
9630
9631 APInt A, B, C, M;
9632 unsigned BitWidth;
9633 auto T = GetQuadraticEquation(AddRec);
9634 if (!T.hasValue())
9635 return None;
9636
9637 // Be careful about the return value: there can be two reasons for not
9638 // returning an actual number. First, if no solutions to the equations
9639 // were found, and second, if the solutions don't leave the given range.
9640 // The first case means that the actual solution is "unknown", the second
9641 // means that it's known, but not valid. If the solution is unknown, we
9642 // cannot make any conclusions.
9643 // Return a pair: the optional solution and a flag indicating if the
9644 // solution was found.
9645 auto SolveForBoundary = [&](APInt Bound) -> std::pair<Optional<APInt>,bool> {
9646 // Solve for signed overflow and unsigned overflow, pick the lower
9647 // solution.
9648 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: checking boundary "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << "SolveQuadraticAddRecRange: checking boundary "
<< Bound << " (before multiplying by " << M
<< ")\n"; } } while (false)
9649 << Bound << " (before multiplying by " << M << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << "SolveQuadraticAddRecRange: checking boundary "
<< Bound << " (before multiplying by " << M
<< ")\n"; } } while (false)
;
9650 Bound *= M; // The quadratic equation multiplier.
9651
9652 Optional<APInt> SO = None;
9653 if (BitWidth > 1) {
9654 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << "SolveQuadraticAddRecRange: solving for "
"signed overflow\n"; } } while (false)
9655 "signed overflow\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << "SolveQuadraticAddRecRange: solving for "
"signed overflow\n"; } } while (false)
;
9656 SO = APIntOps::SolveQuadraticEquationWrap(A, B, -Bound, BitWidth);
9657 }
9658 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << "SolveQuadraticAddRecRange: solving for "
"unsigned overflow\n"; } } while (false)
9659 "unsigned overflow\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("scalar-evolution")) { dbgs() << "SolveQuadraticAddRecRange: solving for "
"unsigned overflow\n"; } } while (false)
;
9660 Optional<APInt> UO = APIntOps::SolveQuadraticEquationWrap(A, B, -Bound,
9661 BitWidth+1);
9662
9663 auto LeavesRange = [&] (const APInt &X) {
9664 ConstantInt *C0 = ConstantInt::get(SE.getContext(), X);
9665 ConstantInt *V0 = EvaluateConstantChrecAtConstant(AddRec, C0, SE);
9666 if (Range.contains(V0->getValue()))
9667 return false;
9668 // X should be at least 1, so X-1 is non-negative.
9669 ConstantInt *C1 = ConstantInt::get(SE.getContext(), X-1);
9670 ConstantInt *V1 = EvaluateConstantChrecAtConstant(AddRec, C1, SE);
9671 if (Range.contains(V1->getValue()))
9672 return true;
9673 return false;
9674 };
9675
9676 // If SolveQuadraticEquationWrap returns None, it means that there can
9677 // be a solution, but the function failed to find it. We cannot treat it
9678 // as "no solution".
9679 if (!SO.hasValue() || !UO.hasValue())
9680 return { None, false };
9681
9682 // Check the smaller value first to see if it leaves the range.
9683 // At this point, both SO and UO must have values.
9684 Optional<APInt> Min = MinOptional(SO, UO);
9685 if (LeavesRange(*Min))
9686 return { Min, true };
9687 Optional<APInt> Max = Min == SO ? UO : SO;
9688 if (LeavesRange(*Max))
9689 return { Max, true };
9690
9691 // Solutions were found, but were eliminated, hence the "true".
9692 return { None, true };
9693 };
9694
9695 std::tie(A, B, C, M, BitWidth) = *T;
9696 // Lower bound is inclusive, subtract 1 to represent the exiting value.
9697 APInt Lower = Range.getLower().sextOrSelf(A.getBitWidth()) - 1;
9698 APInt Upper = Range.getUpper().sextOrSelf(A.getBitWidth());
9699 auto SL = SolveForBoundary(Lower);
9700 auto SU = SolveForBoundary(Upper);
9701 // If any of the solutions was unknown, no meaninigful conclusions can
9702 // be made.
9703 if (!SL.second || !SU.second)
9704 return None;
9705
9706 // Claim: The correct solution is not some value between Min and Max.
9707 //
9708 // Justification: Assuming that Min and Max are different values, one of
9709 // them is when the first signed overflow happens, the other is when the
9710 // first unsigned overflow happens. Crossing the range boundary is only
9711 // possible via an overflow (treating 0 as a special case of it, modeling
9712 // an overflow as crossing k*2^W for some k).
9713 //
9714 // The interesting case here is when Min was eliminated as an invalid
9715 // solution, but Max was not. The argument is that if there was another
9716 // overflow between Min and Max, it would also have been eliminated if
9717 // it was considered.
9718 //
9719 // For a given boundary, it is possible to have two overflows of the same
9720 // type (signed/unsigned) without having the other type in between: this
9721 // can happen when the vertex of the parabola is between the iterations
9722 // corresponding to the overflows. This is only possible when the two
9723 // overflows cross k*2^W for the same k. In such case, if the second one
9724 // left the range (and was the first one to do so), the first overflow
9725 // would have to enter the range, which would mean that either we had left
9726 // the range before or that we started outside of it. Both of these cases
9727 // are contradictions.
9728 //
9729 // Claim: In the case where SolveForBoundary returns None, the correct
9730 // solution is not some value between the Max for this boundary and the
9731 // Min of the other boundary.
9732 //
9733 // Justification: Assume that we had such Max_A and Min_B corresponding
9734 // to range boundaries A and B and such that Max_A < Min_B. If there was
9735 // a solution between Max_A and Min_B, it would have to be caused by an
9736 // overflow corresponding to either A or B. It cannot correspond to B,
9737 // since Min_B is the first occurrence of such an overflow. If it
9738 // corresponded to A, it would have to be either a signed or an unsigned
9739 // overflow that is larger than both eliminated overflows for A. But
9740 // between the eliminated overflows and this overflow, the values would
9741 // cover the entire value space, thus crossing the other boundary, which
9742 // is a contradiction.
9743
9744 return TruncIfPossible(MinOptional(SL.first, SU.first), BitWidth);
9745}
9746
9747ScalarEvolution::ExitLimit
9748ScalarEvolution::howFarToZero(const SCEV *V, const Loop *L, bool ControlsExit,
9749 bool AllowPredicates) {
9750
9751 // This is only used for loops with a "x != y" exit test. The exit condition
9752 // is now expressed as a single expression, V = x-y. So the exit test is
9753 // effectively V != 0. We know and take advantage of the fact that this
9754 // expression only being used in a comparison by zero context.
9755
9756 SmallPtrSet<const SCEVPredicate *, 4> Predicates;
9757 // If the value is a constant
9758 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
9759 // If the value is already zero, the branch will execute zero times.
9760 if (C->getValue()->isZero()) return C;
9761 return getCouldNotCompute(); // Otherwise it will loop infinitely.
9762 }
9763
9764 const SCEVAddRecExpr *AddRec =
9765 dyn_cast<SCEVAddRecExpr>(stripInjectiveFunctions(V));
9766
9767 if (!AddRec && AllowPredicates)
9768 // Try to make this an AddRec using runtime tests, in the first X
9769 // iterations of this loop, where X is the SCEV expression found by the
9770 // algorithm below.
9771 AddRec = convertSCEVToAddRecWithPredicates(V, L, Predicates);
9772
9773 if (!AddRec || AddRec->getLoop() != L)
9774 return getCouldNotCompute();
9775
9776 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
9777 // the quadratic equation to solve it.
9778 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
9779 // We can only use this value if the chrec ends up with an exact zero
9780 // value at this index. When solving for "X*X != 5", for example, we
9781 // should not accept a root of 2.
9782 if (auto S = SolveQuadraticAddRecExact(AddRec, *this)) {
9783 const auto *R = cast<SCEVConstant>(getConstant(S.getValue()));
9784 return ExitLimit(R, R, false, Predicates);
9785 }
9786 return getCouldNotCompute();
9787 }
9788
9789 // Otherwise we can only handle this if it is affine.
9790 if (!AddRec->isAffine())
9791 return getCouldNotCompute();
9792
9793 // If this is an affine expression, the execution count of this branch is
9794 // the minimum unsigned root of the following equation:
9795 //
9796 // Start + Step*N = 0 (mod 2^BW)
9797 //
9798 // equivalent to:
9799 //
9800 // Step*N = -Start (mod 2^BW)
9801 //
9802 // where BW is the common bit width of Start and Step.
9803
9804 // Get the initial value for the loop.
9805 const SCEV *Start = getSCEVAtScope(AddRec->getStart(), L->getParentLoop());
9806 const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1), L->getParentLoop());
9807
9808 // For now we handle only constant steps.
9809 //
9810 // TODO: Handle a nonconstant Step given AddRec<NUW>. If the
9811 // AddRec is NUW, then (in an unsigned sense) it cannot be counting up to wrap
9812 // to 0, it must be counting down to equal 0. Consequently, N = Start / -Step.
9813 // We have not yet seen any such cases.
9814 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Step);
9815 if (!StepC || StepC->getValue()->isZero())
9816 return getCouldNotCompute();
9817
9818 // For positive steps (counting up until unsigned overflow):
9819 // N = -Start/Step (as unsigned)
9820 // For negative steps (counting down to zero):
9821 // N = Start/-Step
9822 // First compute the unsigned distance from zero in the direction of Step.
9823 bool CountDown = StepC->getAPInt().isNegative();
9824 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(Start);
9825
9826 // Handle unitary steps, which cannot wraparound.
9827 // 1*N = -Start; -1*N = Start (mod 2^BW), so:
9828 // N = Distance (as unsigned)
9829 if (StepC->getValue()->isOne() || StepC->getValue()->isMinusOne()) {
9830 APInt MaxBECount = getUnsignedRangeMax(applyLoopGuards(Distance, L));
9831 MaxBECount = APIntOps::umin(MaxBECount, getUnsignedRangeMax(Distance));
9832
9833 // When a loop like "for (int i = 0; i != n; ++i) { /* body */ }" is rotated,
9834 // we end up with a loop whose backedge-taken count is n - 1. Detect this
9835 // case, and see if we can improve the bound.
9836 //
9837 // Explicitly handling this here is necessary because getUnsignedRange
9838 // isn't context-sensitive; it doesn't know that we only care about the
9839 // range inside the loop.
9840 const SCEV *Zero = getZero(Distance->getType());
9841 const SCEV *One = getOne(Distance->getType());
9842 const SCEV *DistancePlusOne = getAddExpr(Distance, One);
9843 if (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_NE, DistancePlusOne, Zero)) {
9844 // If Distance + 1 doesn't overflow, we can compute the maximum distance
9845 // as "unsigned_max(Distance + 1) - 1".
9846 ConstantRange CR = getUnsignedRange(DistancePlusOne);
9847 MaxBECount = APIntOps::umin(MaxBECount, CR.getUnsignedMax() - 1);
9848 }
9849 return ExitLimit(Distance, getConstant(MaxBECount), false, Predicates);
9850 }
9851
9852 // If the condition controls loop exit (the loop exits only if the expression
9853 // is true) and the addition is no-wrap we can use unsigned divide to
9854 // compute the backedge count. In this case, the step may not divide the
9855 // distance, but we don't care because if the condition is "missed" the loop
9856 // will have undefined behavior due to wrapping.
9857 if (ControlsExit && AddRec->hasNoSelfWrap() &&
9858 loopHasNoAbnormalExits(AddRec->getLoop())) {
9859 const SCEV *Exact =
9860 getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step);
9861 const SCEV *Max = getCouldNotCompute();
9862 if (Exact != getCouldNotCompute()) {
9863 APInt MaxInt = getUnsignedRangeMax(applyLoopGuards(Exact, L));
9864 Max = getConstant(APIntOps::umin(MaxInt, getUnsignedRangeMax(Exact)));
9865 }
9866 return ExitLimit(Exact, Max, false, Predicates);
9867 }
9868
9869 // Solve the general equation.
9870 const SCEV *E = SolveLinEquationWithOverflow(StepC->getAPInt(),
9871 getNegativeSCEV(Start), *this);
9872
9873 const SCEV *M = E;
9874 if (E != getCouldNotCompute()) {
9875 APInt MaxWithGuards = getUnsignedRangeMax(applyLoopGuards(E, L));
9876 M = getConstant(APIntOps::umin(MaxWithGuards, getUnsignedRangeMax(E)));
9877 }
9878 return ExitLimit(E, M, false, Predicates);
9879}
9880
9881ScalarEvolution::ExitLimit
9882ScalarEvolution::howFarToNonZero(const SCEV *V, const Loop *L) {
9883 // Loops that look like: while (X == 0) are very strange indeed. We don't
9884 // handle them yet except for the trivial case. This could be expanded in the
9885 // future as needed.
9886
9887 // If the value is a constant, check to see if it is known to be non-zero
9888 // already. If so, the backedge will execute zero times.
9889 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
9890 if (!C->getValue()->isZero())
9891 return getZero(C->getType());
9892 return getCouldNotCompute(); // Otherwise it will loop infinitely.
9893 }
9894
9895 // We could implement others, but I really doubt anyone writes loops like
9896 // this, and if they did, they would already be constant folded.
9897 return getCouldNotCompute();
9898}
9899
9900std::pair<const BasicBlock *, const BasicBlock *>
9901ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(const BasicBlock *BB)
9902 const {
9903 // If the block has a unique predecessor, then there is no path from the
9904 // predecessor to the block that does not go through the direct edge
9905 // from the predecessor to the block.
9906 if (const BasicBlock *Pred = BB->getSinglePredecessor())
9907 return {Pred, BB};
9908
9909 // A loop's header is defined to be a block that dominates the loop.
9910 // If the header has a unique predecessor outside the loop, it must be
9911 // a block that has exactly one successor that can reach the loop.
9912 if (const Loop *L = LI.getLoopFor(BB))
9913 return {L->getLoopPredecessor(), L->getHeader()};
9914
9915 return {nullptr, nullptr};
9916}
9917
9918/// SCEV structural equivalence is usually sufficient for testing whether two
9919/// expressions are equal, however for the purposes of looking for a condition
9920/// guarding a loop, it can be useful to be a little more general, since a
9921/// front-end may have replicated the controlling expression.
9922static bool HasSameValue(const SCEV *A, const SCEV *B) {
9923 // Quick check to see if they are the same SCEV.
9924 if (A == B) return true;
9925
9926 auto ComputesEqualValues = [](const Instruction *A, const Instruction *B) {
9927 // Not all instructions that are "identical" compute the same value. For
9928 // instance, two distinct alloca instructions allocating the same type are
9929 // identical and do not read memory; but compute distinct values.
9930 return A->isIdenticalTo(B) && (isa<BinaryOperator>(A) || isa<GetElementPtrInst>(A));
9931 };
9932
9933 // Otherwise, if they're both SCEVUnknown, it's possible that they hold
9934 // two different instructions with the same value. Check for this case.
9935 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(A))
9936 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(B))
9937 if (const Instruction *AI = dyn_cast<Instruction>(AU->getValue()))
9938 if (const Instruction *BI = dyn_cast<Instruction>(BU->getValue()))
9939 if (ComputesEqualValues(AI, BI))
9940 return true;
9941
9942 // Otherwise assume they may have a different value.
9943 return false;
9944}
9945
9946bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred,
9947 const SCEV *&LHS, const SCEV *&RHS,
9948 unsigned Depth) {
9949 bool Changed = false;
9950 // Simplifies ICMP to trivial true or false by turning it into '0 == 0' or
9951 // '0 != 0'.
9952 auto TrivialCase = [&](bool TriviallyTrue) {
9953 LHS = RHS = getConstant(ConstantInt::getFalse(getContext()));
9954 Pred = TriviallyTrue ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
9955 return true;
9956 };
9957 // If we hit the max recursion limit bail out.
9958 if (Depth >= 3)
9959 return false;
9960
9961 // Canonicalize a constant to the right side.
9962 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
9963 // Check for both operands constant.
9964 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
9965 if (ConstantExpr::getICmp(Pred,
9966 LHSC->getValue(),
9967 RHSC->getValue())->isNullValue())
9968 return TrivialCase(false);
9969 else
9970 return TrivialCase(true);
9971 }
9972 // Otherwise swap the operands to put the constant on the right.
9973 std::swap(LHS, RHS);
9974 Pred = ICmpInst::getSwappedPredicate(Pred);
9975 Changed = true;
9976 }
9977
9978 // If we're comparing an addrec with a value which is loop-invariant in the
9979 // addrec's loop, put the addrec on the left. Also make a dominance check,
9980 // as both operands could be addrecs loop-invariant in each other's loop.
9981 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) {
9982 const Loop *L = AR->getLoop();
9983 if (isLoopInvariant(LHS, L) && properlyDominates(LHS, L->getHeader())) {
9984 std::swap(LHS, RHS);
9985 Pred = ICmpInst::getSwappedPredicate(Pred);
9986 Changed = true;
9987 }
9988 }
9989
9990 // If there's a constant operand, canonicalize comparisons with boundary
9991 // cases, and canonicalize *-or-equal comparisons to regular comparisons.
9992 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS)) {
9993 const APInt &RA = RC->getAPInt();
9994
9995 bool SimplifiedByConstantRange = false;
9996
9997 if (!ICmpInst::isEquality(Pred)) {
9998 ConstantRange ExactCR = ConstantRange::makeExactICmpRegion(Pred, RA);
9999 if (ExactCR.isFullSet())
10000 return TrivialCase(true);
10001 else if (ExactCR.isEmptySet())
10002 return TrivialCase(false);
10003
10004 APInt NewRHS;
10005 CmpInst::Predicate NewPred;
10006 if (ExactCR.getEquivalentICmp(NewPred, NewRHS) &&
10007 ICmpInst::isEquality(NewPred)) {
10008 // We were able to convert an inequality to an equality.
10009 Pred = NewPred;
10010 RHS = getConstant(NewRHS);
10011 Changed = SimplifiedByConstantRange = true;
10012 }
10013 }
10014
10015 if (!SimplifiedByConstantRange) {
10016 switch (Pred) {
10017 default:
10018 break;
10019 case ICmpInst::ICMP_EQ:
10020 case ICmpInst::ICMP_NE:
10021 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b.
10022 if (!RA)
10023 if (const SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(LHS))
10024 if (const SCEVMulExpr *ME =
10025 dyn_cast<SCEVMulExpr>(AE->getOperand(0)))
10026 if (AE->getNumOperands() == 2 && ME->getNumOperands() == 2 &&
10027 ME->getOperand(0)->isAllOnesValue()) {
10028 RHS = AE->getOperand(1);
10029 LHS = ME->getOperand(1);
10030 Changed = true;
10031 }
10032 break;
10033
10034
10035 // The "Should have been caught earlier!" messages refer to the fact
10036 // that the ExactCR.isFullSet() or ExactCR.isEmptySet() check above
10037 // should have fired on the corresponding cases, and canonicalized the
10038 // check to trivial case.
10039
10040 case ICmpInst::ICMP_UGE:
10041 assert(!RA.isMinValue() && "Should have been caught earlier!")(static_cast <bool> (!RA.isMinValue() && "Should have been caught earlier!"
) ? void (0) : __assert_fail ("!RA.isMinValue() && \"Should have been caught earlier!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10041, __extension__
__PRETTY_FUNCTION__))
;
10042 Pred = ICmpInst::ICMP_UGT;
10043 RHS = getConstant(RA - 1);
10044 Changed = true;
10045 break;
10046 case ICmpInst::ICMP_ULE:
10047 assert(!RA.isMaxValue() && "Should have been caught earlier!")(static_cast <bool> (!RA.isMaxValue() && "Should have been caught earlier!"
) ? void (0) : __assert_fail ("!RA.isMaxValue() && \"Should have been caught earlier!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10047, __extension__
__PRETTY_FUNCTION__))
;
10048 Pred = ICmpInst::ICMP_ULT;
10049 RHS = getConstant(RA + 1);
10050 Changed = true;
10051 break;
10052 case ICmpInst::ICMP_SGE:
10053 assert(!RA.isMinSignedValue() && "Should have been caught earlier!")(static_cast <bool> (!RA.isMinSignedValue() && "Should have been caught earlier!"
) ? void (0) : __assert_fail ("!RA.isMinSignedValue() && \"Should have been caught earlier!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10053, __extension__
__PRETTY_FUNCTION__))
;
10054 Pred = ICmpInst::ICMP_SGT;
10055 RHS = getConstant(RA - 1);
10056 Changed = true;
10057 break;
10058 case ICmpInst::ICMP_SLE:
10059 assert(!RA.isMaxSignedValue() && "Should have been caught earlier!")(static_cast <bool> (!RA.isMaxSignedValue() && "Should have been caught earlier!"
) ? void (0) : __assert_fail ("!RA.isMaxSignedValue() && \"Should have been caught earlier!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10059, __extension__
__PRETTY_FUNCTION__))
;
10060 Pred = ICmpInst::ICMP_SLT;
10061 RHS = getConstant(RA + 1);
10062 Changed = true;
10063 break;
10064 }
10065 }
10066 }
10067
10068 // Check for obvious equality.
10069 if (HasSameValue(LHS, RHS)) {
10070 if (ICmpInst::isTrueWhenEqual(Pred))
10071 return TrivialCase(true);
10072 if (ICmpInst::isFalseWhenEqual(Pred))
10073 return TrivialCase(false);
10074 }
10075
10076 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
10077 // adding or subtracting 1 from one of the operands.
10078 switch (Pred) {
10079 case ICmpInst::ICMP_SLE:
10080 if (!getSignedRangeMax(RHS).isMaxSignedValue()) {
10081 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
10082 SCEV::FlagNSW);
10083 Pred = ICmpInst::ICMP_SLT;
10084 Changed = true;
10085 } else if (!getSignedRangeMin(LHS).isMinSignedValue()) {
10086 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS,
10087 SCEV::FlagNSW);
10088 Pred = ICmpInst::ICMP_SLT;
10089 Changed = true;
10090 }
10091 break;
10092 case ICmpInst::ICMP_SGE:
10093 if (!getSignedRangeMin(RHS).isMinSignedValue()) {
10094 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS,
10095 SCEV::FlagNSW);
10096 Pred = ICmpInst::ICMP_SGT;
10097 Changed = true;
10098 } else if (!getSignedRangeMax(LHS).isMaxSignedValue()) {
10099 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
10100 SCEV::FlagNSW);
10101 Pred = ICmpInst::ICMP_SGT;
10102 Changed = true;
10103 }
10104 break;
10105 case ICmpInst::ICMP_ULE:
10106 if (!getUnsignedRangeMax(RHS).isMaxValue()) {
10107 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
10108 SCEV::FlagNUW);
10109 Pred = ICmpInst::ICMP_ULT;
10110 Changed = true;
10111 } else if (!getUnsignedRangeMin(LHS).isMinValue()) {
10112 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS);
10113 Pred = ICmpInst::ICMP_ULT;
10114 Changed = true;
10115 }
10116 break;
10117 case ICmpInst::ICMP_UGE:
10118 if (!getUnsignedRangeMin(RHS).isMinValue()) {
10119 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS);
10120 Pred = ICmpInst::ICMP_UGT;
10121 Changed = true;
10122 } else if (!getUnsignedRangeMax(LHS).isMaxValue()) {
10123 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
10124 SCEV::FlagNUW);
10125 Pred = ICmpInst::ICMP_UGT;
10126 Changed = true;
10127 }
10128 break;
10129 default:
10130 break;
10131 }
10132
10133 // TODO: More simplifications are possible here.
10134
10135 // Recursively simplify until we either hit a recursion limit or nothing
10136 // changes.
10137 if (Changed)
10138 return SimplifyICmpOperands(Pred, LHS, RHS, Depth+1);
10139
10140 return Changed;
10141}
10142
10143bool ScalarEvolution::isKnownNegative(const SCEV *S) {
10144 return getSignedRangeMax(S).isNegative();
10145}
10146
10147bool ScalarEvolution::isKnownPositive(const SCEV *S) {
10148 return getSignedRangeMin(S).isStrictlyPositive();
10149}
10150
10151bool ScalarEvolution::isKnownNonNegative(const SCEV *S) {
10152 return !getSignedRangeMin(S).isNegative();
10153}
10154
10155bool ScalarEvolution::isKnownNonPositive(const SCEV *S) {
10156 return !getSignedRangeMax(S).isStrictlyPositive();
10157}
10158
10159bool ScalarEvolution::isKnownNonZero(const SCEV *S) {
10160 return getUnsignedRangeMin(S) != 0;
10161}
10162
10163std::pair<const SCEV *, const SCEV *>
10164ScalarEvolution::SplitIntoInitAndPostInc(const Loop *L, const SCEV *S) {
10165 // Compute SCEV on entry of loop L.
10166 const SCEV *Start = SCEVInitRewriter::rewrite(S, L, *this);
10167 if (Start == getCouldNotCompute())
10168 return { Start, Start };
10169 // Compute post increment SCEV for loop L.
10170 const SCEV *PostInc = SCEVPostIncRewriter::rewrite(S, L, *this);
10171 assert(PostInc != getCouldNotCompute() && "Unexpected could not compute")(static_cast <bool> (PostInc != getCouldNotCompute() &&
"Unexpected could not compute") ? void (0) : __assert_fail (
"PostInc != getCouldNotCompute() && \"Unexpected could not compute\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10171, __extension__
__PRETTY_FUNCTION__))
;
10172 return { Start, PostInc };
10173}
10174
10175bool ScalarEvolution::isKnownViaInduction(ICmpInst::Predicate Pred,
10176 const SCEV *LHS, const SCEV *RHS) {
10177 // First collect all loops.
10178 SmallPtrSet<const Loop *, 8> LoopsUsed;
10179 getUsedLoops(LHS, LoopsUsed);
10180 getUsedLoops(RHS, LoopsUsed);
10181
10182 if (LoopsUsed.empty())
10183 return false;
10184
10185 // Domination relationship must be a linear order on collected loops.
10186#ifndef NDEBUG
10187 for (auto *L1 : LoopsUsed)
10188 for (auto *L2 : LoopsUsed)
10189 assert((DT.dominates(L1->getHeader(), L2->getHeader()) ||(static_cast <bool> ((DT.dominates(L1->getHeader(), L2
->getHeader()) || DT.dominates(L2->getHeader(), L1->
getHeader())) && "Domination relationship is not a linear order"
) ? void (0) : __assert_fail ("(DT.dominates(L1->getHeader(), L2->getHeader()) || DT.dominates(L2->getHeader(), L1->getHeader())) && \"Domination relationship is not a linear order\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10191, __extension__
__PRETTY_FUNCTION__))
10190 DT.dominates(L2->getHeader(), L1->getHeader())) &&(static_cast <bool> ((DT.dominates(L1->getHeader(), L2
->getHeader()) || DT.dominates(L2->getHeader(), L1->
getHeader())) && "Domination relationship is not a linear order"
) ? void (0) : __assert_fail ("(DT.dominates(L1->getHeader(), L2->getHeader()) || DT.dominates(L2->getHeader(), L1->getHeader())) && \"Domination relationship is not a linear order\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10191, __extension__
__PRETTY_FUNCTION__))
10191 "Domination relationship is not a linear order")(static_cast <bool> ((DT.dominates(L1->getHeader(), L2
->getHeader()) || DT.dominates(L2->getHeader(), L1->
getHeader())) && "Domination relationship is not a linear order"
) ? void (0) : __assert_fail ("(DT.dominates(L1->getHeader(), L2->getHeader()) || DT.dominates(L2->getHeader(), L1->getHeader())) && \"Domination relationship is not a linear order\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10191, __extension__
__PRETTY_FUNCTION__))
;
10192#endif
10193
10194 const Loop *MDL =
10195 *std::max_element(LoopsUsed.begin(), LoopsUsed.end(),
10196 [&](const Loop *L1, const Loop *L2) {
10197 return DT.properlyDominates(L1->getHeader(), L2->getHeader());
10198 });
10199
10200 // Get init and post increment value for LHS.
10201 auto SplitLHS = SplitIntoInitAndPostInc(MDL, LHS);
10202 // if LHS contains unknown non-invariant SCEV then bail out.
10203 if (SplitLHS.first == getCouldNotCompute())
10204 return false;
10205 assert (SplitLHS.second != getCouldNotCompute() && "Unexpected CNC")(static_cast <bool> (SplitLHS.second != getCouldNotCompute
() && "Unexpected CNC") ? void (0) : __assert_fail ("SplitLHS.second != getCouldNotCompute() && \"Unexpected CNC\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10205, __extension__
__PRETTY_FUNCTION__))
;
10206 // Get init and post increment value for RHS.
10207 auto SplitRHS = SplitIntoInitAndPostInc(MDL, RHS);
10208 // if RHS contains unknown non-invariant SCEV then bail out.
10209 if (SplitRHS.first == getCouldNotCompute())
10210 return false;
10211 assert (SplitRHS.second != getCouldNotCompute() && "Unexpected CNC")(static_cast <bool> (SplitRHS.second != getCouldNotCompute
() && "Unexpected CNC") ? void (0) : __assert_fail ("SplitRHS.second != getCouldNotCompute() && \"Unexpected CNC\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10211, __extension__
__PRETTY_FUNCTION__))
;
10212 // It is possible that init SCEV contains an invariant load but it does
10213 // not dominate MDL and is not available at MDL loop entry, so we should
10214 // check it here.
10215 if (!isAvailableAtLoopEntry(SplitLHS.first, MDL) ||
10216 !isAvailableAtLoopEntry(SplitRHS.first, MDL))
10217 return false;
10218
10219 // It seems backedge guard check is faster than entry one so in some cases
10220 // it can speed up whole estimation by short circuit
10221 return isLoopBackedgeGuardedByCond(MDL, Pred, SplitLHS.second,
10222 SplitRHS.second) &&
10223 isLoopEntryGuardedByCond(MDL, Pred, SplitLHS.first, SplitRHS.first);
10224}
10225
10226bool ScalarEvolution::isKnownPredicate(ICmpInst::Predicate Pred,
10227 const SCEV *LHS, const SCEV *RHS) {
10228 // Canonicalize the inputs first.
10229 (void)SimplifyICmpOperands(Pred, LHS, RHS);
10230
10231 if (isKnownViaInduction(Pred, LHS, RHS))
10232 return true;
10233
10234 if (isKnownPredicateViaSplitting(Pred, LHS, RHS))
10235 return true;
10236
10237 // Otherwise see what can be done with some simple reasoning.
10238 return isKnownViaNonRecursiveReasoning(Pred, LHS, RHS);
10239}
10240
10241Optional<bool> ScalarEvolution::evaluatePredicate(ICmpInst::Predicate Pred,
10242 const SCEV *LHS,
10243 const SCEV *RHS) {
10244 if (isKnownPredicate(Pred, LHS, RHS))
10245 return true;
10246 else if (isKnownPredicate(ICmpInst::getInversePredicate(Pred), LHS, RHS))
10247 return false;
10248 return None;
10249}
10250
10251bool ScalarEvolution::isKnownPredicateAt(ICmpInst::Predicate Pred,
10252 const SCEV *LHS, const SCEV *RHS,
10253 const Instruction *CtxI) {
10254 // TODO: Analyze guards and assumes from Context's block.
10255 return isKnownPredicate(Pred, LHS, RHS) ||
10256 isBasicBlockEntryGuardedByCond(CtxI->getParent(), Pred, LHS, RHS);
10257}
10258
10259Optional<bool> ScalarEvolution::evaluatePredicateAt(ICmpInst::Predicate Pred,
10260 const SCEV *LHS,
10261 const SCEV *RHS,
10262 const Instruction *CtxI) {
10263 Optional<bool> KnownWithoutContext = evaluatePredicate(Pred, LHS, RHS);
10264 if (KnownWithoutContext)
10265 return KnownWithoutContext;
10266
10267 if (isBasicBlockEntryGuardedByCond(CtxI->getParent(), Pred, LHS, RHS))
10268 return true;
10269 else if (isBasicBlockEntryGuardedByCond(CtxI->getParent(),
10270 ICmpInst::getInversePredicate(Pred),
10271 LHS, RHS))
10272 return false;
10273 return None;
10274}
10275
10276bool ScalarEvolution::isKnownOnEveryIteration(ICmpInst::Predicate Pred,
10277 const SCEVAddRecExpr *LHS,
10278 const SCEV *RHS) {
10279 const Loop *L = LHS->getLoop();
10280 return isLoopEntryGuardedByCond(L, Pred, LHS->getStart(), RHS) &&
10281 isLoopBackedgeGuardedByCond(L, Pred, LHS->getPostIncExpr(*this), RHS);
10282}
10283
10284Optional<ScalarEvolution::MonotonicPredicateType>
10285ScalarEvolution::getMonotonicPredicateType(const SCEVAddRecExpr *LHS,
10286 ICmpInst::Predicate Pred) {
10287 auto Result = getMonotonicPredicateTypeImpl(LHS, Pred);
10288
10289#ifndef NDEBUG
10290 // Verify an invariant: inverting the predicate should turn a monotonically
10291 // increasing change to a monotonically decreasing one, and vice versa.
10292 if (Result) {
10293 auto ResultSwapped =
10294 getMonotonicPredicateTypeImpl(LHS, ICmpInst::getSwappedPredicate(Pred));
10295
10296 assert(ResultSwapped.hasValue() && "should be able to analyze both!")(static_cast <bool> (ResultSwapped.hasValue() &&
"should be able to analyze both!") ? void (0) : __assert_fail
("ResultSwapped.hasValue() && \"should be able to analyze both!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10296, __extension__
__PRETTY_FUNCTION__))
;
10297 assert(ResultSwapped.getValue() != Result.getValue() &&(static_cast <bool> (ResultSwapped.getValue() != Result
.getValue() && "monotonicity should flip as we flip the predicate"
) ? void (0) : __assert_fail ("ResultSwapped.getValue() != Result.getValue() && \"monotonicity should flip as we flip the predicate\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10298, __extension__
__PRETTY_FUNCTION__))
10298 "monotonicity should flip as we flip the predicate")(static_cast <bool> (ResultSwapped.getValue() != Result
.getValue() && "monotonicity should flip as we flip the predicate"
) ? void (0) : __assert_fail ("ResultSwapped.getValue() != Result.getValue() && \"monotonicity should flip as we flip the predicate\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10298, __extension__
__PRETTY_FUNCTION__))
;
10299 }
10300#endif
10301
10302 return Result;
10303}
10304
10305Optional<ScalarEvolution::MonotonicPredicateType>
10306ScalarEvolution::getMonotonicPredicateTypeImpl(const SCEVAddRecExpr *LHS,
10307 ICmpInst::Predicate Pred) {
10308 // A zero step value for LHS means the induction variable is essentially a
10309 // loop invariant value. We don't really depend on the predicate actually
10310 // flipping from false to true (for increasing predicates, and the other way
10311 // around for decreasing predicates), all we care about is that *if* the
10312 // predicate changes then it only changes from false to true.
10313 //
10314 // A zero step value in itself is not very useful, but there may be places
10315 // where SCEV can prove X >= 0 but not prove X > 0, so it is helpful to be
10316 // as general as possible.
10317
10318 // Only handle LE/LT/GE/GT predicates.
10319 if (!ICmpInst::isRelational(Pred))
10320 return None;
10321
10322 bool IsGreater = ICmpInst::isGE(Pred) || ICmpInst::isGT(Pred);
10323 assert((IsGreater || ICmpInst::isLE(Pred) || ICmpInst::isLT(Pred)) &&(static_cast <bool> ((IsGreater || ICmpInst::isLE(Pred)
|| ICmpInst::isLT(Pred)) && "Should be greater or less!"
) ? void (0) : __assert_fail ("(IsGreater || ICmpInst::isLE(Pred) || ICmpInst::isLT(Pred)) && \"Should be greater or less!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10324, __extension__
__PRETTY_FUNCTION__))
10324 "Should be greater or less!")(static_cast <bool> ((IsGreater || ICmpInst::isLE(Pred)
|| ICmpInst::isLT(Pred)) && "Should be greater or less!"
) ? void (0) : __assert_fail ("(IsGreater || ICmpInst::isLE(Pred) || ICmpInst::isLT(Pred)) && \"Should be greater or less!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10324, __extension__
__PRETTY_FUNCTION__))
;
10325
10326 // Check that AR does not wrap.
10327 if (ICmpInst::isUnsigned(Pred)) {
10328 if (!LHS->hasNoUnsignedWrap())
10329 return None;
10330 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
10331 } else {
10332 assert(ICmpInst::isSigned(Pred) &&(static_cast <bool> (ICmpInst::isSigned(Pred) &&
"Relational predicate is either signed or unsigned!") ? void
(0) : __assert_fail ("ICmpInst::isSigned(Pred) && \"Relational predicate is either signed or unsigned!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10333, __extension__
__PRETTY_FUNCTION__))
10333 "Relational predicate is either signed or unsigned!")(static_cast <bool> (ICmpInst::isSigned(Pred) &&
"Relational predicate is either signed or unsigned!") ? void
(0) : __assert_fail ("ICmpInst::isSigned(Pred) && \"Relational predicate is either signed or unsigned!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10333, __extension__
__PRETTY_FUNCTION__))
;
10334 if (!LHS->hasNoSignedWrap())
10335 return None;
10336
10337 const SCEV *Step = LHS->getStepRecurrence(*this);
10338
10339 if (isKnownNonNegative(Step))
10340 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
10341
10342 if (isKnownNonPositive(Step))
10343 return !IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
10344
10345 return None;
10346 }
10347}
10348
10349Optional<ScalarEvolution::LoopInvariantPredicate>
10350ScalarEvolution::getLoopInvariantPredicate(ICmpInst::Predicate Pred,
10351 const SCEV *LHS, const SCEV *RHS,
10352 const Loop *L) {
10353
10354 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
10355 if (!isLoopInvariant(RHS, L)) {
10356 if (!isLoopInvariant(LHS, L))
10357 return None;
10358
10359 std::swap(LHS, RHS);
10360 Pred = ICmpInst::getSwappedPredicate(Pred);
10361 }
10362
10363 const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(LHS);
10364 if (!ArLHS || ArLHS->getLoop() != L)
10365 return None;
10366
10367 auto MonotonicType = getMonotonicPredicateType(ArLHS, Pred);
10368 if (!MonotonicType)
10369 return None;
10370 // If the predicate "ArLHS `Pred` RHS" monotonically increases from false to
10371 // true as the loop iterates, and the backedge is control dependent on
10372 // "ArLHS `Pred` RHS" == true then we can reason as follows:
10373 //
10374 // * if the predicate was false in the first iteration then the predicate
10375 // is never evaluated again, since the loop exits without taking the
10376 // backedge.
10377 // * if the predicate was true in the first iteration then it will
10378 // continue to be true for all future iterations since it is
10379 // monotonically increasing.
10380 //
10381 // For both the above possibilities, we can replace the loop varying
10382 // predicate with its value on the first iteration of the loop (which is
10383 // loop invariant).
10384 //
10385 // A similar reasoning applies for a monotonically decreasing predicate, by
10386 // replacing true with false and false with true in the above two bullets.
10387 bool Increasing = *MonotonicType == ScalarEvolution::MonotonicallyIncreasing;
10388 auto P = Increasing ? Pred : ICmpInst::getInversePredicate(Pred);
10389
10390 if (!isLoopBackedgeGuardedByCond(L, P, LHS, RHS))
10391 return None;
10392
10393 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(), RHS);
10394}
10395
10396Optional<ScalarEvolution::LoopInvariantPredicate>
10397ScalarEvolution::getLoopInvariantExitCondDuringFirstIterations(
10398 ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
10399 const Instruction *CtxI, const SCEV *MaxIter) {
10400 // Try to prove the following set of facts:
10401 // - The predicate is monotonic in the iteration space.
10402 // - If the check does not fail on the 1st iteration:
10403 // - No overflow will happen during first MaxIter iterations;
10404 // - It will not fail on the MaxIter'th iteration.
10405 // If the check does fail on the 1st iteration, we leave the loop and no
10406 // other checks matter.
10407
10408 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
10409 if (!isLoopInvariant(RHS, L)) {
10410 if (!isLoopInvariant(LHS, L))
10411 return None;
10412
10413 std::swap(LHS, RHS);
10414 Pred = ICmpInst::getSwappedPredicate(Pred);
10415 }
10416
10417 auto *AR = dyn_cast<SCEVAddRecExpr>(LHS);
10418 if (!AR || AR->getLoop() != L)
10419 return None;
10420
10421 // The predicate must be relational (i.e. <, <=, >=, >).
10422 if (!ICmpInst::isRelational(Pred))
10423 return None;
10424
10425 // TODO: Support steps other than +/- 1.
10426 const SCEV *Step = AR->getStepRecurrence(*this);
10427 auto *One = getOne(Step->getType());
10428 auto *MinusOne = getNegativeSCEV(One);
10429 if (Step != One && Step != MinusOne)
10430 return None;
10431
10432 // Type mismatch here means that MaxIter is potentially larger than max
10433 // unsigned value in start type, which mean we cannot prove no wrap for the
10434 // indvar.
10435 if (AR->getType() != MaxIter->getType())
10436 return None;
10437
10438 // Value of IV on suggested last iteration.
10439 const SCEV *Last = AR->evaluateAtIteration(MaxIter, *this);
10440 // Does it still meet the requirement?
10441 if (!isLoopBackedgeGuardedByCond(L, Pred, Last, RHS))
10442 return None;
10443 // Because step is +/- 1 and MaxIter has same type as Start (i.e. it does
10444 // not exceed max unsigned value of this type), this effectively proves
10445 // that there is no wrap during the iteration. To prove that there is no
10446 // signed/unsigned wrap, we need to check that
10447 // Start <= Last for step = 1 or Start >= Last for step = -1.
10448 ICmpInst::Predicate NoOverflowPred =
10449 CmpInst::isSigned(Pred) ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
10450 if (Step == MinusOne)
10451 NoOverflowPred = CmpInst::getSwappedPredicate(NoOverflowPred);
10452 const SCEV *Start = AR->getStart();
10453 if (!isKnownPredicateAt(NoOverflowPred, Start, Last, CtxI))
10454 return None;
10455
10456 // Everything is fine.
10457 return ScalarEvolution::LoopInvariantPredicate(Pred, Start, RHS);
10458}
10459
10460bool ScalarEvolution::isKnownPredicateViaConstantRanges(
10461 ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS) {
10462 if (HasSameValue(LHS, RHS))
10463 return ICmpInst::isTrueWhenEqual(Pred);
10464
10465 // This code is split out from isKnownPredicate because it is called from
10466 // within isLoopEntryGuardedByCond.
10467
10468 auto CheckRanges = [&](const ConstantRange &RangeLHS,
10469 const ConstantRange &RangeRHS) {
10470 return RangeLHS.icmp(Pred, RangeRHS);
10471 };
10472
10473 // The check at the top of the function catches the case where the values are
10474 // known to be equal.
10475 if (Pred == CmpInst::ICMP_EQ)
10476 return false;
10477
10478 if (Pred == CmpInst::ICMP_NE) {
10479 if (CheckRanges(getSignedRange(LHS), getSignedRange(RHS)) ||
10480 CheckRanges(getUnsignedRange(LHS), getUnsignedRange(RHS)))
10481 return true;
10482 auto *Diff = getMinusSCEV(LHS, RHS);
10483 return !isa<SCEVCouldNotCompute>(Diff) && isKnownNonZero(Diff);
10484 }
10485
10486 if (CmpInst::isSigned(Pred))
10487 return CheckRanges(getSignedRange(LHS), getSignedRange(RHS));
10488
10489 return CheckRanges(getUnsignedRange(LHS), getUnsignedRange(RHS));
10490}
10491
10492bool ScalarEvolution::isKnownPredicateViaNoOverflow(ICmpInst::Predicate Pred,
10493 const SCEV *LHS,
10494 const SCEV *RHS) {
10495 // Match X to (A + C1)<ExpectedFlags> and Y to (A + C2)<ExpectedFlags>, where
10496 // C1 and C2 are constant integers. If either X or Y are not add expressions,
10497 // consider them as X + 0 and Y + 0 respectively. C1 and C2 are returned via
10498 // OutC1 and OutC2.
10499 auto MatchBinaryAddToConst = [this](const SCEV *X, const SCEV *Y,
10500 APInt &OutC1, APInt &OutC2,
10501 SCEV::NoWrapFlags ExpectedFlags) {
10502 const SCEV *XNonConstOp, *XConstOp;
10503 const SCEV *YNonConstOp, *YConstOp;
10504 SCEV::NoWrapFlags XFlagsPresent;
10505 SCEV::NoWrapFlags YFlagsPresent;
10506
10507 if (!splitBinaryAdd(X, XConstOp, XNonConstOp, XFlagsPresent)) {
10508 XConstOp = getZero(X->getType());
10509 XNonConstOp = X;
10510 XFlagsPresent = ExpectedFlags;
10511 }
10512 if (!isa<SCEVConstant>(XConstOp) ||
10513 (XFlagsPresent & ExpectedFlags) != ExpectedFlags)
10514 return false;
10515
10516 if (!splitBinaryAdd(Y, YConstOp, YNonConstOp, YFlagsPresent)) {
10517 YConstOp = getZero(Y->getType());
10518 YNonConstOp = Y;
10519 YFlagsPresent = ExpectedFlags;
10520 }
10521
10522 if (!isa<SCEVConstant>(YConstOp) ||
10523 (YFlagsPresent & ExpectedFlags) != ExpectedFlags)
10524 return false;
10525
10526 if (YNonConstOp != XNonConstOp)
10527 return false;
10528
10529 OutC1 = cast<SCEVConstant>(XConstOp)->getAPInt();
10530 OutC2 = cast<SCEVConstant>(YConstOp)->getAPInt();
10531
10532 return true;
10533 };
10534
10535 APInt C1;
10536 APInt C2;
10537
10538 switch (Pred) {
10539 default:
10540 break;
10541
10542 case ICmpInst::ICMP_SGE:
10543 std::swap(LHS, RHS);
10544 LLVM_FALLTHROUGH[[gnu::fallthrough]];
10545 case ICmpInst::ICMP_SLE:
10546 // (X + C1)<nsw> s<= (X + C2)<nsw> if C1 s<= C2.
10547 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.sle(C2))
10548 return true;
10549
10550 break;
10551
10552 case ICmpInst::ICMP_SGT:
10553 std::swap(LHS, RHS);
10554 LLVM_FALLTHROUGH[[gnu::fallthrough]];
10555 case ICmpInst::ICMP_SLT:
10556 // (X + C1)<nsw> s< (X + C2)<nsw> if C1 s< C2.
10557 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.slt(C2))
10558 return true;
10559
10560 break;
10561
10562 case ICmpInst::ICMP_UGE:
10563 std::swap(LHS, RHS);
10564 LLVM_FALLTHROUGH[[gnu::fallthrough]];
10565 case ICmpInst::ICMP_ULE:
10566 // (X + C1)<nuw> u<= (X + C2)<nuw> for C1 u<= C2.
10567 if (MatchBinaryAddToConst(RHS, LHS, C2, C1, SCEV::FlagNUW) && C1.ule(C2))
10568 return true;
10569
10570 break;
10571
10572 case ICmpInst::ICMP_UGT:
10573 std::swap(LHS, RHS);
10574 LLVM_FALLTHROUGH[[gnu::fallthrough]];
10575 case ICmpInst::ICMP_ULT:
10576 // (X + C1)<nuw> u< (X + C2)<nuw> if C1 u< C2.
10577 if (MatchBinaryAddToConst(RHS, LHS, C2, C1, SCEV::FlagNUW) && C1.ult(C2))
10578 return true;
10579 break;
10580 }
10581
10582 return false;
10583}
10584
10585bool ScalarEvolution::isKnownPredicateViaSplitting(ICmpInst::Predicate Pred,
10586 const SCEV *LHS,
10587 const SCEV *RHS) {
10588 if (Pred != ICmpInst::ICMP_ULT || ProvingSplitPredicate)
10589 return false;
10590
10591 // Allowing arbitrary number of activations of isKnownPredicateViaSplitting on
10592 // the stack can result in exponential time complexity.
10593 SaveAndRestore<bool> Restore(ProvingSplitPredicate, true);
10594
10595 // If L >= 0 then I `ult` L <=> I >= 0 && I `slt` L
10596 //
10597 // To prove L >= 0 we use isKnownNonNegative whereas to prove I >= 0 we use
10598 // isKnownPredicate. isKnownPredicate is more powerful, but also more
10599 // expensive; and using isKnownNonNegative(RHS) is sufficient for most of the
10600 // interesting cases seen in practice. We can consider "upgrading" L >= 0 to
10601 // use isKnownPredicate later if needed.
10602 return isKnownNonNegative(RHS) &&
10603 isKnownPredicate(CmpInst::ICMP_SGE, LHS, getZero(LHS->getType())) &&
10604 isKnownPredicate(CmpInst::ICMP_SLT, LHS, RHS);
10605}
10606
10607bool ScalarEvolution::isImpliedViaGuard(const BasicBlock *BB,
10608 ICmpInst::Predicate Pred,
10609 const SCEV *LHS, const SCEV *RHS) {
10610 // No need to even try if we know the module has no guards.
10611 if (!HasGuards)
10612 return false;
10613
10614 return any_of(*BB, [&](const Instruction &I) {
10615 using namespace llvm::PatternMatch;
10616
10617 Value *Condition;
10618 return match(&I, m_Intrinsic<Intrinsic::experimental_guard>(
10619 m_Value(Condition))) &&
10620 isImpliedCond(Pred, LHS, RHS, Condition, false);
10621 });
10622}
10623
10624/// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
10625/// protected by a conditional between LHS and RHS. This is used to
10626/// to eliminate casts.
10627bool
10628ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
10629 ICmpInst::Predicate Pred,
10630 const SCEV *LHS, const SCEV *RHS) {
10631 // Interpret a null as meaning no loop, where there is obviously no guard
10632 // (interprocedural conditions notwithstanding).
10633 if (!L) return true;
10634
10635 if (VerifyIR)
10636 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()) &&(static_cast <bool> (!verifyFunction(*L->getHeader()
->getParent(), &dbgs()) && "This cannot be done on broken IR!"
) ? void (0) : __assert_fail ("!verifyFunction(*L->getHeader()->getParent(), &dbgs()) && \"This cannot be done on broken IR!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10637, __extension__
__PRETTY_FUNCTION__))
10637 "This cannot be done on broken IR!")(static_cast <bool> (!verifyFunction(*L->getHeader()
->getParent(), &dbgs()) && "This cannot be done on broken IR!"
) ? void (0) : __assert_fail ("!verifyFunction(*L->getHeader()->getParent(), &dbgs()) && \"This cannot be done on broken IR!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10637, __extension__
__PRETTY_FUNCTION__))
;
10638
10639
10640 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
10641 return true;
10642
10643 BasicBlock *Latch = L->getLoopLatch();
10644 if (!Latch)
10645 return false;
10646
10647 BranchInst *LoopContinuePredicate =
10648 dyn_cast<BranchInst>(Latch->getTerminator());
10649 if (LoopContinuePredicate && LoopContinuePredicate->isConditional() &&
10650 isImpliedCond(Pred, LHS, RHS,
10651 LoopContinuePredicate->getCondition(),
10652 LoopContinuePredicate->getSuccessor(0) != L->getHeader()))
10653 return true;
10654
10655 // We don't want more than one activation of the following loops on the stack
10656 // -- that can lead to O(n!) time complexity.
10657 if (WalkingBEDominatingConds)
10658 return false;
10659
10660 SaveAndRestore<bool> ClearOnExit(WalkingBEDominatingConds, true);
10661
10662 // See if we can exploit a trip count to prove the predicate.
10663 const auto &BETakenInfo = getBackedgeTakenInfo(L);
10664 const SCEV *LatchBECount = BETakenInfo.getExact(Latch, this);
10665 if (LatchBECount != getCouldNotCompute()) {
10666 // We know that Latch branches back to the loop header exactly
10667 // LatchBECount times. This means the backdege condition at Latch is
10668 // equivalent to "{0,+,1} u< LatchBECount".
10669 Type *Ty = LatchBECount->getType();
10670 auto NoWrapFlags = SCEV::NoWrapFlags(SCEV::FlagNUW | SCEV::FlagNW);
10671 const SCEV *LoopCounter =
10672 getAddRecExpr(getZero(Ty), getOne(Ty), L, NoWrapFlags);
10673 if (isImpliedCond(Pred, LHS, RHS, ICmpInst::ICMP_ULT, LoopCounter,
10674 LatchBECount))
10675 return true;
10676 }
10677
10678 // Check conditions due to any @llvm.assume intrinsics.
10679 for (auto &AssumeVH : AC.assumptions()) {
10680 if (!AssumeVH)
10681 continue;
10682 auto *CI = cast<CallInst>(AssumeVH);
10683 if (!DT.dominates(CI, Latch->getTerminator()))
10684 continue;
10685
10686 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
10687 return true;
10688 }
10689
10690 // If the loop is not reachable from the entry block, we risk running into an
10691 // infinite loop as we walk up into the dom tree. These loops do not matter
10692 // anyway, so we just return a conservative answer when we see them.
10693 if (!DT.isReachableFromEntry(L->getHeader()))
10694 return false;
10695
10696 if (isImpliedViaGuard(Latch, Pred, LHS, RHS))
10697 return true;
10698
10699 for (DomTreeNode *DTN = DT[Latch], *HeaderDTN = DT[L->getHeader()];
10700 DTN != HeaderDTN; DTN = DTN->getIDom()) {
10701 assert(DTN && "should reach the loop header before reaching the root!")(static_cast <bool> (DTN && "should reach the loop header before reaching the root!"
) ? void (0) : __assert_fail ("DTN && \"should reach the loop header before reaching the root!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10701, __extension__
__PRETTY_FUNCTION__))
;
10702
10703 BasicBlock *BB = DTN->getBlock();
10704 if (isImpliedViaGuard(BB, Pred, LHS, RHS))
10705 return true;
10706
10707 BasicBlock *PBB = BB->getSinglePredecessor();
10708 if (!PBB)
10709 continue;
10710
10711 BranchInst *ContinuePredicate = dyn_cast<BranchInst>(PBB->getTerminator());
10712 if (!ContinuePredicate || !ContinuePredicate->isConditional())
10713 continue;
10714
10715 Value *Condition = ContinuePredicate->getCondition();
10716
10717 // If we have an edge `E` within the loop body that dominates the only
10718 // latch, the condition guarding `E` also guards the backedge. This
10719 // reasoning works only for loops with a single latch.
10720
10721 BasicBlockEdge DominatingEdge(PBB, BB);
10722 if (DominatingEdge.isSingleEdge()) {
10723 // We're constructively (and conservatively) enumerating edges within the
10724 // loop body that dominate the latch. The dominator tree better agree
10725 // with us on this:
10726 assert(DT.dominates(DominatingEdge, Latch) && "should be!")(static_cast <bool> (DT.dominates(DominatingEdge, Latch
) && "should be!") ? void (0) : __assert_fail ("DT.dominates(DominatingEdge, Latch) && \"should be!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10726, __extension__
__PRETTY_FUNCTION__))
;
10727
10728 if (isImpliedCond(Pred, LHS, RHS, Condition,
10729 BB != ContinuePredicate->getSuccessor(0)))
10730 return true;
10731 }
10732 }
10733
10734 return false;
10735}
10736
10737bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB,
10738 ICmpInst::Predicate Pred,
10739 const SCEV *LHS,
10740 const SCEV *RHS) {
10741 if (VerifyIR)
10742 assert(!verifyFunction(*BB->getParent(), &dbgs()) &&(static_cast <bool> (!verifyFunction(*BB->getParent(
), &dbgs()) && "This cannot be done on broken IR!"
) ? void (0) : __assert_fail ("!verifyFunction(*BB->getParent(), &dbgs()) && \"This cannot be done on broken IR!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10743, __extension__
__PRETTY_FUNCTION__))
10743 "This cannot be done on broken IR!")(static_cast <bool> (!verifyFunction(*BB->getParent(
), &dbgs()) && "This cannot be done on broken IR!"
) ? void (0) : __assert_fail ("!verifyFunction(*BB->getParent(), &dbgs()) && \"This cannot be done on broken IR!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10743, __extension__
__PRETTY_FUNCTION__))
;
10744
10745 // If we cannot prove strict comparison (e.g. a > b), maybe we can prove
10746 // the facts (a >= b && a != b) separately. A typical situation is when the
10747 // non-strict comparison is known from ranges and non-equality is known from
10748 // dominating predicates. If we are proving strict comparison, we always try
10749 // to prove non-equality and non-strict comparison separately.
10750 auto NonStrictPredicate = ICmpInst::getNonStrictPredicate(Pred);
10751 const bool ProvingStrictComparison = (Pred != NonStrictPredicate);
10752 bool ProvedNonStrictComparison = false;
10753 bool ProvedNonEquality = false;
10754
10755 auto SplitAndProve =
10756 [&](std::function<bool(ICmpInst::Predicate)> Fn) -> bool {
10757 if (!ProvedNonStrictComparison)
10758 ProvedNonStrictComparison = Fn(NonStrictPredicate);
10759 if (!ProvedNonEquality)
10760 ProvedNonEquality = Fn(ICmpInst::ICMP_NE);
10761 if (ProvedNonStrictComparison && ProvedNonEquality)
10762 return true;
10763 return false;
10764 };
10765
10766 if (ProvingStrictComparison) {
10767 auto ProofFn = [&](ICmpInst::Predicate P) {
10768 return isKnownViaNonRecursiveReasoning(P, LHS, RHS);
10769 };
10770 if (SplitAndProve(ProofFn))
10771 return true;
10772 }
10773
10774 // Try to prove (Pred, LHS, RHS) using isImpliedViaGuard.
10775 auto ProveViaGuard = [&](const BasicBlock *Block) {
10776 if (isImpliedViaGuard(Block, Pred, LHS, RHS))
10777 return true;
10778 if (ProvingStrictComparison) {
10779 auto ProofFn = [&](ICmpInst::Predicate P) {
10780 return isImpliedViaGuard(Block, P, LHS, RHS);
10781 };
10782 if (SplitAndProve(ProofFn))
10783 return true;
10784 }
10785 return false;
10786 };
10787
10788 // Try to prove (Pred, LHS, RHS) using isImpliedCond.
10789 auto ProveViaCond = [&](const Value *Condition, bool Inverse) {
10790 const Instruction *CtxI = &BB->front();
10791 if (isImpliedCond(Pred, LHS, RHS, Condition, Inverse, CtxI))
10792 return true;
10793 if (ProvingStrictComparison) {
10794 auto ProofFn = [&](ICmpInst::Predicate P) {
10795 return isImpliedCond(P, LHS, RHS, Condition, Inverse, CtxI);
10796 };
10797 if (SplitAndProve(ProofFn))
10798 return true;
10799 }
10800 return false;
10801 };
10802
10803 // Starting at the block's predecessor, climb up the predecessor chain, as long
10804 // as there are predecessors that can be found that have unique successors
10805 // leading to the original block.
10806 const Loop *ContainingLoop = LI.getLoopFor(BB);
10807 const BasicBlock *PredBB;
10808 if (ContainingLoop && ContainingLoop->getHeader() == BB)
10809 PredBB = ContainingLoop->getLoopPredecessor();
10810 else
10811 PredBB = BB->getSinglePredecessor();
10812 for (std::pair<const BasicBlock *, const BasicBlock *> Pair(PredBB, BB);
10813 Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
10814 if (ProveViaGuard(Pair.first))
10815 return true;
10816
10817 const BranchInst *LoopEntryPredicate =
10818 dyn_cast<BranchInst>(Pair.first->getTerminator());
10819 if (!LoopEntryPredicate ||
10820 LoopEntryPredicate->isUnconditional())
10821 continue;
10822
10823 if (ProveViaCond(LoopEntryPredicate->getCondition(),
10824 LoopEntryPredicate->getSuccessor(0) != Pair.second))
10825 return true;
10826 }
10827
10828 // Check conditions due to any @llvm.assume intrinsics.
10829 for (auto &AssumeVH : AC.assumptions()) {
10830 if (!AssumeVH)
10831 continue;
10832 auto *CI = cast<CallInst>(AssumeVH);
10833 if (!DT.dominates(CI, BB))
10834 continue;
10835
10836 if (ProveViaCond(CI->getArgOperand(0), false))
10837 return true;
10838 }
10839
10840 return false;
10841}
10842
10843bool ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L,
10844 ICmpInst::Predicate Pred,
10845 const SCEV *LHS,
10846 const SCEV *RHS) {
10847 // Interpret a null as meaning no loop, where there is obviously no guard
10848 // (interprocedural conditions notwithstanding).
10849 if (!L)
10850 return false;
10851
10852 // Both LHS and RHS must be available at loop entry.
10853 assert(isAvailableAtLoopEntry(LHS, L) &&(static_cast <bool> (isAvailableAtLoopEntry(LHS, L) &&
"LHS is not available at Loop Entry") ? void (0) : __assert_fail
("isAvailableAtLoopEntry(LHS, L) && \"LHS is not available at Loop Entry\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10854, __extension__
__PRETTY_FUNCTION__))
10854 "LHS is not available at Loop Entry")(static_cast <bool> (isAvailableAtLoopEntry(LHS, L) &&
"LHS is not available at Loop Entry") ? void (0) : __assert_fail
("isAvailableAtLoopEntry(LHS, L) && \"LHS is not available at Loop Entry\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10854, __extension__
__PRETTY_FUNCTION__))
;
10855 assert(isAvailableAtLoopEntry(RHS, L) &&(static_cast <bool> (isAvailableAtLoopEntry(RHS, L) &&
"RHS is not available at Loop Entry") ? void (0) : __assert_fail
("isAvailableAtLoopEntry(RHS, L) && \"RHS is not available at Loop Entry\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10856, __extension__
__PRETTY_FUNCTION__))
10856 "RHS is not available at Loop Entry")(static_cast <bool> (isAvailableAtLoopEntry(RHS, L) &&
"RHS is not available at Loop Entry") ? void (0) : __assert_fail
("isAvailableAtLoopEntry(RHS, L) && \"RHS is not available at Loop Entry\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10856, __extension__
__PRETTY_FUNCTION__))
;
10857
10858 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
10859 return true;
10860
10861 return isBasicBlockEntryGuardedByCond(L->getHeader(), Pred, LHS, RHS);
10862}
10863
10864bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS,
10865 const SCEV *RHS,
10866 const Value *FoundCondValue, bool Inverse,
10867 const Instruction *CtxI) {
10868 // False conditions implies anything. Do not bother analyzing it further.
10869 if (FoundCondValue ==
10870 ConstantInt::getBool(FoundCondValue->getContext(), Inverse))
10871 return true;
10872
10873 if (!PendingLoopPredicates.insert(FoundCondValue).second)
10874 return false;
10875
10876 auto ClearOnExit =
10877 make_scope_exit([&]() { PendingLoopPredicates.erase(FoundCondValue); });
10878
10879 // Recursively handle And and Or conditions.
10880 const Value *Op0, *Op1;
10881 if (match(FoundCondValue, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) {
10882 if (!Inverse)
10883 return isImpliedCond(Pred, LHS, RHS, Op0, Inverse, CtxI) ||
10884 isImpliedCond(Pred, LHS, RHS, Op1, Inverse, CtxI);
10885 } else if (match(FoundCondValue, m_LogicalOr(m_Value(Op0), m_Value(Op1)))) {
10886 if (Inverse)
10887 return isImpliedCond(Pred, LHS, RHS, Op0, Inverse, CtxI) ||
10888 isImpliedCond(Pred, LHS, RHS, Op1, Inverse, CtxI);
10889 }
10890
10891 const ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue);
10892 if (!ICI) return false;
10893
10894 // Now that we found a conditional branch that dominates the loop or controls
10895 // the loop latch. Check to see if it is the comparison we are looking for.
10896 ICmpInst::Predicate FoundPred;
10897 if (Inverse)
10898 FoundPred = ICI->getInversePredicate();
10899 else
10900 FoundPred = ICI->getPredicate();
10901
10902 const SCEV *FoundLHS = getSCEV(ICI->getOperand(0));
10903 const SCEV *FoundRHS = getSCEV(ICI->getOperand(1));
10904
10905 return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS, CtxI);
10906}
10907
10908bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS,
10909 const SCEV *RHS,
10910 ICmpInst::Predicate FoundPred,
10911 const SCEV *FoundLHS, const SCEV *FoundRHS,
10912 const Instruction *CtxI) {
10913 // Balance the types.
10914 if (getTypeSizeInBits(LHS->getType()) <
10915 getTypeSizeInBits(FoundLHS->getType())) {
10916 // For unsigned and equality predicates, try to prove that both found
10917 // operands fit into narrow unsigned range. If so, try to prove facts in
10918 // narrow types.
10919 if (!CmpInst::isSigned(FoundPred) && !FoundLHS->getType()->isPointerTy()) {
10920 auto *NarrowType = LHS->getType();
10921 auto *WideType = FoundLHS->getType();
10922 auto BitWidth = getTypeSizeInBits(NarrowType);
10923 const SCEV *MaxValue = getZeroExtendExpr(
10924 getConstant(APInt::getMaxValue(BitWidth)), WideType);
10925 if (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_ULE, FoundLHS,
10926 MaxValue) &&
10927 isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_ULE, FoundRHS,
10928 MaxValue)) {
10929 const SCEV *TruncFoundLHS = getTruncateExpr(FoundLHS, NarrowType);
10930 const SCEV *TruncFoundRHS = getTruncateExpr(FoundRHS, NarrowType);
10931 if (isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred, TruncFoundLHS,
10932 TruncFoundRHS, CtxI))
10933 return true;
10934 }
10935 }
10936
10937 if (LHS->getType()->isPointerTy())
10938 return false;
10939 if (CmpInst::isSigned(Pred)) {
10940 LHS = getSignExtendExpr(LHS, FoundLHS->getType());
10941 RHS = getSignExtendExpr(RHS, FoundLHS->getType());
10942 } else {
10943 LHS = getZeroExtendExpr(LHS, FoundLHS->getType());
10944 RHS = getZeroExtendExpr(RHS, FoundLHS->getType());
10945 }
10946 } else if (getTypeSizeInBits(LHS->getType()) >
10947 getTypeSizeInBits(FoundLHS->getType())) {
10948 if (FoundLHS->getType()->isPointerTy())
10949 return false;
10950 if (CmpInst::isSigned(FoundPred)) {
10951 FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType());
10952 FoundRHS = getSignExtendExpr(FoundRHS, LHS->getType());
10953 } else {
10954 FoundLHS = getZeroExtendExpr(FoundLHS, LHS->getType());
10955 FoundRHS = getZeroExtendExpr(FoundRHS, LHS->getType());
10956 }
10957 }
10958 return isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred, FoundLHS,
10959 FoundRHS, CtxI);
10960}
10961
10962bool ScalarEvolution::isImpliedCondBalancedTypes(
10963 ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS,
10964 ICmpInst::Predicate FoundPred, const SCEV *FoundLHS, const SCEV *FoundRHS,
10965 const Instruction *CtxI) {
10966 assert(getTypeSizeInBits(LHS->getType()) ==(static_cast <bool> (getTypeSizeInBits(LHS->getType(
)) == getTypeSizeInBits(FoundLHS->getType()) && "Types should be balanced!"
) ? void (0) : __assert_fail ("getTypeSizeInBits(LHS->getType()) == getTypeSizeInBits(FoundLHS->getType()) && \"Types should be balanced!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10968, __extension__
__PRETTY_FUNCTION__))
10967 getTypeSizeInBits(FoundLHS->getType()) &&(static_cast <bool> (getTypeSizeInBits(LHS->getType(
)) == getTypeSizeInBits(FoundLHS->getType()) && "Types should be balanced!"
) ? void (0) : __assert_fail ("getTypeSizeInBits(LHS->getType()) == getTypeSizeInBits(FoundLHS->getType()) && \"Types should be balanced!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10968, __extension__
__PRETTY_FUNCTION__))
10968 "Types should be balanced!")(static_cast <bool> (getTypeSizeInBits(LHS->getType(
)) == getTypeSizeInBits(FoundLHS->getType()) && "Types should be balanced!"
) ? void (0) : __assert_fail ("getTypeSizeInBits(LHS->getType()) == getTypeSizeInBits(FoundLHS->getType()) && \"Types should be balanced!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 10968, __extension__
__PRETTY_FUNCTION__))
;
10969 // Canonicalize the query to match the way instcombine will have
10970 // canonicalized the comparison.
10971 if (SimplifyICmpOperands(Pred, LHS, RHS))
10972 if (LHS == RHS)
10973 return CmpInst::isTrueWhenEqual(Pred);
10974 if (SimplifyICmpOperands(FoundPred, FoundLHS, FoundRHS))
10975 if (FoundLHS == FoundRHS)
10976 return CmpInst::isFalseWhenEqual(FoundPred);
10977
10978 // Check to see if we can make the LHS or RHS match.
10979 if (LHS == FoundRHS || RHS == FoundLHS) {
10980 if (isa<SCEVConstant>(RHS)) {
10981 std::swap(FoundLHS, FoundRHS);
10982 FoundPred = ICmpInst::getSwappedPredicate(FoundPred);
10983 } else {
10984 std::swap(LHS, RHS);
10985 Pred = ICmpInst::getSwappedPredicate(Pred);
10986 }
10987 }
10988
10989 // Check whether the found predicate is the same as the desired predicate.
10990 if (FoundPred == Pred)
10991 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, CtxI);
10992
10993 // Check whether swapping the found predicate makes it the same as the
10994 // desired predicate.
10995 if (ICmpInst::getSwappedPredicate(FoundPred) == Pred) {
10996 // We can write the implication
10997 // 0. LHS Pred RHS <- FoundLHS SwapPred FoundRHS
10998 // using one of the following ways:
10999 // 1. LHS Pred RHS <- FoundRHS Pred FoundLHS
11000 // 2. RHS SwapPred LHS <- FoundLHS SwapPred FoundRHS
11001 // 3. LHS Pred RHS <- ~FoundLHS Pred ~FoundRHS
11002 // 4. ~LHS SwapPred ~RHS <- FoundLHS SwapPred FoundRHS
11003 // Forms 1. and 2. require swapping the operands of one condition. Don't
11004 // do this if it would break canonical constant/addrec ordering.
11005 if (!isa<SCEVConstant>(RHS) && !isa<SCEVAddRecExpr>(LHS))
11006 return isImpliedCondOperands(FoundPred, RHS, LHS, FoundLHS, FoundRHS,
11007 CtxI);
11008 if (!isa<SCEVConstant>(FoundRHS) && !isa<SCEVAddRecExpr>(FoundLHS))
11009 return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS, CtxI);
11010
11011 // There's no clear preference between forms 3. and 4., try both. Avoid
11012 // forming getNotSCEV of pointer values as the resulting subtract is
11013 // not legal.
11014 if (!LHS->getType()->isPointerTy() && !RHS->getType()->isPointerTy() &&
11015 isImpliedCondOperands(FoundPred, getNotSCEV(LHS), getNotSCEV(RHS),
11016 FoundLHS, FoundRHS, CtxI))
11017 return true;
11018
11019 if (!FoundLHS->getType()->isPointerTy() &&
11020 !FoundRHS->getType()->isPointerTy() &&
11021 isImpliedCondOperands(Pred, LHS, RHS, getNotSCEV(FoundLHS),
11022 getNotSCEV(FoundRHS), CtxI))
11023 return true;
11024
11025 return false;
11026 }
11027
11028 auto IsSignFlippedPredicate = [](CmpInst::Predicate P1,
11029 CmpInst::Predicate P2) {
11030 assert(P1 != P2 && "Handled earlier!")(static_cast <bool> (P1 != P2 && "Handled earlier!"
) ? void (0) : __assert_fail ("P1 != P2 && \"Handled earlier!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11030, __extension__
__PRETTY_FUNCTION__))
;
11031 return CmpInst::isRelational(P2) &&
11032 P1 == CmpInst::getFlippedSignednessPredicate(P2);
11033 };
11034 if (IsSignFlippedPredicate(Pred, FoundPred)) {
11035 // Unsigned comparison is the same as signed comparison when both the
11036 // operands are non-negative or negative.
11037 if ((isKnownNonNegative(FoundLHS) && isKnownNonNegative(FoundRHS)) ||
11038 (isKnownNegative(FoundLHS) && isKnownNegative(FoundRHS)))
11039 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, CtxI);
11040 // Create local copies that we can freely swap and canonicalize our
11041 // conditions to "le/lt".
11042 ICmpInst::Predicate CanonicalPred = Pred, CanonicalFoundPred = FoundPred;
11043 const SCEV *CanonicalLHS = LHS, *CanonicalRHS = RHS,
11044 *CanonicalFoundLHS = FoundLHS, *CanonicalFoundRHS = FoundRHS;
11045 if (ICmpInst::isGT(CanonicalPred) || ICmpInst::isGE(CanonicalPred)) {
11046 CanonicalPred = ICmpInst::getSwappedPredicate(CanonicalPred);
11047 CanonicalFoundPred = ICmpInst::getSwappedPredicate(CanonicalFoundPred);
11048 std::swap(CanonicalLHS, CanonicalRHS);
11049 std::swap(CanonicalFoundLHS, CanonicalFoundRHS);
11050 }
11051 assert((ICmpInst::isLT(CanonicalPred) || ICmpInst::isLE(CanonicalPred)) &&(static_cast <bool> ((ICmpInst::isLT(CanonicalPred) || ICmpInst
::isLE(CanonicalPred)) && "Must be!") ? void (0) : __assert_fail
("(ICmpInst::isLT(CanonicalPred) || ICmpInst::isLE(CanonicalPred)) && \"Must be!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11052, __extension__
__PRETTY_FUNCTION__))
11052 "Must be!")(static_cast <bool> ((ICmpInst::isLT(CanonicalPred) || ICmpInst
::isLE(CanonicalPred)) && "Must be!") ? void (0) : __assert_fail
("(ICmpInst::isLT(CanonicalPred) || ICmpInst::isLE(CanonicalPred)) && \"Must be!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11052, __extension__
__PRETTY_FUNCTION__))
;
11053 assert((ICmpInst::isLT(CanonicalFoundPred) ||(static_cast <bool> ((ICmpInst::isLT(CanonicalFoundPred
) || ICmpInst::isLE(CanonicalFoundPred)) && "Must be!"
) ? void (0) : __assert_fail ("(ICmpInst::isLT(CanonicalFoundPred) || ICmpInst::isLE(CanonicalFoundPred)) && \"Must be!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11055, __extension__
__PRETTY_FUNCTION__))
11054 ICmpInst::isLE(CanonicalFoundPred)) &&(static_cast <bool> ((ICmpInst::isLT(CanonicalFoundPred
) || ICmpInst::isLE(CanonicalFoundPred)) && "Must be!"
) ? void (0) : __assert_fail ("(ICmpInst::isLT(CanonicalFoundPred) || ICmpInst::isLE(CanonicalFoundPred)) && \"Must be!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11055, __extension__
__PRETTY_FUNCTION__))
11055 "Must be!")(static_cast <bool> ((ICmpInst::isLT(CanonicalFoundPred
) || ICmpInst::isLE(CanonicalFoundPred)) && "Must be!"
) ? void (0) : __assert_fail ("(ICmpInst::isLT(CanonicalFoundPred) || ICmpInst::isLE(CanonicalFoundPred)) && \"Must be!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11055, __extension__
__PRETTY_FUNCTION__))
;
11056 if (ICmpInst::isSigned(CanonicalPred) && isKnownNonNegative(CanonicalRHS))
11057 // Use implication:
11058 // x <u y && y >=s 0 --> x <s y.
11059 // If we can prove the left part, the right part is also proven.
11060 return isImpliedCondOperands(CanonicalFoundPred, CanonicalLHS,
11061 CanonicalRHS, CanonicalFoundLHS,
11062 CanonicalFoundRHS);
11063 if (ICmpInst::isUnsigned(CanonicalPred) && isKnownNegative(CanonicalRHS))
11064 // Use implication:
11065 // x <s y && y <s 0 --> x <u y.
11066 // If we can prove the left part, the right part is also proven.
11067 return isImpliedCondOperands(CanonicalFoundPred, CanonicalLHS,
11068 CanonicalRHS, CanonicalFoundLHS,
11069 CanonicalFoundRHS);
11070 }
11071
11072 // Check if we can make progress by sharpening ranges.
11073 if (FoundPred == ICmpInst::ICMP_NE &&
11074 (isa<SCEVConstant>(FoundLHS) || isa<SCEVConstant>(FoundRHS))) {
11075
11076 const SCEVConstant *C = nullptr;
11077 const SCEV *V = nullptr;
11078
11079 if (isa<SCEVConstant>(FoundLHS)) {
11080 C = cast<SCEVConstant>(FoundLHS);
11081 V = FoundRHS;
11082 } else {
11083 C = cast<SCEVConstant>(FoundRHS);
11084 V = FoundLHS;
11085 }
11086
11087 // The guarding predicate tells us that C != V. If the known range
11088 // of V is [C, t), we can sharpen the range to [C + 1, t). The
11089 // range we consider has to correspond to same signedness as the
11090 // predicate we're interested in folding.
11091
11092 APInt Min = ICmpInst::isSigned(Pred) ?
11093 getSignedRangeMin(V) : getUnsignedRangeMin(V);
11094
11095 if (Min == C->getAPInt()) {
11096 // Given (V >= Min && V != Min) we conclude V >= (Min + 1).
11097 // This is true even if (Min + 1) wraps around -- in case of
11098 // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)).
11099
11100 APInt SharperMin = Min + 1;
11101
11102 switch (Pred) {
11103 case ICmpInst::ICMP_SGE:
11104 case ICmpInst::ICMP_UGE:
11105 // We know V `Pred` SharperMin. If this implies LHS `Pred`
11106 // RHS, we're done.
11107 if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(SharperMin),
11108 CtxI))
11109 return true;
11110 LLVM_FALLTHROUGH[[gnu::fallthrough]];
11111
11112 case ICmpInst::ICMP_SGT:
11113 case ICmpInst::ICMP_UGT:
11114 // We know from the range information that (V `Pred` Min ||
11115 // V == Min). We know from the guarding condition that !(V
11116 // == Min). This gives us
11117 //
11118 // V `Pred` Min || V == Min && !(V == Min)
11119 // => V `Pred` Min
11120 //
11121 // If V `Pred` Min implies LHS `Pred` RHS, we're done.
11122
11123 if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(Min), CtxI))
11124 return true;
11125 break;
11126
11127 // `LHS < RHS` and `LHS <= RHS` are handled in the same way as `RHS > LHS` and `RHS >= LHS` respectively.
11128 case ICmpInst::ICMP_SLE:
11129 case ICmpInst::ICMP_ULE:
11130 if (isImpliedCondOperands(CmpInst::getSwappedPredicate(Pred), RHS,
11131 LHS, V, getConstant(SharperMin), CtxI))
11132 return true;
11133 LLVM_FALLTHROUGH[[gnu::fallthrough]];
11134
11135 case ICmpInst::ICMP_SLT:
11136 case ICmpInst::ICMP_ULT:
11137 if (isImpliedCondOperands(CmpInst::getSwappedPredicate(Pred), RHS,
11138 LHS, V, getConstant(Min), CtxI))
11139 return true;
11140 break;
11141
11142 default:
11143 // No change
11144 break;
11145 }
11146 }
11147 }
11148
11149 // Check whether the actual condition is beyond sufficient.
11150 if (FoundPred == ICmpInst::ICMP_EQ)
11151 if (ICmpInst::isTrueWhenEqual(Pred))
11152 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, CtxI))
11153 return true;
11154 if (Pred == ICmpInst::ICMP_NE)
11155 if (!ICmpInst::isTrueWhenEqual(FoundPred))
11156 if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS, CtxI))
11157 return true;
11158
11159 // Otherwise assume the worst.
11160 return false;
11161}
11162
11163bool ScalarEvolution::splitBinaryAdd(const SCEV *Expr,
11164 const SCEV *&L, const SCEV *&R,
11165 SCEV::NoWrapFlags &Flags) {
11166 const auto *AE = dyn_cast<SCEVAddExpr>(Expr);
11167 if (!AE || AE->getNumOperands() != 2)
11168 return false;
11169
11170 L = AE->getOperand(0);
11171 R = AE->getOperand(1);
11172 Flags = AE->getNoWrapFlags();
11173 return true;
11174}
11175
11176Optional<APInt> ScalarEvolution::computeConstantDifference(const SCEV *More,
11177 const SCEV *Less) {
11178 // We avoid subtracting expressions here because this function is usually
11179 // fairly deep in the call stack (i.e. is called many times).
11180
11181 // X - X = 0.
11182 if (More == Less)
11183 return APInt(getTypeSizeInBits(More->getType()), 0);
11184
11185 if (isa<SCEVAddRecExpr>(Less) && isa<SCEVAddRecExpr>(More)) {
11186 const auto *LAR = cast<SCEVAddRecExpr>(Less);
11187 const auto *MAR = cast<SCEVAddRecExpr>(More);
11188
11189 if (LAR->getLoop() != MAR->getLoop())
11190 return None;
11191
11192 // We look at affine expressions only; not for correctness but to keep
11193 // getStepRecurrence cheap.
11194 if (!LAR->isAffine() || !MAR->isAffine())
11195 return None;
11196
11197 if (LAR->getStepRecurrence(*this) != MAR->getStepRecurrence(*this))
11198 return None;
11199
11200 Less = LAR->getStart();
11201 More = MAR->getStart();
11202
11203 // fall through
11204 }
11205
11206 if (isa<SCEVConstant>(Less) && isa<SCEVConstant>(More)) {
11207 const auto &M = cast<SCEVConstant>(More)->getAPInt();
11208 const auto &L = cast<SCEVConstant>(Less)->getAPInt();
11209 return M - L;
11210 }
11211
11212 SCEV::NoWrapFlags Flags;
11213 const SCEV *LLess = nullptr, *RLess = nullptr;
11214 const SCEV *LMore = nullptr, *RMore = nullptr;
11215 const SCEVConstant *C1 = nullptr, *C2 = nullptr;
11216 // Compare (X + C1) vs X.
11217 if (splitBinaryAdd(Less, LLess, RLess, Flags))
11218 if ((C1 = dyn_cast<SCEVConstant>(LLess)))
11219 if (RLess == More)
11220 return -(C1->getAPInt());
11221
11222 // Compare X vs (X + C2).
11223 if (splitBinaryAdd(More, LMore, RMore, Flags))
11224 if ((C2 = dyn_cast<SCEVConstant>(LMore)))
11225 if (RMore == Less)
11226 return C2->getAPInt();
11227
11228 // Compare (X + C1) vs (X + C2).
11229 if (C1 && C2 && RLess == RMore)
11230 return C2->getAPInt() - C1->getAPInt();
11231
11232 return None;
11233}
11234
11235bool ScalarEvolution::isImpliedCondOperandsViaAddRecStart(
11236 ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS,
11237 const SCEV *FoundLHS, const SCEV *FoundRHS, const Instruction *CtxI) {
11238 // Try to recognize the following pattern:
11239 //
11240 // FoundRHS = ...
11241 // ...
11242 // loop:
11243 // FoundLHS = {Start,+,W}
11244 // context_bb: // Basic block from the same loop
11245 // known(Pred, FoundLHS, FoundRHS)
11246 //
11247 // If some predicate is known in the context of a loop, it is also known on
11248 // each iteration of this loop, including the first iteration. Therefore, in
11249 // this case, `FoundLHS Pred FoundRHS` implies `Start Pred FoundRHS`. Try to
11250 // prove the original pred using this fact.
11251 if (!CtxI)
11252 return false;
11253 const BasicBlock *ContextBB = CtxI->getParent();
11254 // Make sure AR varies in the context block.
11255 if (auto *AR = dyn_cast<SCEVAddRecExpr>(FoundLHS)) {
11256 const Loop *L = AR->getLoop();
11257 // Make sure that context belongs to the loop and executes on 1st iteration
11258 // (if it ever executes at all).
11259 if (!L->contains(ContextBB) || !DT.dominates(ContextBB, L->getLoopLatch()))
11260 return false;
11261 if (!isAvailableAtLoopEntry(FoundRHS, AR->getLoop()))
11262 return false;
11263 return isImpliedCondOperands(Pred, LHS, RHS, AR->getStart(), FoundRHS);
11264 }
11265
11266 if (auto *AR = dyn_cast<SCEVAddRecExpr>(FoundRHS)) {
11267 const Loop *L = AR->getLoop();
11268 // Make sure that context belongs to the loop and executes on 1st iteration
11269 // (if it ever executes at all).
11270 if (!L->contains(ContextBB) || !DT.dominates(ContextBB, L->getLoopLatch()))
11271 return false;
11272 if (!isAvailableAtLoopEntry(FoundLHS, AR->getLoop()))
11273 return false;
11274 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, AR->getStart());
11275 }
11276
11277 return false;
11278}
11279
11280bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow(
11281 ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS,
11282 const SCEV *FoundLHS, const SCEV *FoundRHS) {
11283 if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT)
11284 return false;
11285
11286 const auto *AddRecLHS = dyn_cast<SCEVAddRecExpr>(LHS);
11287 if (!AddRecLHS)
11288 return false;
11289
11290 const auto *AddRecFoundLHS = dyn_cast<SCEVAddRecExpr>(FoundLHS);
11291 if (!AddRecFoundLHS)
11292 return false;
11293
11294 // We'd like to let SCEV reason about control dependencies, so we constrain
11295 // both the inequalities to be about add recurrences on the same loop. This
11296 // way we can use isLoopEntryGuardedByCond later.
11297
11298 const Loop *L = AddRecFoundLHS->getLoop();
11299 if (L != AddRecLHS->getLoop())
11300 return false;
11301
11302 // FoundLHS u< FoundRHS u< -C => (FoundLHS + C) u< (FoundRHS + C) ... (1)
11303 //
11304 // FoundLHS s< FoundRHS s< INT_MIN - C => (FoundLHS + C) s< (FoundRHS + C)
11305 // ... (2)
11306 //
11307 // Informal proof for (2), assuming (1) [*]:
11308 //
11309 // We'll also assume (A s< B) <=> ((A + INT_MIN) u< (B + INT_MIN)) ... (3)[**]
11310 //
11311 // Then
11312 //
11313 // FoundLHS s< FoundRHS s< INT_MIN - C
11314 // <=> (FoundLHS + INT_MIN) u< (FoundRHS + INT_MIN) u< -C [ using (3) ]
11315 // <=> (FoundLHS + INT_MIN + C) u< (FoundRHS + INT_MIN + C) [ using (1) ]
11316 // <=> (FoundLHS + INT_MIN + C + INT_MIN) s<
11317 // (FoundRHS + INT_MIN + C + INT_MIN) [ using (3) ]
11318 // <=> FoundLHS + C s< FoundRHS + C
11319 //
11320 // [*]: (1) can be proved by ruling out overflow.
11321 //
11322 // [**]: This can be proved by analyzing all the four possibilities:
11323 // (A s< 0, B s< 0), (A s< 0, B s>= 0), (A s>= 0, B s< 0) and
11324 // (A s>= 0, B s>= 0).
11325 //
11326 // Note:
11327 // Despite (2), "FoundRHS s< INT_MIN - C" does not mean that "FoundRHS + C"
11328 // will not sign underflow. For instance, say FoundLHS = (i8 -128), FoundRHS
11329 // = (i8 -127) and C = (i8 -100). Then INT_MIN - C = (i8 -28), and FoundRHS
11330 // s< (INT_MIN - C). Lack of sign overflow / underflow in "FoundRHS + C" is
11331 // neither necessary nor sufficient to prove "(FoundLHS + C) s< (FoundRHS +
11332 // C)".
11333
11334 Optional<APInt> LDiff = computeConstantDifference(LHS, FoundLHS);
11335 Optional<APInt> RDiff = computeConstantDifference(RHS, FoundRHS);
11336 if (!LDiff || !RDiff || *LDiff != *RDiff)
11337 return false;
11338
11339 if (LDiff->isMinValue())
11340 return true;
11341
11342 APInt FoundRHSLimit;
11343
11344 if (Pred == CmpInst::ICMP_ULT) {
11345 FoundRHSLimit = -(*RDiff);
11346 } else {
11347 assert(Pred == CmpInst::ICMP_SLT && "Checked above!")(static_cast <bool> (Pred == CmpInst::ICMP_SLT &&
"Checked above!") ? void (0) : __assert_fail ("Pred == CmpInst::ICMP_SLT && \"Checked above!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11347, __extension__
__PRETTY_FUNCTION__))
;
11348 FoundRHSLimit = APInt::getSignedMinValue(getTypeSizeInBits(RHS->getType())) - *RDiff;
11349 }
11350
11351 // Try to prove (1) or (2), as needed.
11352 return isAvailableAtLoopEntry(FoundRHS, L) &&
11353 isLoopEntryGuardedByCond(L, Pred, FoundRHS,
11354 getConstant(FoundRHSLimit));
11355}
11356
11357bool ScalarEvolution::isImpliedViaMerge(ICmpInst::Predicate Pred,
11358 const SCEV *LHS, const SCEV *RHS,
11359 const SCEV *FoundLHS,
11360 const SCEV *FoundRHS, unsigned Depth) {
11361 const PHINode *LPhi = nullptr, *RPhi = nullptr;
11362
11363 auto ClearOnExit = make_scope_exit([&]() {
11364 if (LPhi) {
11365 bool Erased = PendingMerges.erase(LPhi);
11366 assert(Erased && "Failed to erase LPhi!")(static_cast <bool> (Erased && "Failed to erase LPhi!"
) ? void (0) : __assert_fail ("Erased && \"Failed to erase LPhi!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11366, __extension__
__PRETTY_FUNCTION__))
;
11367 (void)Erased;
11368 }
11369 if (RPhi) {
11370 bool Erased = PendingMerges.erase(RPhi);
11371 assert(Erased && "Failed to erase RPhi!")(static_cast <bool> (Erased && "Failed to erase RPhi!"
) ? void (0) : __assert_fail ("Erased && \"Failed to erase RPhi!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11371, __extension__
__PRETTY_FUNCTION__))
;
11372 (void)Erased;
11373 }
11374 });
11375
11376 // Find respective Phis and check that they are not being pending.
11377 if (const SCEVUnknown *LU = dyn_cast<SCEVUnknown>(LHS))
11378 if (auto *Phi = dyn_cast<PHINode>(LU->getValue())) {
11379 if (!PendingMerges.insert(Phi).second)
11380 return false;
11381 LPhi = Phi;
11382 }
11383 if (const SCEVUnknown *RU = dyn_cast<SCEVUnknown>(RHS))
11384 if (auto *Phi = dyn_cast<PHINode>(RU->getValue())) {
11385 // If we detect a loop of Phi nodes being processed by this method, for
11386 // example:
11387 //
11388 // %a = phi i32 [ %some1, %preheader ], [ %b, %latch ]
11389 // %b = phi i32 [ %some2, %preheader ], [ %a, %latch ]
11390 //
11391 // we don't want to deal with a case that complex, so return conservative
11392 // answer false.
11393 if (!PendingMerges.insert(Phi).second)
11394 return false;
11395 RPhi = Phi;
11396 }
11397
11398 // If none of LHS, RHS is a Phi, nothing to do here.
11399 if (!LPhi && !RPhi)
11400 return false;
11401
11402 // If there is a SCEVUnknown Phi we are interested in, make it left.
11403 if (!LPhi) {
11404 std::swap(LHS, RHS);
11405 std::swap(FoundLHS, FoundRHS);
11406 std::swap(LPhi, RPhi);
11407 Pred = ICmpInst::getSwappedPredicate(Pred);
11408 }
11409
11410 assert(LPhi && "LPhi should definitely be a SCEVUnknown Phi!")(static_cast <bool> (LPhi && "LPhi should definitely be a SCEVUnknown Phi!"
) ? void (0) : __assert_fail ("LPhi && \"LPhi should definitely be a SCEVUnknown Phi!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11410, __extension__
__PRETTY_FUNCTION__))
;
11411 const BasicBlock *LBB = LPhi->getParent();
11412 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(RHS);
11413
11414 auto ProvedEasily = [&](const SCEV *S1, const SCEV *S2) {
11415 return isKnownViaNonRecursiveReasoning(Pred, S1, S2) ||
11416 isImpliedCondOperandsViaRanges(Pred, S1, S2, FoundLHS, FoundRHS) ||
11417 isImpliedViaOperations(Pred, S1, S2, FoundLHS, FoundRHS, Depth);
11418 };
11419
11420 if (RPhi && RPhi->getParent() == LBB) {
11421 // Case one: RHS is also a SCEVUnknown Phi from the same basic block.
11422 // If we compare two Phis from the same block, and for each entry block
11423 // the predicate is true for incoming values from this block, then the
11424 // predicate is also true for the Phis.
11425 for (const BasicBlock *IncBB : predecessors(LBB)) {
11426 const SCEV *L = getSCEV(LPhi->getIncomingValueForBlock(IncBB));
11427 const SCEV *R = getSCEV(RPhi->getIncomingValueForBlock(IncBB));
11428 if (!ProvedEasily(L, R))
11429 return false;
11430 }
11431 } else if (RAR && RAR->getLoop()->getHeader() == LBB) {
11432 // Case two: RHS is also a Phi from the same basic block, and it is an
11433 // AddRec. It means that there is a loop which has both AddRec and Unknown
11434 // PHIs, for it we can compare incoming values of AddRec from above the loop
11435 // and latch with their respective incoming values of LPhi.
11436 // TODO: Generalize to handle loops with many inputs in a header.
11437 if (LPhi->getNumIncomingValues() != 2) return false;
11438
11439 auto *RLoop = RAR->getLoop();
11440 auto *Predecessor = RLoop->getLoopPredecessor();
11441 assert(Predecessor && "Loop with AddRec with no predecessor?")(static_cast <bool> (Predecessor && "Loop with AddRec with no predecessor?"
) ? void (0) : __assert_fail ("Predecessor && \"Loop with AddRec with no predecessor?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11441, __extension__
__PRETTY_FUNCTION__))
;
11442 const SCEV *L1 = getSCEV(LPhi->getIncomingValueForBlock(Predecessor));
11443 if (!ProvedEasily(L1, RAR->getStart()))
11444 return false;
11445 auto *Latch = RLoop->getLoopLatch();
11446 assert(Latch && "Loop with AddRec with no latch?")(static_cast <bool> (Latch && "Loop with AddRec with no latch?"
) ? void (0) : __assert_fail ("Latch && \"Loop with AddRec with no latch?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11446, __extension__
__PRETTY_FUNCTION__))
;
11447 const SCEV *L2 = getSCEV(LPhi->getIncomingValueForBlock(Latch));
11448 if (!ProvedEasily(L2, RAR->getPostIncExpr(*this)))
11449 return false;
11450 } else {
11451 // In all other cases go over inputs of LHS and compare each of them to RHS,
11452 // the predicate is true for (LHS, RHS) if it is true for all such pairs.
11453 // At this point RHS is either a non-Phi, or it is a Phi from some block
11454 // different from LBB.
11455 for (const BasicBlock *IncBB : predecessors(LBB)) {
11456 // Check that RHS is available in this block.
11457 if (!dominates(RHS, IncBB))
11458 return false;
11459 const SCEV *L = getSCEV(LPhi->getIncomingValueForBlock(IncBB));
11460 // Make sure L does not refer to a value from a potentially previous
11461 // iteration of a loop.
11462 if (!properlyDominates(L, IncBB))
11463 return false;
11464 if (!ProvedEasily(L, RHS))
11465 return false;
11466 }
11467 }
11468 return true;
11469}
11470
11471bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred,
11472 const SCEV *LHS, const SCEV *RHS,
11473 const SCEV *FoundLHS,
11474 const SCEV *FoundRHS,
11475 const Instruction *CtxI) {
11476 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundLHS, FoundRHS))
11477 return true;
11478
11479 if (isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS, FoundRHS))
11480 return true;
11481
11482 if (isImpliedCondOperandsViaAddRecStart(Pred, LHS, RHS, FoundLHS, FoundRHS,
11483 CtxI))
11484 return true;
11485
11486 return isImpliedCondOperandsHelper(Pred, LHS, RHS,
11487 FoundLHS, FoundRHS);
11488}
11489
11490/// Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
11491template <typename MinMaxExprType>
11492static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr,
11493 const SCEV *Candidate) {
11494 const MinMaxExprType *MinMaxExpr = dyn_cast<MinMaxExprType>(MaybeMinMaxExpr);
11495 if (!MinMaxExpr)
11496 return false;
11497
11498 return is_contained(MinMaxExpr->operands(), Candidate);
11499}
11500
11501static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE,
11502 ICmpInst::Predicate Pred,
11503 const SCEV *LHS, const SCEV *RHS) {
11504 // If both sides are affine addrecs for the same loop, with equal
11505 // steps, and we know the recurrences don't wrap, then we only
11506 // need to check the predicate on the starting values.
11507
11508 if (!ICmpInst::isRelational(Pred))
11509 return false;
11510
11511 const SCEVAddRecExpr *LAR = dyn_cast<SCEVAddRecExpr>(LHS);
11512 if (!LAR)
11513 return false;
11514 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(RHS);
11515 if (!RAR)
11516 return false;
11517 if (LAR->getLoop() != RAR->getLoop())
11518 return false;
11519 if (!LAR->isAffine() || !RAR->isAffine())
11520 return false;
11521
11522 if (LAR->getStepRecurrence(SE) != RAR->getStepRecurrence(SE))
11523 return false;
11524
11525 SCEV::NoWrapFlags NW = ICmpInst::isSigned(Pred) ?
11526 SCEV::FlagNSW : SCEV::FlagNUW;
11527 if (!LAR->getNoWrapFlags(NW) || !RAR->getNoWrapFlags(NW))
11528 return false;
11529
11530 return SE.isKnownPredicate(Pred, LAR->getStart(), RAR->getStart());
11531}
11532
11533/// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max
11534/// expression?
11535static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE,
11536 ICmpInst::Predicate Pred,
11537 const SCEV *LHS, const SCEV *RHS) {
11538 switch (Pred) {
11539 default:
11540 return false;
11541
11542 case ICmpInst::ICMP_SGE:
11543 std::swap(LHS, RHS);
11544 LLVM_FALLTHROUGH[[gnu::fallthrough]];
11545 case ICmpInst::ICMP_SLE:
11546 return
11547 // min(A, ...) <= A
11548 IsMinMaxConsistingOf<SCEVSMinExpr>(LHS, RHS) ||
11549 // A <= max(A, ...)
11550 IsMinMaxConsistingOf<SCEVSMaxExpr>(RHS, LHS);
11551
11552 case ICmpInst::ICMP_UGE:
11553 std::swap(LHS, RHS);
11554 LLVM_FALLTHROUGH[[gnu::fallthrough]];
11555 case ICmpInst::ICMP_ULE:
11556 return
11557 // min(A, ...) <= A
11558 // FIXME: what about umin_seq?
11559 IsMinMaxConsistingOf<SCEVUMinExpr>(LHS, RHS) ||
11560 // A <= max(A, ...)
11561 IsMinMaxConsistingOf<SCEVUMaxExpr>(RHS, LHS);
11562 }
11563
11564 llvm_unreachable("covered switch fell through?!")::llvm::llvm_unreachable_internal("covered switch fell through?!"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11564)
;
11565}
11566
11567bool ScalarEvolution::isImpliedViaOperations(ICmpInst::Predicate Pred,
11568 const SCEV *LHS, const SCEV *RHS,
11569 const SCEV *FoundLHS,
11570 const SCEV *FoundRHS,
11571 unsigned Depth) {
11572 assert(getTypeSizeInBits(LHS->getType()) ==(static_cast <bool> (getTypeSizeInBits(LHS->getType(
)) == getTypeSizeInBits(RHS->getType()) && "LHS and RHS have different sizes?"
) ? void (0) : __assert_fail ("getTypeSizeInBits(LHS->getType()) == getTypeSizeInBits(RHS->getType()) && \"LHS and RHS have different sizes?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11574, __extension__
__PRETTY_FUNCTION__))
11573 getTypeSizeInBits(RHS->getType()) &&(static_cast <bool> (getTypeSizeInBits(LHS->getType(
)) == getTypeSizeInBits(RHS->getType()) && "LHS and RHS have different sizes?"
) ? void (0) : __assert_fail ("getTypeSizeInBits(LHS->getType()) == getTypeSizeInBits(RHS->getType()) && \"LHS and RHS have different sizes?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11574, __extension__
__PRETTY_FUNCTION__))
11574 "LHS and RHS have different sizes?")(static_cast <bool> (getTypeSizeInBits(LHS->getType(
)) == getTypeSizeInBits(RHS->getType()) && "LHS and RHS have different sizes?"
) ? void (0) : __assert_fail ("getTypeSizeInBits(LHS->getType()) == getTypeSizeInBits(RHS->getType()) && \"LHS and RHS have different sizes?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11574, __extension__
__PRETTY_FUNCTION__))
;
11575 assert(getTypeSizeInBits(FoundLHS->getType()) ==(static_cast <bool> (getTypeSizeInBits(FoundLHS->getType
()) == getTypeSizeInBits(FoundRHS->getType()) && "FoundLHS and FoundRHS have different sizes?"
) ? void (0) : __assert_fail ("getTypeSizeInBits(FoundLHS->getType()) == getTypeSizeInBits(FoundRHS->getType()) && \"FoundLHS and FoundRHS have different sizes?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11577, __extension__
__PRETTY_FUNCTION__))
11576 getTypeSizeInBits(FoundRHS->getType()) &&(static_cast <bool> (getTypeSizeInBits(FoundLHS->getType
()) == getTypeSizeInBits(FoundRHS->getType()) && "FoundLHS and FoundRHS have different sizes?"
) ? void (0) : __assert_fail ("getTypeSizeInBits(FoundLHS->getType()) == getTypeSizeInBits(FoundRHS->getType()) && \"FoundLHS and FoundRHS have different sizes?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11577, __extension__
__PRETTY_FUNCTION__))
11577 "FoundLHS and FoundRHS have different sizes?")(static_cast <bool> (getTypeSizeInBits(FoundLHS->getType
()) == getTypeSizeInBits(FoundRHS->getType()) && "FoundLHS and FoundRHS have different sizes?"
) ? void (0) : __assert_fail ("getTypeSizeInBits(FoundLHS->getType()) == getTypeSizeInBits(FoundRHS->getType()) && \"FoundLHS and FoundRHS have different sizes?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11577, __extension__
__PRETTY_FUNCTION__))
;
11578 // We want to avoid hurting the compile time with analysis of too big trees.
11579 if (Depth > MaxSCEVOperationsImplicationDepth)
11580 return false;
11581
11582 // We only want to work with GT comparison so far.
11583 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_SLT) {
11584 Pred = CmpInst::getSwappedPredicate(Pred);
11585 std::swap(LHS, RHS);
11586 std::swap(FoundLHS, FoundRHS);
11587 }
11588
11589 // For unsigned, try to reduce it to corresponding signed comparison.
11590 if (Pred == ICmpInst::ICMP_UGT)
11591 // We can replace unsigned predicate with its signed counterpart if all
11592 // involved values are non-negative.
11593 // TODO: We could have better support for unsigned.
11594 if (isKnownNonNegative(FoundLHS) && isKnownNonNegative(FoundRHS)) {
11595 // Knowing that both FoundLHS and FoundRHS are non-negative, and knowing
11596 // FoundLHS >u FoundRHS, we also know that FoundLHS >s FoundRHS. Let us
11597 // use this fact to prove that LHS and RHS are non-negative.
11598 const SCEV *MinusOne = getMinusOne(LHS->getType());
11599 if (isImpliedCondOperands(ICmpInst::ICMP_SGT, LHS, MinusOne, FoundLHS,
11600 FoundRHS) &&
11601 isImpliedCondOperands(ICmpInst::ICMP_SGT, RHS, MinusOne, FoundLHS,
11602 FoundRHS))
11603 Pred = ICmpInst::ICMP_SGT;
11604 }
11605
11606 if (Pred != ICmpInst::ICMP_SGT)
11607 return false;
11608
11609 auto GetOpFromSExt = [&](const SCEV *S) {
11610 if (auto *Ext = dyn_cast<SCEVSignExtendExpr>(S))
11611 return Ext->getOperand();
11612 // TODO: If S is a SCEVConstant then you can cheaply "strip" the sext off
11613 // the constant in some cases.
11614 return S;
11615 };
11616
11617 // Acquire values from extensions.
11618 auto *OrigLHS = LHS;
11619 auto *OrigFoundLHS = FoundLHS;
11620 LHS = GetOpFromSExt(LHS);
11621 FoundLHS = GetOpFromSExt(FoundLHS);
11622
11623 // Is the SGT predicate can be proved trivially or using the found context.
11624 auto IsSGTViaContext = [&](const SCEV *S1, const SCEV *S2) {
11625 return isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_SGT, S1, S2) ||
11626 isImpliedViaOperations(ICmpInst::ICMP_SGT, S1, S2, OrigFoundLHS,
11627 FoundRHS, Depth + 1);
11628 };
11629
11630 if (auto *LHSAddExpr = dyn_cast<SCEVAddExpr>(LHS)) {
11631 // We want to avoid creation of any new non-constant SCEV. Since we are
11632 // going to compare the operands to RHS, we should be certain that we don't
11633 // need any size extensions for this. So let's decline all cases when the
11634 // sizes of types of LHS and RHS do not match.
11635 // TODO: Maybe try to get RHS from sext to catch more cases?
11636 if (getTypeSizeInBits(LHS->getType()) != getTypeSizeInBits(RHS->getType()))
11637 return false;
11638
11639 // Should not overflow.
11640 if (!LHSAddExpr->hasNoSignedWrap())
11641 return false;
11642
11643 auto *LL = LHSAddExpr->getOperand(0);
11644 auto *LR = LHSAddExpr->getOperand(1);
11645 auto *MinusOne = getMinusOne(RHS->getType());
11646
11647 // Checks that S1 >= 0 && S2 > RHS, trivially or using the found context.
11648 auto IsSumGreaterThanRHS = [&](const SCEV *S1, const SCEV *S2) {
11649 return IsSGTViaContext(S1, MinusOne) && IsSGTViaContext(S2, RHS);
11650 };
11651 // Try to prove the following rule:
11652 // (LHS = LL + LR) && (LL >= 0) && (LR > RHS) => (LHS > RHS).
11653 // (LHS = LL + LR) && (LR >= 0) && (LL > RHS) => (LHS > RHS).
11654 if (IsSumGreaterThanRHS(LL, LR) || IsSumGreaterThanRHS(LR, LL))
11655 return true;
11656 } else if (auto *LHSUnknownExpr = dyn_cast<SCEVUnknown>(LHS)) {
11657 Value *LL, *LR;
11658 // FIXME: Once we have SDiv implemented, we can get rid of this matching.
11659
11660 using namespace llvm::PatternMatch;
11661
11662 if (match(LHSUnknownExpr->getValue(), m_SDiv(m_Value(LL), m_Value(LR)))) {
11663 // Rules for division.
11664 // We are going to perform some comparisons with Denominator and its
11665 // derivative expressions. In general case, creating a SCEV for it may
11666 // lead to a complex analysis of the entire graph, and in particular it
11667 // can request trip count recalculation for the same loop. This would
11668 // cache as SCEVCouldNotCompute to avoid the infinite recursion. To avoid
11669 // this, we only want to create SCEVs that are constants in this section.
11670 // So we bail if Denominator is not a constant.
11671 if (!isa<ConstantInt>(LR))
11672 return false;
11673
11674 auto *Denominator = cast<SCEVConstant>(getSCEV(LR));
11675
11676 // We want to make sure that LHS = FoundLHS / Denominator. If it is so,
11677 // then a SCEV for the numerator already exists and matches with FoundLHS.
11678 auto *Numerator = getExistingSCEV(LL);
11679 if (!Numerator || Numerator->getType() != FoundLHS->getType())
11680 return false;
11681
11682 // Make sure that the numerator matches with FoundLHS and the denominator
11683 // is positive.
11684 if (!HasSameValue(Numerator, FoundLHS) || !isKnownPositive(Denominator))
11685 return false;
11686
11687 auto *DTy = Denominator->getType();
11688 auto *FRHSTy = FoundRHS->getType();
11689 if (DTy->isPointerTy() != FRHSTy->isPointerTy())
11690 // One of types is a pointer and another one is not. We cannot extend
11691 // them properly to a wider type, so let us just reject this case.
11692 // TODO: Usage of getEffectiveSCEVType for DTy, FRHSTy etc should help
11693 // to avoid this check.
11694 return false;
11695
11696 // Given that:
11697 // FoundLHS > FoundRHS, LHS = FoundLHS / Denominator, Denominator > 0.
11698 auto *WTy = getWiderType(DTy, FRHSTy);
11699 auto *DenominatorExt = getNoopOrSignExtend(Denominator, WTy);
11700 auto *FoundRHSExt = getNoopOrSignExtend(FoundRHS, WTy);
11701
11702 // Try to prove the following rule:
11703 // (FoundRHS > Denominator - 2) && (RHS <= 0) => (LHS > RHS).
11704 // For example, given that FoundLHS > 2. It means that FoundLHS is at
11705 // least 3. If we divide it by Denominator < 4, we will have at least 1.
11706 auto *DenomMinusTwo = getMinusSCEV(DenominatorExt, getConstant(WTy, 2));
11707 if (isKnownNonPositive(RHS) &&
11708 IsSGTViaContext(FoundRHSExt, DenomMinusTwo))
11709 return true;
11710
11711 // Try to prove the following rule:
11712 // (FoundRHS > -1 - Denominator) && (RHS < 0) => (LHS > RHS).
11713 // For example, given that FoundLHS > -3. Then FoundLHS is at least -2.
11714 // If we divide it by Denominator > 2, then:
11715 // 1. If FoundLHS is negative, then the result is 0.
11716 // 2. If FoundLHS is non-negative, then the result is non-negative.
11717 // Anyways, the result is non-negative.
11718 auto *MinusOne = getMinusOne(WTy);
11719 auto *NegDenomMinusOne = getMinusSCEV(MinusOne, DenominatorExt);
11720 if (isKnownNegative(RHS) &&
11721 IsSGTViaContext(FoundRHSExt, NegDenomMinusOne))
11722 return true;
11723 }
11724 }
11725
11726 // If our expression contained SCEVUnknown Phis, and we split it down and now
11727 // need to prove something for them, try to prove the predicate for every
11728 // possible incoming values of those Phis.
11729 if (isImpliedViaMerge(Pred, OrigLHS, RHS, OrigFoundLHS, FoundRHS, Depth + 1))
11730 return true;
11731
11732 return false;
11733}
11734
11735static bool isKnownPredicateExtendIdiom(ICmpInst::Predicate Pred,
11736 const SCEV *LHS, const SCEV *RHS) {
11737 // zext x u<= sext x, sext x s<= zext x
11738 switch (Pred) {
11739 case ICmpInst::ICMP_SGE:
11740 std::swap(LHS, RHS);
11741 LLVM_FALLTHROUGH[[gnu::fallthrough]];
11742 case ICmpInst::ICMP_SLE: {
11743 // If operand >=s 0 then ZExt == SExt. If operand <s 0 then SExt <s ZExt.
11744 const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(LHS);
11745 const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(RHS);
11746 if (SExt && ZExt && SExt->getOperand() == ZExt->getOperand())
11747 return true;
11748 break;
11749 }
11750 case ICmpInst::ICMP_UGE:
11751 std::swap(LHS, RHS);
11752 LLVM_FALLTHROUGH[[gnu::fallthrough]];
11753 case ICmpInst::ICMP_ULE: {
11754 // If operand >=s 0 then ZExt == SExt. If operand <s 0 then ZExt <u SExt.
11755 const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(LHS);
11756 const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(RHS);
11757 if (SExt && ZExt && SExt->getOperand() == ZExt->getOperand())
11758 return true;
11759 break;
11760 }
11761 default:
11762 break;
11763 };
11764 return false;
11765}
11766
11767bool
11768ScalarEvolution::isKnownViaNonRecursiveReasoning(ICmpInst::Predicate Pred,
11769 const SCEV *LHS, const SCEV *RHS) {
11770 return isKnownPredicateExtendIdiom(Pred, LHS, RHS) ||
11771 isKnownPredicateViaConstantRanges(Pred, LHS, RHS) ||
11772 IsKnownPredicateViaMinOrMax(*this, Pred, LHS, RHS) ||
11773 IsKnownPredicateViaAddRecStart(*this, Pred, LHS, RHS) ||
11774 isKnownPredicateViaNoOverflow(Pred, LHS, RHS);
11775}
11776
11777bool
11778ScalarEvolution::isImpliedCondOperandsHelper(ICmpInst::Predicate Pred,
11779 const SCEV *LHS, const SCEV *RHS,
11780 const SCEV *FoundLHS,
11781 const SCEV *FoundRHS) {
11782 switch (Pred) {
11783 default: llvm_unreachable("Unexpected ICmpInst::Predicate value!")::llvm::llvm_unreachable_internal("Unexpected ICmpInst::Predicate value!"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11783)
;
11784 case ICmpInst::ICMP_EQ:
11785 case ICmpInst::ICMP_NE:
11786 if (HasSameValue(LHS, FoundLHS) && HasSameValue(RHS, FoundRHS))
11787 return true;
11788 break;
11789 case ICmpInst::ICMP_SLT:
11790 case ICmpInst::ICMP_SLE:
11791 if (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_SLE, LHS, FoundLHS) &&
11792 isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_SGE, RHS, FoundRHS))
11793 return true;
11794 break;
11795 case ICmpInst::ICMP_SGT:
11796 case ICmpInst::ICMP_SGE:
11797 if (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_SGE, LHS, FoundLHS) &&
11798 isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_SLE, RHS, FoundRHS))
11799 return true;
11800 break;
11801 case ICmpInst::ICMP_ULT:
11802 case ICmpInst::ICMP_ULE:
11803 if (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_ULE, LHS, FoundLHS) &&
11804 isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_UGE, RHS, FoundRHS))
11805 return true;
11806 break;
11807 case ICmpInst::ICMP_UGT:
11808 case ICmpInst::ICMP_UGE:
11809 if (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_UGE, LHS, FoundLHS) &&
11810 isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_ULE, RHS, FoundRHS))
11811 return true;
11812 break;
11813 }
11814
11815 // Maybe it can be proved via operations?
11816 if (isImpliedViaOperations(Pred, LHS, RHS, FoundLHS, FoundRHS))
11817 return true;
11818
11819 return false;
11820}
11821
11822bool ScalarEvolution::isImpliedCondOperandsViaRanges(ICmpInst::Predicate Pred,
11823 const SCEV *LHS,
11824 const SCEV *RHS,
11825 const SCEV *FoundLHS,
11826 const SCEV *FoundRHS) {
11827 if (!isa<SCEVConstant>(RHS) || !isa<SCEVConstant>(FoundRHS))
11828 // The restriction on `FoundRHS` be lifted easily -- it exists only to
11829 // reduce the compile time impact of this optimization.
11830 return false;
11831
11832 Optional<APInt> Addend = computeConstantDifference(LHS, FoundLHS);
11833 if (!Addend)
11834 return false;
11835
11836 const APInt &ConstFoundRHS = cast<SCEVConstant>(FoundRHS)->getAPInt();
11837
11838 // `FoundLHSRange` is the range we know `FoundLHS` to be in by virtue of the
11839 // antecedent "`FoundLHS` `Pred` `FoundRHS`".
11840 ConstantRange FoundLHSRange =
11841 ConstantRange::makeExactICmpRegion(Pred, ConstFoundRHS);
11842
11843 // Since `LHS` is `FoundLHS` + `Addend`, we can compute a range for `LHS`:
11844 ConstantRange LHSRange = FoundLHSRange.add(ConstantRange(*Addend));
11845
11846 // We can also compute the range of values for `LHS` that satisfy the
11847 // consequent, "`LHS` `Pred` `RHS`":
11848 const APInt &ConstRHS = cast<SCEVConstant>(RHS)->getAPInt();
11849 // The antecedent implies the consequent if every value of `LHS` that
11850 // satisfies the antecedent also satisfies the consequent.
11851 return LHSRange.icmp(Pred, ConstRHS);
11852}
11853
11854bool ScalarEvolution::canIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride,
11855 bool IsSigned) {
11856 assert(isKnownPositive(Stride) && "Positive stride expected!")(static_cast <bool> (isKnownPositive(Stride) &&
"Positive stride expected!") ? void (0) : __assert_fail ("isKnownPositive(Stride) && \"Positive stride expected!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11856, __extension__
__PRETTY_FUNCTION__))
;
11857
11858 unsigned BitWidth = getTypeSizeInBits(RHS->getType());
11859 const SCEV *One = getOne(Stride->getType());
11860
11861 if (IsSigned) {
11862 APInt MaxRHS = getSignedRangeMax(RHS);
11863 APInt MaxValue = APInt::getSignedMaxValue(BitWidth);
11864 APInt MaxStrideMinusOne = getSignedRangeMax(getMinusSCEV(Stride, One));
11865
11866 // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow!
11867 return (std::move(MaxValue) - MaxStrideMinusOne).slt(MaxRHS);
11868 }
11869
11870 APInt MaxRHS = getUnsignedRangeMax(RHS);
11871 APInt MaxValue = APInt::getMaxValue(BitWidth);
11872 APInt MaxStrideMinusOne = getUnsignedRangeMax(getMinusSCEV(Stride, One));
11873
11874 // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow!
11875 return (std::move(MaxValue) - MaxStrideMinusOne).ult(MaxRHS);
11876}
11877
11878bool ScalarEvolution::canIVOverflowOnGT(const SCEV *RHS, const SCEV *Stride,
11879 bool IsSigned) {
11880
11881 unsigned BitWidth = getTypeSizeInBits(RHS->getType());
11882 const SCEV *One = getOne(Stride->getType());
11883
11884 if (IsSigned) {
11885 APInt MinRHS = getSignedRangeMin(RHS);
11886 APInt MinValue = APInt::getSignedMinValue(BitWidth);
11887 APInt MaxStrideMinusOne = getSignedRangeMax(getMinusSCEV(Stride, One));
11888
11889 // SMinRHS - SMaxStrideMinusOne < SMinValue => overflow!
11890 return (std::move(MinValue) + MaxStrideMinusOne).sgt(MinRHS);
11891 }
11892
11893 APInt MinRHS = getUnsignedRangeMin(RHS);
11894 APInt MinValue = APInt::getMinValue(BitWidth);
11895 APInt MaxStrideMinusOne = getUnsignedRangeMax(getMinusSCEV(Stride, One));
11896
11897 // UMinRHS - UMaxStrideMinusOne < UMinValue => overflow!
11898 return (std::move(MinValue) + MaxStrideMinusOne).ugt(MinRHS);
11899}
11900
11901const SCEV *ScalarEvolution::getUDivCeilSCEV(const SCEV *N, const SCEV *D) {
11902 // umin(N, 1) + floor((N - umin(N, 1)) / D)
11903 // This is equivalent to "1 + floor((N - 1) / D)" for N != 0. The umin
11904 // expression fixes the case of N=0.
11905 const SCEV *MinNOne = getUMinExpr(N, getOne(N->getType()));
11906 const SCEV *NMinusOne = getMinusSCEV(N, MinNOne);
11907 return getAddExpr(MinNOne, getUDivExpr(NMinusOne, D));
11908}
11909
11910const SCEV *ScalarEvolution::computeMaxBECountForLT(const SCEV *Start,
11911 const SCEV *Stride,
11912 const SCEV *End,
11913 unsigned BitWidth,
11914 bool IsSigned) {
11915 // The logic in this function assumes we can represent a positive stride.
11916 // If we can't, the backedge-taken count must be zero.
11917 if (IsSigned && BitWidth == 1)
11918 return getZero(Stride->getType());
11919
11920 // This code has only been closely audited for negative strides in the
11921 // unsigned comparison case, it may be correct for signed comparison, but
11922 // that needs to be established.
11923 assert((!IsSigned || !isKnownNonPositive(Stride)) &&(static_cast <bool> ((!IsSigned || !isKnownNonPositive(
Stride)) && "Stride is expected strictly positive for signed case!"
) ? void (0) : __assert_fail ("(!IsSigned || !isKnownNonPositive(Stride)) && \"Stride is expected strictly positive for signed case!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11924, __extension__
__PRETTY_FUNCTION__))
11924 "Stride is expected strictly positive for signed case!")(static_cast <bool> ((!IsSigned || !isKnownNonPositive(
Stride)) && "Stride is expected strictly positive for signed case!"
) ? void (0) : __assert_fail ("(!IsSigned || !isKnownNonPositive(Stride)) && \"Stride is expected strictly positive for signed case!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 11924, __extension__
__PRETTY_FUNCTION__))
;
11925
11926 // Calculate the maximum backedge count based on the range of values
11927 // permitted by Start, End, and Stride.
11928 APInt MinStart =
11929 IsSigned ? getSignedRangeMin(Start) : getUnsignedRangeMin(Start);
11930
11931 APInt MinStride =
11932 IsSigned ? getSignedRangeMin(Stride) : getUnsignedRangeMin(Stride);
11933
11934 // We assume either the stride is positive, or the backedge-taken count
11935 // is zero. So force StrideForMaxBECount to be at least one.
11936 APInt One(BitWidth, 1);
11937 APInt StrideForMaxBECount = IsSigned ? APIntOps::smax(One, MinStride)
11938 : APIntOps::umax(One, MinStride);
11939
11940 APInt MaxValue = IsSigned ? APInt::getSignedMaxValue(BitWidth)
11941 : APInt::getMaxValue(BitWidth);
11942 APInt Limit = MaxValue - (StrideForMaxBECount - 1);
11943
11944 // Although End can be a MAX expression we estimate MaxEnd considering only
11945 // the case End = RHS of the loop termination condition. This is safe because
11946 // in the other case (End - Start) is zero, leading to a zero maximum backedge
11947 // taken count.
11948 APInt MaxEnd = IsSigned ? APIntOps::smin(getSignedRangeMax(End), Limit)
11949 : APIntOps::umin(getUnsignedRangeMax(End), Limit);
11950
11951 // MaxBECount = ceil((max(MaxEnd, MinStart) - MinStart) / Stride)
11952 MaxEnd = IsSigned ? APIntOps::smax(MaxEnd, MinStart)
11953 : APIntOps::umax(MaxEnd, MinStart);
11954
11955 return getUDivCeilSCEV(getConstant(MaxEnd - MinStart) /* Delta */,
11956 getConstant(StrideForMaxBECount) /* Step */);
11957}
11958
11959ScalarEvolution::ExitLimit
11960ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
11961 const Loop *L, bool IsSigned,
11962 bool ControlsExit, bool AllowPredicates) {
11963 SmallPtrSet<const SCEVPredicate *, 4> Predicates;
11964
11965 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(LHS);
11966 bool PredicatedIV = false;
11967
11968 auto canAssumeNoSelfWrap = [&](const SCEVAddRecExpr *AR) {
11969 // Can we prove this loop *must* be UB if overflow of IV occurs?
11970 // Reasoning goes as follows:
11971 // * Suppose the IV did self wrap.
11972 // * If Stride evenly divides the iteration space, then once wrap
11973 // occurs, the loop must revisit the same values.
11974 // * We know that RHS is invariant, and that none of those values
11975 // caused this exit to be taken previously. Thus, this exit is
11976 // dynamically dead.
11977 // * If this is the sole exit, then a dead exit implies the loop
11978 // must be infinite if there are no abnormal exits.
11979 // * If the loop were infinite, then it must either not be mustprogress
11980 // or have side effects. Otherwise, it must be UB.
11981 // * It can't (by assumption), be UB so we have contradicted our
11982 // premise and can conclude the IV did not in fact self-wrap.
11983 if (!isLoopInvariant(RHS, L))
11984 return false;
11985
11986 auto *StrideC = dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this));
11987 if (!StrideC || !StrideC->getAPInt().isPowerOf2())
11988 return false;
11989
11990 if (!ControlsExit || !loopHasNoAbnormalExits(L))
11991 return false;
11992
11993 return loopIsFiniteByAssumption(L);
11994 };
11995
11996 if (!IV) {
11997 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(LHS)) {
11998 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(ZExt->getOperand());
11999 if (AR && AR->getLoop() == L && AR->isAffine()) {
12000 auto canProveNUW = [&]() {
12001 if (!isLoopInvariant(RHS, L))
12002 return false;
12003
12004 if (!isKnownNonZero(AR->getStepRecurrence(*this)))
12005 // We need the sequence defined by AR to strictly increase in the
12006 // unsigned integer domain for the logic below to hold.
12007 return false;
12008
12009 const unsigned InnerBitWidth = getTypeSizeInBits(AR->getType());
12010 const unsigned OuterBitWidth = getTypeSizeInBits(RHS->getType());
12011 // If RHS <=u Limit, then there must exist a value V in the sequence
12012 // defined by AR (e.g. {Start,+,Step}) such that V >u RHS, and
12013 // V <=u UINT_MAX. Thus, we must exit the loop before unsigned
12014 // overflow occurs. This limit also implies that a signed comparison
12015 // (in the wide bitwidth) is equivalent to an unsigned comparison as
12016 // the high bits on both sides must be zero.
12017 APInt StrideMax = getUnsignedRangeMax(AR->getStepRecurrence(*this));
12018 APInt Limit = APInt::getMaxValue(InnerBitWidth) - (StrideMax - 1);
12019 Limit = Limit.zext(OuterBitWidth);
12020 return getUnsignedRangeMax(applyLoopGuards(RHS, L)).ule(Limit);
12021 };
12022 auto Flags = AR->getNoWrapFlags();
12023 if (!hasFlags(Flags, SCEV::FlagNUW) && canProveNUW())
12024 Flags = setFlags(Flags, SCEV::FlagNUW);
12025
12026 setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), Flags);
12027 if (AR->hasNoUnsignedWrap()) {
12028 // Emulate what getZeroExtendExpr would have done during construction
12029 // if we'd been able to infer the fact just above at that time.
12030 const SCEV *Step = AR->getStepRecurrence(*this);
12031 Type *Ty = ZExt->getType();
12032 auto *S = getAddRecExpr(
12033 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, 0),
12034 getZeroExtendExpr(Step, Ty, 0), L, AR->getNoWrapFlags());
12035 IV = dyn_cast<SCEVAddRecExpr>(S);
12036 }
12037 }
12038 }
12039 }
12040
12041
12042 if (!IV && AllowPredicates) {
12043 // Try to make this an AddRec using runtime tests, in the first X
12044 // iterations of this loop, where X is the SCEV expression found by the
12045 // algorithm below.
12046 IV = convertSCEVToAddRecWithPredicates(LHS, L, Predicates);
12047 PredicatedIV = true;
12048 }
12049
12050 // Avoid weird loops
12051 if (!IV || IV->getLoop() != L || !IV->isAffine())
12052 return getCouldNotCompute();
12053
12054 // A precondition of this method is that the condition being analyzed
12055 // reaches an exiting branch which dominates the latch. Given that, we can
12056 // assume that an increment which violates the nowrap specification and
12057 // produces poison must cause undefined behavior when the resulting poison
12058 // value is branched upon and thus we can conclude that the backedge is
12059 // taken no more often than would be required to produce that poison value.
12060 // Note that a well defined loop can exit on the iteration which violates
12061 // the nowrap specification if there is another exit (either explicit or
12062 // implicit/exceptional) which causes the loop to execute before the
12063 // exiting instruction we're analyzing would trigger UB.
12064 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
12065 bool NoWrap = ControlsExit && IV->getNoWrapFlags(WrapType);
12066 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
12067
12068 const SCEV *Stride = IV->getStepRecurrence(*this);
12069
12070 bool PositiveStride = isKnownPositive(Stride);
12071
12072 // Avoid negative or zero stride values.
12073 if (!PositiveStride) {
12074 // We can compute the correct backedge taken count for loops with unknown
12075 // strides if we can prove that the loop is not an infinite loop with side
12076 // effects. Here's the loop structure we are trying to handle -
12077 //
12078 // i = start
12079 // do {
12080 // A[i] = i;
12081 // i += s;
12082 // } while (i < end);
12083 //
12084 // The backedge taken count for such loops is evaluated as -
12085 // (max(end, start + stride) - start - 1) /u stride
12086 //
12087 // The additional preconditions that we need to check to prove correctness
12088 // of the above formula is as follows -
12089 //
12090 // a) IV is either nuw or nsw depending upon signedness (indicated by the
12091 // NoWrap flag).
12092 // b) the loop is guaranteed to be finite (e.g. is mustprogress and has
12093 // no side effects within the loop)
12094 // c) loop has a single static exit (with no abnormal exits)
12095 //
12096 // Precondition a) implies that if the stride is negative, this is a single
12097 // trip loop. The backedge taken count formula reduces to zero in this case.
12098 //
12099 // Precondition b) and c) combine to imply that if rhs is invariant in L,
12100 // then a zero stride means the backedge can't be taken without executing
12101 // undefined behavior.
12102 //
12103 // The positive stride case is the same as isKnownPositive(Stride) returning
12104 // true (original behavior of the function).
12105 //
12106 if (PredicatedIV || !NoWrap || !loopIsFiniteByAssumption(L) ||
12107 !loopHasNoAbnormalExits(L))
12108 return getCouldNotCompute();
12109
12110 // This bailout is protecting the logic in computeMaxBECountForLT which
12111 // has not yet been sufficiently auditted or tested with negative strides.
12112 // We used to filter out all known-non-positive cases here, we're in the
12113 // process of being less restrictive bit by bit.
12114 if (IsSigned && isKnownNonPositive(Stride))
12115 return getCouldNotCompute();
12116
12117 if (!isKnownNonZero(Stride)) {
12118 // If we have a step of zero, and RHS isn't invariant in L, we don't know
12119 // if it might eventually be greater than start and if so, on which
12120 // iteration. We can't even produce a useful upper bound.
12121 if (!isLoopInvariant(RHS, L))
12122 return getCouldNotCompute();
12123
12124 // We allow a potentially zero stride, but we need to divide by stride
12125 // below. Since the loop can't be infinite and this check must control
12126 // the sole exit, we can infer the exit must be taken on the first
12127 // iteration (e.g. backedge count = 0) if the stride is zero. Given that,
12128 // we know the numerator in the divides below must be zero, so we can
12129 // pick an arbitrary non-zero value for the denominator (e.g. stride)
12130 // and produce the right result.
12131 // FIXME: Handle the case where Stride is poison?
12132 auto wouldZeroStrideBeUB = [&]() {
12133 // Proof by contradiction. Suppose the stride were zero. If we can
12134 // prove that the backedge *is* taken on the first iteration, then since
12135 // we know this condition controls the sole exit, we must have an
12136 // infinite loop. We can't have a (well defined) infinite loop per
12137 // check just above.
12138 // Note: The (Start - Stride) term is used to get the start' term from
12139 // (start' + stride,+,stride). Remember that we only care about the
12140 // result of this expression when stride == 0 at runtime.
12141 auto *StartIfZero = getMinusSCEV(IV->getStart(), Stride);
12142 return isLoopEntryGuardedByCond(L, Cond, StartIfZero, RHS);
12143 };
12144 if (!wouldZeroStrideBeUB()) {
12145 Stride = getUMaxExpr(Stride, getOne(Stride->getType()));
12146 }
12147 }
12148 } else if (!Stride->isOne() && !NoWrap) {
12149 auto isUBOnWrap = [&]() {
12150 // From no-self-wrap, we need to then prove no-(un)signed-wrap. This
12151 // follows trivially from the fact that every (un)signed-wrapped, but
12152 // not self-wrapped value must be LT than the last value before
12153 // (un)signed wrap. Since we know that last value didn't exit, nor
12154 // will any smaller one.
12155 return canAssumeNoSelfWrap(IV);
12156 };
12157
12158 // Avoid proven overflow cases: this will ensure that the backedge taken
12159 // count will not generate any unsigned overflow. Relaxed no-overflow
12160 // conditions exploit NoWrapFlags, allowing to optimize in presence of
12161 // undefined behaviors like the case of C language.
12162 if (canIVOverflowOnLT(RHS, Stride, IsSigned) && !isUBOnWrap())
12163 return getCouldNotCompute();
12164 }
12165
12166 // On all paths just preceeding, we established the following invariant:
12167 // IV can be assumed not to overflow up to and including the exiting
12168 // iteration. We proved this in one of two ways:
12169 // 1) We can show overflow doesn't occur before the exiting iteration
12170 // 1a) canIVOverflowOnLT, and b) step of one
12171 // 2) We can show that if overflow occurs, the loop must execute UB
12172 // before any possible exit.
12173 // Note that we have not yet proved RHS invariant (in general).
12174
12175 const SCEV *Start = IV->getStart();
12176
12177 // Preserve pointer-typed Start/RHS to pass to isLoopEntryGuardedByCond.
12178 // If we convert to integers, isLoopEntryGuardedByCond will miss some cases.
12179 // Use integer-typed versions for actual computation; we can't subtract
12180 // pointers in general.
12181 const SCEV *OrigStart = Start;
12182 const SCEV *OrigRHS = RHS;
12183 if (Start->getType()->isPointerTy()) {
12184 Start = getLosslessPtrToIntExpr(Start);
12185 if (isa<SCEVCouldNotCompute>(Start))
12186 return Start;
12187 }
12188 if (RHS->getType()->isPointerTy()) {
12189 RHS = getLosslessPtrToIntExpr(RHS);
12190 if (isa<SCEVCouldNotCompute>(RHS))
12191 return RHS;
12192 }
12193
12194 // When the RHS is not invariant, we do not know the end bound of the loop and
12195 // cannot calculate the ExactBECount needed by ExitLimit. However, we can
12196 // calculate the MaxBECount, given the start, stride and max value for the end
12197 // bound of the loop (RHS), and the fact that IV does not overflow (which is
12198 // checked above).
12199 if (!isLoopInvariant(RHS, L)) {
12200 const SCEV *MaxBECount = computeMaxBECountForLT(
12201 Start, Stride, RHS, getTypeSizeInBits(LHS->getType()), IsSigned);
12202 return ExitLimit(getCouldNotCompute() /* ExactNotTaken */, MaxBECount,
12203 false /*MaxOrZero*/, Predicates);
12204 }
12205
12206 // We use the expression (max(End,Start)-Start)/Stride to describe the
12207 // backedge count, as if the backedge is taken at least once max(End,Start)
12208 // is End and so the result is as above, and if not max(End,Start) is Start
12209 // so we get a backedge count of zero.
12210 const SCEV *BECount = nullptr;
12211 auto *OrigStartMinusStride = getMinusSCEV(OrigStart, Stride);
12212 assert(isAvailableAtLoopEntry(OrigStartMinusStride, L) && "Must be!")(static_cast <bool> (isAvailableAtLoopEntry(OrigStartMinusStride
, L) && "Must be!") ? void (0) : __assert_fail ("isAvailableAtLoopEntry(OrigStartMinusStride, L) && \"Must be!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12212, __extension__
__PRETTY_FUNCTION__))
;
12213 assert(isAvailableAtLoopEntry(OrigStart, L) && "Must be!")(static_cast <bool> (isAvailableAtLoopEntry(OrigStart, L
) && "Must be!") ? void (0) : __assert_fail ("isAvailableAtLoopEntry(OrigStart, L) && \"Must be!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12213, __extension__
__PRETTY_FUNCTION__))
;
12214 assert(isAvailableAtLoopEntry(OrigRHS, L) && "Must be!")(static_cast <bool> (isAvailableAtLoopEntry(OrigRHS, L)
&& "Must be!") ? void (0) : __assert_fail ("isAvailableAtLoopEntry(OrigRHS, L) && \"Must be!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12214, __extension__
__PRETTY_FUNCTION__))
;
12215 // Can we prove (max(RHS,Start) > Start - Stride?
12216 if (isLoopEntryGuardedByCond(L, Cond, OrigStartMinusStride, OrigStart) &&
12217 isLoopEntryGuardedByCond(L, Cond, OrigStartMinusStride, OrigRHS)) {
12218 // In this case, we can use a refined formula for computing backedge taken
12219 // count. The general formula remains:
12220 // "End-Start /uceiling Stride" where "End = max(RHS,Start)"
12221 // We want to use the alternate formula:
12222 // "((End - 1) - (Start - Stride)) /u Stride"
12223 // Let's do a quick case analysis to show these are equivalent under
12224 // our precondition that max(RHS,Start) > Start - Stride.
12225 // * For RHS <= Start, the backedge-taken count must be zero.
12226 // "((End - 1) - (Start - Stride)) /u Stride" reduces to
12227 // "((Start - 1) - (Start - Stride)) /u Stride" which simplies to
12228 // "Stride - 1 /u Stride" which is indeed zero for all non-zero values
12229 // of Stride. For 0 stride, we've use umin(1,Stride) above, reducing
12230 // this to the stride of 1 case.
12231 // * For RHS >= Start, the backedge count must be "RHS-Start /uceil Stride".
12232 // "((End - 1) - (Start - Stride)) /u Stride" reduces to
12233 // "((RHS - 1) - (Start - Stride)) /u Stride" reassociates to
12234 // "((RHS - (Start - Stride) - 1) /u Stride".
12235 // Our preconditions trivially imply no overflow in that form.
12236 const SCEV *MinusOne = getMinusOne(Stride->getType());
12237 const SCEV *Numerator =
12238 getMinusSCEV(getAddExpr(RHS, MinusOne), getMinusSCEV(Start, Stride));
12239 BECount = getUDivExpr(Numerator, Stride);
12240 }
12241
12242 const SCEV *BECountIfBackedgeTaken = nullptr;
12243 if (!BECount) {
12244 auto canProveRHSGreaterThanEqualStart = [&]() {
12245 auto CondGE = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
12246 if (isLoopEntryGuardedByCond(L, CondGE, OrigRHS, OrigStart))
12247 return true;
12248
12249 // (RHS > Start - 1) implies RHS >= Start.
12250 // * "RHS >= Start" is trivially equivalent to "RHS > Start - 1" if
12251 // "Start - 1" doesn't overflow.
12252 // * For signed comparison, if Start - 1 does overflow, it's equal
12253 // to INT_MAX, and "RHS >s INT_MAX" is trivially false.
12254 // * For unsigned comparison, if Start - 1 does overflow, it's equal
12255 // to UINT_MAX, and "RHS >u UINT_MAX" is trivially false.
12256 //
12257 // FIXME: Should isLoopEntryGuardedByCond do this for us?
12258 auto CondGT = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
12259 auto *StartMinusOne = getAddExpr(OrigStart,
12260 getMinusOne(OrigStart->getType()));
12261 return isLoopEntryGuardedByCond(L, CondGT, OrigRHS, StartMinusOne);
12262 };
12263
12264 // If we know that RHS >= Start in the context of loop, then we know that
12265 // max(RHS, Start) = RHS at this point.
12266 const SCEV *End;
12267 if (canProveRHSGreaterThanEqualStart()) {
12268 End = RHS;
12269 } else {
12270 // If RHS < Start, the backedge will be taken zero times. So in
12271 // general, we can write the backedge-taken count as:
12272 //
12273 // RHS >= Start ? ceil(RHS - Start) / Stride : 0
12274 //
12275 // We convert it to the following to make it more convenient for SCEV:
12276 //
12277 // ceil(max(RHS, Start) - Start) / Stride
12278 End = IsSigned ? getSMaxExpr(RHS, Start) : getUMaxExpr(RHS, Start);
12279
12280 // See what would happen if we assume the backedge is taken. This is
12281 // used to compute MaxBECount.
12282 BECountIfBackedgeTaken = getUDivCeilSCEV(getMinusSCEV(RHS, Start), Stride);
12283 }
12284
12285 // At this point, we know:
12286 //
12287 // 1. If IsSigned, Start <=s End; otherwise, Start <=u End
12288 // 2. The index variable doesn't overflow.
12289 //
12290 // Therefore, we know N exists such that
12291 // (Start + Stride * N) >= End, and computing "(Start + Stride * N)"
12292 // doesn't overflow.
12293 //
12294 // Using this information, try to prove whether the addition in
12295 // "(Start - End) + (Stride - 1)" has unsigned overflow.
12296 const SCEV *One = getOne(Stride->getType());
12297 bool MayAddOverflow = [&] {
12298 if (auto *StrideC = dyn_cast<SCEVConstant>(Stride)) {
12299 if (StrideC->getAPInt().isPowerOf2()) {
12300 // Suppose Stride is a power of two, and Start/End are unsigned
12301 // integers. Let UMAX be the largest representable unsigned
12302 // integer.
12303 //
12304 // By the preconditions of this function, we know
12305 // "(Start + Stride * N) >= End", and this doesn't overflow.
12306 // As a formula:
12307 //
12308 // End <= (Start + Stride * N) <= UMAX
12309 //
12310 // Subtracting Start from all the terms:
12311 //
12312 // End - Start <= Stride * N <= UMAX - Start
12313 //
12314 // Since Start is unsigned, UMAX - Start <= UMAX. Therefore:
12315 //
12316 // End - Start <= Stride * N <= UMAX
12317 //
12318 // Stride * N is a multiple of Stride. Therefore,
12319 //
12320 // End - Start <= Stride * N <= UMAX - (UMAX mod Stride)
12321 //
12322 // Since Stride is a power of two, UMAX + 1 is divisible by Stride.
12323 // Therefore, UMAX mod Stride == Stride - 1. So we can write:
12324 //
12325 // End - Start <= Stride * N <= UMAX - Stride - 1
12326 //
12327 // Dropping the middle term:
12328 //
12329 // End - Start <= UMAX - Stride - 1
12330 //
12331 // Adding Stride - 1 to both sides:
12332 //
12333 // (End - Start) + (Stride - 1) <= UMAX
12334 //
12335 // In other words, the addition doesn't have unsigned overflow.
12336 //
12337 // A similar proof works if we treat Start/End as signed values.
12338 // Just rewrite steps before "End - Start <= Stride * N <= UMAX" to
12339 // use signed max instead of unsigned max. Note that we're trying
12340 // to prove a lack of unsigned overflow in either case.
12341 return false;
12342 }
12343 }
12344 if (Start == Stride || Start == getMinusSCEV(Stride, One)) {
12345 // If Start is equal to Stride, (End - Start) + (Stride - 1) == End - 1.
12346 // If !IsSigned, 0 <u Stride == Start <=u End; so 0 <u End - 1 <u End.
12347 // If IsSigned, 0 <s Stride == Start <=s End; so 0 <s End - 1 <s End.
12348 //
12349 // If Start is equal to Stride - 1, (End - Start) + Stride - 1 == End.
12350 return false;
12351 }
12352 return true;
12353 }();
12354
12355 const SCEV *Delta = getMinusSCEV(End, Start);
12356 if (!MayAddOverflow) {
12357 // floor((D + (S - 1)) / S)
12358 // We prefer this formulation if it's legal because it's fewer operations.
12359 BECount =
12360 getUDivExpr(getAddExpr(Delta, getMinusSCEV(Stride, One)), Stride);
12361 } else {
12362 BECount = getUDivCeilSCEV(Delta, Stride);
12363 }
12364 }
12365
12366 const SCEV *MaxBECount;
12367 bool MaxOrZero = false;
12368 if (isa<SCEVConstant>(BECount)) {
12369 MaxBECount = BECount;
12370 } else if (BECountIfBackedgeTaken &&
12371 isa<SCEVConstant>(BECountIfBackedgeTaken)) {
12372 // If we know exactly how many times the backedge will be taken if it's
12373 // taken at least once, then the backedge count will either be that or
12374 // zero.
12375 MaxBECount = BECountIfBackedgeTaken;
12376 MaxOrZero = true;
12377 } else {
12378 MaxBECount = computeMaxBECountForLT(
12379 Start, Stride, RHS, getTypeSizeInBits(LHS->getType()), IsSigned);
12380 }
12381
12382 if (isa<SCEVCouldNotCompute>(MaxBECount) &&
12383 !isa<SCEVCouldNotCompute>(BECount))
12384 MaxBECount = getConstant(getUnsignedRangeMax(BECount));
12385
12386 return ExitLimit(BECount, MaxBECount, MaxOrZero, Predicates);
12387}
12388
12389ScalarEvolution::ExitLimit
12390ScalarEvolution::howManyGreaterThans(const SCEV *LHS, const SCEV *RHS,
12391 const Loop *L, bool IsSigned,
12392 bool ControlsExit, bool AllowPredicates) {
12393 SmallPtrSet<const SCEVPredicate *, 4> Predicates;
12394 // We handle only IV > Invariant
12395 if (!isLoopInvariant(RHS, L))
12396 return getCouldNotCompute();
12397
12398 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(LHS);
12399 if (!IV && AllowPredicates)
12400 // Try to make this an AddRec using runtime tests, in the first X
12401 // iterations of this loop, where X is the SCEV expression found by the
12402 // algorithm below.
12403 IV = convertSCEVToAddRecWithPredicates(LHS, L, Predicates);
12404
12405 // Avoid weird loops
12406 if (!IV || IV->getLoop() != L || !IV->isAffine())
12407 return getCouldNotCompute();
12408
12409 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
12410 bool NoWrap = ControlsExit && IV->getNoWrapFlags(WrapType);
12411 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
12412
12413 const SCEV *Stride = getNegativeSCEV(IV->getStepRecurrence(*this));
12414
12415 // Avoid negative or zero stride values
12416 if (!isKnownPositive(Stride))
12417 return getCouldNotCompute();
12418
12419 // Avoid proven overflow cases: this will ensure that the backedge taken count
12420 // will not generate any unsigned overflow. Relaxed no-overflow conditions
12421 // exploit NoWrapFlags, allowing to optimize in presence of undefined
12422 // behaviors like the case of C language.
12423 if (!Stride->isOne() && !NoWrap)
12424 if (canIVOverflowOnGT(RHS, Stride, IsSigned))
12425 return getCouldNotCompute();
12426
12427 const SCEV *Start = IV->getStart();
12428 const SCEV *End = RHS;
12429 if (!isLoopEntryGuardedByCond(L, Cond, getAddExpr(Start, Stride), RHS)) {
12430 // If we know that Start >= RHS in the context of loop, then we know that
12431 // min(RHS, Start) = RHS at this point.
12432 if (isLoopEntryGuardedByCond(
12433 L, IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, Start, RHS))
12434 End = RHS;
12435 else
12436 End = IsSigned ? getSMinExpr(RHS, Start) : getUMinExpr(RHS, Start);
12437 }
12438
12439 if (Start->getType()->isPointerTy()) {
12440 Start = getLosslessPtrToIntExpr(Start);
12441 if (isa<SCEVCouldNotCompute>(Start))
12442 return Start;
12443 }
12444 if (End->getType()->isPointerTy()) {
12445 End = getLosslessPtrToIntExpr(End);
12446 if (isa<SCEVCouldNotCompute>(End))
12447 return End;
12448 }
12449
12450 // Compute ((Start - End) + (Stride - 1)) / Stride.
12451 // FIXME: This can overflow. Holding off on fixing this for now;
12452 // howManyGreaterThans will hopefully be gone soon.
12453 const SCEV *One = getOne(Stride->getType());
12454 const SCEV *BECount = getUDivExpr(
12455 getAddExpr(getMinusSCEV(Start, End), getMinusSCEV(Stride, One)), Stride);
12456
12457 APInt MaxStart = IsSigned ? getSignedRangeMax(Start)
12458 : getUnsignedRangeMax(Start);
12459
12460 APInt MinStride = IsSigned ? getSignedRangeMin(Stride)
12461 : getUnsignedRangeMin(Stride);
12462
12463 unsigned BitWidth = getTypeSizeInBits(LHS->getType());
12464 APInt Limit = IsSigned ? APInt::getSignedMinValue(BitWidth) + (MinStride - 1)
12465 : APInt::getMinValue(BitWidth) + (MinStride - 1);
12466
12467 // Although End can be a MIN expression we estimate MinEnd considering only
12468 // the case End = RHS. This is safe because in the other case (Start - End)
12469 // is zero, leading to a zero maximum backedge taken count.
12470 APInt MinEnd =
12471 IsSigned ? APIntOps::smax(getSignedRangeMin(RHS), Limit)
12472 : APIntOps::umax(getUnsignedRangeMin(RHS), Limit);
12473
12474 const SCEV *MaxBECount = isa<SCEVConstant>(BECount)
12475 ? BECount
12476 : getUDivCeilSCEV(getConstant(MaxStart - MinEnd),
12477 getConstant(MinStride));
12478
12479 if (isa<SCEVCouldNotCompute>(MaxBECount))
12480 MaxBECount = BECount;
12481
12482 return ExitLimit(BECount, MaxBECount, false, Predicates);
12483}
12484
12485const SCEV *SCEVAddRecExpr::getNumIterationsInRange(const ConstantRange &Range,
12486 ScalarEvolution &SE) const {
12487 if (Range.isFullSet()) // Infinite loop.
12488 return SE.getCouldNotCompute();
12489
12490 // If the start is a non-zero constant, shift the range to simplify things.
12491 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(getStart()))
12492 if (!SC->getValue()->isZero()) {
12493 SmallVector<const SCEV *, 4> Operands(operands());
12494 Operands[0] = SE.getZero(SC->getType());
12495 const SCEV *Shifted = SE.getAddRecExpr(Operands, getLoop(),
12496 getNoWrapFlags(FlagNW));
12497 if (const auto *ShiftedAddRec = dyn_cast<SCEVAddRecExpr>(Shifted))
12498 return ShiftedAddRec->getNumIterationsInRange(
12499 Range.subtract(SC->getAPInt()), SE);
12500 // This is strange and shouldn't happen.
12501 return SE.getCouldNotCompute();
12502 }
12503
12504 // The only time we can solve this is when we have all constant indices.
12505 // Otherwise, we cannot determine the overflow conditions.
12506 if (any_of(operands(), [](const SCEV *Op) { return !isa<SCEVConstant>(Op); }))
12507 return SE.getCouldNotCompute();
12508
12509 // Okay at this point we know that all elements of the chrec are constants and
12510 // that the start element is zero.
12511
12512 // First check to see if the range contains zero. If not, the first
12513 // iteration exits.
12514 unsigned BitWidth = SE.getTypeSizeInBits(getType());
12515 if (!Range.contains(APInt(BitWidth, 0)))
12516 return SE.getZero(getType());
12517
12518 if (isAffine()) {
12519 // If this is an affine expression then we have this situation:
12520 // Solve {0,+,A} in Range === Ax in Range
12521
12522 // We know that zero is in the range. If A is positive then we know that
12523 // the upper value of the range must be the first possible exit value.
12524 // If A is negative then the lower of the range is the last possible loop
12525 // value. Also note that we already checked for a full range.
12526 APInt A = cast<SCEVConstant>(getOperand(1))->getAPInt();
12527 APInt End = A.sge(1) ? (Range.getUpper() - 1) : Range.getLower();
12528
12529 // The exit value should be (End+A)/A.
12530 APInt ExitVal = (End + A).udiv(A);
12531 ConstantInt *ExitValue = ConstantInt::get(SE.getContext(), ExitVal);
12532
12533 // Evaluate at the exit value. If we really did fall out of the valid
12534 // range, then we computed our trip count, otherwise wrap around or other
12535 // things must have happened.
12536 ConstantInt *Val = EvaluateConstantChrecAtConstant(this, ExitValue, SE);
12537 if (Range.contains(Val->getValue()))
12538 return SE.getCouldNotCompute(); // Something strange happened
12539
12540 // Ensure that the previous value is in the range.
12541 assert(Range.contains((static_cast <bool> (Range.contains( EvaluateConstantChrecAtConstant
(this, ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->
getValue()) && "Linear scev computation is off in a bad way!"
) ? void (0) : __assert_fail ("Range.contains( EvaluateConstantChrecAtConstant(this, ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) && \"Linear scev computation is off in a bad way!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12544, __extension__
__PRETTY_FUNCTION__))
12542 EvaluateConstantChrecAtConstant(this,(static_cast <bool> (Range.contains( EvaluateConstantChrecAtConstant
(this, ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->
getValue()) && "Linear scev computation is off in a bad way!"
) ? void (0) : __assert_fail ("Range.contains( EvaluateConstantChrecAtConstant(this, ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) && \"Linear scev computation is off in a bad way!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12544, __extension__
__PRETTY_FUNCTION__))
12543 ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) &&(static_cast <bool> (Range.contains( EvaluateConstantChrecAtConstant
(this, ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->
getValue()) && "Linear scev computation is off in a bad way!"
) ? void (0) : __assert_fail ("Range.contains( EvaluateConstantChrecAtConstant(this, ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) && \"Linear scev computation is off in a bad way!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12544, __extension__
__PRETTY_FUNCTION__))
12544 "Linear scev computation is off in a bad way!")(static_cast <bool> (Range.contains( EvaluateConstantChrecAtConstant
(this, ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->
getValue()) && "Linear scev computation is off in a bad way!"
) ? void (0) : __assert_fail ("Range.contains( EvaluateConstantChrecAtConstant(this, ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) && \"Linear scev computation is off in a bad way!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12544, __extension__
__PRETTY_FUNCTION__))
;
12545 return SE.getConstant(ExitValue);
12546 }
12547
12548 if (isQuadratic()) {
12549 if (auto S = SolveQuadraticAddRecRange(this, Range, SE))
12550 return SE.getConstant(S.getValue());
12551 }
12552
12553 return SE.getCouldNotCompute();
12554}
12555
12556const SCEVAddRecExpr *
12557SCEVAddRecExpr::getPostIncExpr(ScalarEvolution &SE) const {
12558 assert(getNumOperands() > 1 && "AddRec with zero step?")(static_cast <bool> (getNumOperands() > 1 &&
"AddRec with zero step?") ? void (0) : __assert_fail ("getNumOperands() > 1 && \"AddRec with zero step?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12558, __extension__
__PRETTY_FUNCTION__))
;
12559 // There is a temptation to just call getAddExpr(this, getStepRecurrence(SE)),
12560 // but in this case we cannot guarantee that the value returned will be an
12561 // AddRec because SCEV does not have a fixed point where it stops
12562 // simplification: it is legal to return ({rec1} + {rec2}). For example, it
12563 // may happen if we reach arithmetic depth limit while simplifying. So we
12564 // construct the returned value explicitly.
12565 SmallVector<const SCEV *, 3> Ops;
12566 // If this is {A,+,B,+,C,...,+,N}, then its step is {B,+,C,+,...,+,N}, and
12567 // (this + Step) is {A+B,+,B+C,+...,+,N}.
12568 for (unsigned i = 0, e = getNumOperands() - 1; i < e; ++i)
12569 Ops.push_back(SE.getAddExpr(getOperand(i), getOperand(i + 1)));
12570 // We know that the last operand is not a constant zero (otherwise it would
12571 // have been popped out earlier). This guarantees us that if the result has
12572 // the same last operand, then it will also not be popped out, meaning that
12573 // the returned value will be an AddRec.
12574 const SCEV *Last = getOperand(getNumOperands() - 1);
12575 assert(!Last->isZero() && "Recurrency with zero step?")(static_cast <bool> (!Last->isZero() && "Recurrency with zero step?"
) ? void (0) : __assert_fail ("!Last->isZero() && \"Recurrency with zero step?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12575, __extension__
__PRETTY_FUNCTION__))
;
12576 Ops.push_back(Last);
12577 return cast<SCEVAddRecExpr>(SE.getAddRecExpr(Ops, getLoop(),
12578 SCEV::FlagAnyWrap));
12579}
12580
12581// Return true when S contains at least an undef value.
12582bool ScalarEvolution::containsUndefs(const SCEV *S) const {
12583 return SCEVExprContains(S, [](const SCEV *S) {
12584 if (const auto *SU = dyn_cast<SCEVUnknown>(S))
12585 return isa<UndefValue>(SU->getValue());
12586 return false;
12587 });
12588}
12589
12590/// Return the size of an element read or written by Inst.
12591const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) {
12592 Type *Ty;
12593 if (StoreInst *Store = dyn_cast<StoreInst>(Inst))
12594 Ty = Store->getValueOperand()->getType();
12595 else if (LoadInst *Load = dyn_cast<LoadInst>(Inst))
12596 Ty = Load->getType();
12597 else
12598 return nullptr;
12599
12600 Type *ETy = getEffectiveSCEVType(PointerType::getUnqual(Ty));
12601 return getSizeOfExpr(ETy, Ty);
12602}
12603
12604//===----------------------------------------------------------------------===//
12605// SCEVCallbackVH Class Implementation
12606//===----------------------------------------------------------------------===//
12607
12608void ScalarEvolution::SCEVCallbackVH::deleted() {
12609 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!")(static_cast <bool> (SE && "SCEVCallbackVH called with a null ScalarEvolution!"
) ? void (0) : __assert_fail ("SE && \"SCEVCallbackVH called with a null ScalarEvolution!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12609, __extension__
__PRETTY_FUNCTION__))
;
12610 if (PHINode *PN = dyn_cast<PHINode>(getValPtr()))
12611 SE->ConstantEvolutionLoopExitValue.erase(PN);
12612 SE->eraseValueFromMap(getValPtr());
12613 // this now dangles!
12614}
12615
12616void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) {
12617 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!")(static_cast <bool> (SE && "SCEVCallbackVH called with a null ScalarEvolution!"
) ? void (0) : __assert_fail ("SE && \"SCEVCallbackVH called with a null ScalarEvolution!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12617, __extension__
__PRETTY_FUNCTION__))
;
12618
12619 // Forget all the expressions associated with users of the old value,
12620 // so that future queries will recompute the expressions using the new
12621 // value.
12622 Value *Old = getValPtr();
12623 SmallVector<User *, 16> Worklist(Old->users());
12624 SmallPtrSet<User *, 8> Visited;
12625 while (!Worklist.empty()) {
12626 User *U = Worklist.pop_back_val();
12627 // Deleting the Old value will cause this to dangle. Postpone
12628 // that until everything else is done.
12629 if (U == Old)
12630 continue;
12631 if (!Visited.insert(U).second)
12632 continue;
12633 if (PHINode *PN = dyn_cast<PHINode>(U))
12634 SE->ConstantEvolutionLoopExitValue.erase(PN);
12635 SE->eraseValueFromMap(U);
12636 llvm::append_range(Worklist, U->users());
12637 }
12638 // Delete the Old value.
12639 if (PHINode *PN = dyn_cast<PHINode>(Old))
12640 SE->ConstantEvolutionLoopExitValue.erase(PN);
12641 SE->eraseValueFromMap(Old);
12642 // this now dangles!
12643}
12644
12645ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se)
12646 : CallbackVH(V), SE(se) {}
12647
12648//===----------------------------------------------------------------------===//
12649// ScalarEvolution Class Implementation
12650//===----------------------------------------------------------------------===//
12651
12652ScalarEvolution::ScalarEvolution(Function &F, TargetLibraryInfo &TLI,
12653 AssumptionCache &AC, DominatorTree &DT,
12654 LoopInfo &LI)
12655 : F(F), TLI(TLI), AC(AC), DT(DT), LI(LI),
12656 CouldNotCompute(new SCEVCouldNotCompute()), ValuesAtScopes(64),
12657 LoopDispositions(64), BlockDispositions(64) {
12658 // To use guards for proving predicates, we need to scan every instruction in
12659 // relevant basic blocks, and not just terminators. Doing this is a waste of
12660 // time if the IR does not actually contain any calls to
12661 // @llvm.experimental.guard, so do a quick check and remember this beforehand.
12662 //
12663 // This pessimizes the case where a pass that preserves ScalarEvolution wants
12664 // to _add_ guards to the module when there weren't any before, and wants
12665 // ScalarEvolution to optimize based on those guards. For now we prefer to be
12666 // efficient in lieu of being smart in that rather obscure case.
12667
12668 auto *GuardDecl = F.getParent()->getFunction(
12669 Intrinsic::getName(Intrinsic::experimental_guard));
12670 HasGuards = GuardDecl && !GuardDecl->use_empty();
12671}
12672
12673ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg)
12674 : F(Arg.F), HasGuards(Arg.HasGuards), TLI(Arg.TLI), AC(Arg.AC), DT(Arg.DT),
12675 LI(Arg.LI), CouldNotCompute(std::move(Arg.CouldNotCompute)),
12676 ValueExprMap(std::move(Arg.ValueExprMap)),
12677 PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)),
12678 PendingPhiRanges(std::move(Arg.PendingPhiRanges)),
12679 PendingMerges(std::move(Arg.PendingMerges)),
12680 MinTrailingZerosCache(std::move(Arg.MinTrailingZerosCache)),
12681 BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)),
12682 PredicatedBackedgeTakenCounts(
12683 std::move(Arg.PredicatedBackedgeTakenCounts)),
12684 BECountUsers(std::move(Arg.BECountUsers)),
12685 ConstantEvolutionLoopExitValue(
12686 std::move(Arg.ConstantEvolutionLoopExitValue)),
12687 ValuesAtScopes(std::move(Arg.ValuesAtScopes)),
12688 ValuesAtScopesUsers(std::move(Arg.ValuesAtScopesUsers)),
12689 LoopDispositions(std::move(Arg.LoopDispositions)),
12690 LoopPropertiesCache(std::move(Arg.LoopPropertiesCache)),
12691 BlockDispositions(std::move(Arg.BlockDispositions)),
12692 SCEVUsers(std::move(Arg.SCEVUsers)),
12693 UnsignedRanges(std::move(Arg.UnsignedRanges)),
12694 SignedRanges(std::move(Arg.SignedRanges)),
12695 UniqueSCEVs(std::move(Arg.UniqueSCEVs)),
12696 UniquePreds(std::move(Arg.UniquePreds)),
12697 SCEVAllocator(std::move(Arg.SCEVAllocator)),
12698 LoopUsers(std::move(Arg.LoopUsers)),
12699 PredicatedSCEVRewrites(std::move(Arg.PredicatedSCEVRewrites)),
12700 FirstUnknown(Arg.FirstUnknown) {
12701 Arg.FirstUnknown = nullptr;
12702}
12703
12704ScalarEvolution::~ScalarEvolution() {
12705 // Iterate through all the SCEVUnknown instances and call their
12706 // destructors, so that they release their references to their values.
12707 for (SCEVUnknown *U = FirstUnknown; U;) {
12708 SCEVUnknown *Tmp = U;
12709 U = U->Next;
12710 Tmp->~SCEVUnknown();
12711 }
12712 FirstUnknown = nullptr;
12713
12714 ExprValueMap.clear();
12715 ValueExprMap.clear();
12716 HasRecMap.clear();
12717 BackedgeTakenCounts.clear();
12718 PredicatedBackedgeTakenCounts.clear();
12719
12720 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage")(static_cast <bool> (PendingLoopPredicates.empty() &&
"isImpliedCond garbage") ? void (0) : __assert_fail ("PendingLoopPredicates.empty() && \"isImpliedCond garbage\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12720, __extension__
__PRETTY_FUNCTION__))
;
12721 assert(PendingPhiRanges.empty() && "getRangeRef garbage")(static_cast <bool> (PendingPhiRanges.empty() &&
"getRangeRef garbage") ? void (0) : __assert_fail ("PendingPhiRanges.empty() && \"getRangeRef garbage\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12721, __extension__
__PRETTY_FUNCTION__))
;
12722 assert(PendingMerges.empty() && "isImpliedViaMerge garbage")(static_cast <bool> (PendingMerges.empty() && "isImpliedViaMerge garbage"
) ? void (0) : __assert_fail ("PendingMerges.empty() && \"isImpliedViaMerge garbage\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12722, __extension__
__PRETTY_FUNCTION__))
;
12723 assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!")(static_cast <bool> (!WalkingBEDominatingConds &&
"isLoopBackedgeGuardedByCond garbage!") ? void (0) : __assert_fail
("!WalkingBEDominatingConds && \"isLoopBackedgeGuardedByCond garbage!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12723, __extension__
__PRETTY_FUNCTION__))
;
12724 assert(!ProvingSplitPredicate && "ProvingSplitPredicate garbage!")(static_cast <bool> (!ProvingSplitPredicate && "ProvingSplitPredicate garbage!"
) ? void (0) : __assert_fail ("!ProvingSplitPredicate && \"ProvingSplitPredicate garbage!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12724, __extension__
__PRETTY_FUNCTION__))
;
12725}
12726
12727bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) {
12728 return !isa<SCEVCouldNotCompute>(getBackedgeTakenCount(L));
12729}
12730
12731static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE,
12732 const Loop *L) {
12733 // Print all inner loops first
12734 for (Loop *I : *L)
12735 PrintLoopInfo(OS, SE, I);
12736
12737 OS << "Loop ";
12738 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
12739 OS << ": ";
12740
12741 SmallVector<BasicBlock *, 8> ExitingBlocks;
12742 L->getExitingBlocks(ExitingBlocks);
12743 if (ExitingBlocks.size() != 1)
12744 OS << "<multiple exits> ";
12745
12746 if (SE->hasLoopInvariantBackedgeTakenCount(L))
12747 OS << "backedge-taken count is " << *SE->getBackedgeTakenCount(L) << "\n";
12748 else
12749 OS << "Unpredictable backedge-taken count.\n";
12750
12751 if (ExitingBlocks.size() > 1)
12752 for (BasicBlock *ExitingBlock : ExitingBlocks) {
12753 OS << " exit count for " << ExitingBlock->getName() << ": "
12754 << *SE->getExitCount(L, ExitingBlock) << "\n";
12755 }
12756
12757 OS << "Loop ";
12758 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
12759 OS << ": ";
12760
12761 if (!isa<SCEVCouldNotCompute>(SE->getConstantMaxBackedgeTakenCount(L))) {
12762 OS << "max backedge-taken count is " << *SE->getConstantMaxBackedgeTakenCount(L);
12763 if (SE->isBackedgeTakenCountMaxOrZero(L))
12764 OS << ", actual taken count either this or zero.";
12765 } else {
12766 OS << "Unpredictable max backedge-taken count. ";
12767 }
12768
12769 OS << "\n"
12770 "Loop ";
12771 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
12772 OS << ": ";
12773
12774 SCEVUnionPredicate Pred;
12775 auto PBT = SE->getPredicatedBackedgeTakenCount(L, Pred);
12776 if (!isa<SCEVCouldNotCompute>(PBT)) {
12777 OS << "Predicated backedge-taken count is " << *PBT << "\n";
12778 OS << " Predicates:\n";
12779 Pred.print(OS, 4);
12780 } else {
12781 OS << "Unpredictable predicated backedge-taken count. ";
12782 }
12783 OS << "\n";
12784
12785 if (SE->hasLoopInvariantBackedgeTakenCount(L)) {
12786 OS << "Loop ";
12787 L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
12788 OS << ": ";
12789 OS << "Trip multiple is " << SE->getSmallConstantTripMultiple(L) << "\n";
12790 }
12791}
12792
12793static StringRef loopDispositionToStr(ScalarEvolution::LoopDisposition LD) {
12794 switch (LD) {
12795 case ScalarEvolution::LoopVariant:
12796 return "Variant";
12797 case ScalarEvolution::LoopInvariant:
12798 return "Invariant";
12799 case ScalarEvolution::LoopComputable:
12800 return "Computable";
12801 }
12802 llvm_unreachable("Unknown ScalarEvolution::LoopDisposition kind!")::llvm::llvm_unreachable_internal("Unknown ScalarEvolution::LoopDisposition kind!"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12802)
;
12803}
12804
12805void ScalarEvolution::print(raw_ostream &OS) const {
12806 // ScalarEvolution's implementation of the print method is to print
12807 // out SCEV values of all instructions that are interesting. Doing
12808 // this potentially causes it to create new SCEV objects though,
12809 // which technically conflicts with the const qualifier. This isn't
12810 // observable from outside the class though, so casting away the
12811 // const isn't dangerous.
12812 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
12813
12814 if (ClassifyExpressions) {
12815 OS << "Classifying expressions for: ";
12816 F.printAsOperand(OS, /*PrintType=*/false);
12817 OS << "\n";
12818 for (Instruction &I : instructions(F))
12819 if (isSCEVable(I.getType()) && !isa<CmpInst>(I)) {
12820 OS << I << '\n';
12821 OS << " --> ";
12822 const SCEV *SV = SE.getSCEV(&I);
12823 SV->print(OS);
12824 if (!isa<SCEVCouldNotCompute>(SV)) {
12825 OS << " U: ";
12826 SE.getUnsignedRange(SV).print(OS);
12827 OS << " S: ";
12828 SE.getSignedRange(SV).print(OS);
12829 }
12830
12831 const Loop *L = LI.getLoopFor(I.getParent());
12832
12833 const SCEV *AtUse = SE.getSCEVAtScope(SV, L);
12834 if (AtUse != SV) {
12835 OS << " --> ";
12836 AtUse->print(OS);
12837 if (!isa<SCEVCouldNotCompute>(AtUse)) {
12838 OS << " U: ";
12839 SE.getUnsignedRange(AtUse).print(OS);
12840 OS << " S: ";
12841 SE.getSignedRange(AtUse).print(OS);
12842 }
12843 }
12844
12845 if (L) {
12846 OS << "\t\t" "Exits: ";
12847 const SCEV *ExitValue = SE.getSCEVAtScope(SV, L->getParentLoop());
12848 if (!SE.isLoopInvariant(ExitValue, L)) {
12849 OS << "<<Unknown>>";
12850 } else {
12851 OS << *ExitValue;
12852 }
12853
12854 bool First = true;
12855 for (auto *Iter = L; Iter; Iter = Iter->getParentLoop()) {
12856 if (First) {
12857 OS << "\t\t" "LoopDispositions: { ";
12858 First = false;
12859 } else {
12860 OS << ", ";
12861 }
12862
12863 Iter->getHeader()->printAsOperand(OS, /*PrintType=*/false);
12864 OS << ": " << loopDispositionToStr(SE.getLoopDisposition(SV, Iter));
12865 }
12866
12867 for (auto *InnerL : depth_first(L)) {
12868 if (InnerL == L)
12869 continue;
12870 if (First) {
12871 OS << "\t\t" "LoopDispositions: { ";
12872 First = false;
12873 } else {
12874 OS << ", ";
12875 }
12876
12877 InnerL->getHeader()->printAsOperand(OS, /*PrintType=*/false);
12878 OS << ": " << loopDispositionToStr(SE.getLoopDisposition(SV, InnerL));
12879 }
12880
12881 OS << " }";
12882 }
12883
12884 OS << "\n";
12885 }
12886 }
12887
12888 OS << "Determining loop execution counts for: ";
12889 F.printAsOperand(OS, /*PrintType=*/false);
12890 OS << "\n";
12891 for (Loop *I : LI)
12892 PrintLoopInfo(OS, &SE, I);
12893}
12894
12895ScalarEvolution::LoopDisposition
12896ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) {
12897 auto &Values = LoopDispositions[S];
12898 for (auto &V : Values) {
12899 if (V.getPointer() == L)
12900 return V.getInt();
12901 }
12902 Values.emplace_back(L, LoopVariant);
12903 LoopDisposition D = computeLoopDisposition(S, L);
12904 auto &Values2 = LoopDispositions[S];
12905 for (auto &V : llvm::reverse(Values2)) {
12906 if (V.getPointer() == L) {
12907 V.setInt(D);
12908 break;
12909 }
12910 }
12911 return D;
12912}
12913
12914ScalarEvolution::LoopDisposition
12915ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) {
12916 switch (S->getSCEVType()) {
12917 case scConstant:
12918 return LoopInvariant;
12919 case scPtrToInt:
12920 case scTruncate:
12921 case scZeroExtend:
12922 case scSignExtend:
12923 return getLoopDisposition(cast<SCEVCastExpr>(S)->getOperand(), L);
12924 case scAddRecExpr: {
12925 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S);
12926
12927 // If L is the addrec's loop, it's computable.
12928 if (AR->getLoop() == L)
12929 return LoopComputable;
12930
12931 // Add recurrences are never invariant in the function-body (null loop).
12932 if (!L)
12933 return LoopVariant;
12934
12935 // Everything that is not defined at loop entry is variant.
12936 if (DT.dominates(L->getHeader(), AR->getLoop()->getHeader()))
12937 return LoopVariant;
12938 assert(!L->contains(AR->getLoop()) && "Containing loop's header does not"(static_cast <bool> (!L->contains(AR->getLoop()) &&
"Containing loop's header does not" " dominate the contained loop's header?"
) ? void (0) : __assert_fail ("!L->contains(AR->getLoop()) && \"Containing loop's header does not\" \" dominate the contained loop's header?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12939, __extension__
__PRETTY_FUNCTION__))
12939 " dominate the contained loop's header?")(static_cast <bool> (!L->contains(AR->getLoop()) &&
"Containing loop's header does not" " dominate the contained loop's header?"
) ? void (0) : __assert_fail ("!L->contains(AR->getLoop()) && \"Containing loop's header does not\" \" dominate the contained loop's header?\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12939, __extension__
__PRETTY_FUNCTION__))
;
12940
12941 // This recurrence is invariant w.r.t. L if AR's loop contains L.
12942 if (AR->getLoop()->contains(L))
12943 return LoopInvariant;
12944
12945 // This recurrence is variant w.r.t. L if any of its operands
12946 // are variant.
12947 for (auto *Op : AR->operands())
12948 if (!isLoopInvariant(Op, L))
12949 return LoopVariant;
12950
12951 // Otherwise it's loop-invariant.
12952 return LoopInvariant;
12953 }
12954 case scAddExpr:
12955 case scMulExpr:
12956 case scUMaxExpr:
12957 case scSMaxExpr:
12958 case scUMinExpr:
12959 case scSMinExpr:
12960 case scSequentialUMinExpr: {
12961 bool HasVarying = false;
12962 for (auto *Op : cast<SCEVNAryExpr>(S)->operands()) {
12963 LoopDisposition D = getLoopDisposition(Op, L);
12964 if (D == LoopVariant)
12965 return LoopVariant;
12966 if (D == LoopComputable)
12967 HasVarying = true;
12968 }
12969 return HasVarying ? LoopComputable : LoopInvariant;
12970 }
12971 case scUDivExpr: {
12972 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S);
12973 LoopDisposition LD = getLoopDisposition(UDiv->getLHS(), L);
12974 if (LD == LoopVariant)
12975 return LoopVariant;
12976 LoopDisposition RD = getLoopDisposition(UDiv->getRHS(), L);
12977 if (RD == LoopVariant)
12978 return LoopVariant;
12979 return (LD == LoopInvariant && RD == LoopInvariant) ?
12980 LoopInvariant : LoopComputable;
12981 }
12982 case scUnknown:
12983 // All non-instruction values are loop invariant. All instructions are loop
12984 // invariant if they are not contained in the specified loop.
12985 // Instructions are never considered invariant in the function body
12986 // (null loop) because they are defined within the "loop".
12987 if (auto *I = dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue()))
12988 return (L && !L->contains(I)) ? LoopInvariant : LoopVariant;
12989 return LoopInvariant;
12990 case scCouldNotCompute:
12991 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!")::llvm::llvm_unreachable_internal("Attempt to use a SCEVCouldNotCompute object!"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 12991)
;
12992 }
12993 llvm_unreachable("Unknown SCEV kind!")::llvm::llvm_unreachable_internal("Unknown SCEV kind!", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 12993)
;
12994}
12995
12996bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) {
12997 return getLoopDisposition(S, L) == LoopInvariant;
12998}
12999
13000bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) {
13001 return getLoopDisposition(S, L) == LoopComputable;
13002}
13003
13004ScalarEvolution::BlockDisposition
13005ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) {
13006 auto &Values = BlockDispositions[S];
13007 for (auto &V : Values) {
13008 if (V.getPointer() == BB)
13009 return V.getInt();
13010 }
13011 Values.emplace_back(BB, DoesNotDominateBlock);
13012 BlockDisposition D = computeBlockDisposition(S, BB);
13013 auto &Values2 = BlockDispositions[S];
13014 for (auto &V : llvm::reverse(Values2)) {
13015 if (V.getPointer() == BB) {
13016 V.setInt(D);
13017 break;
13018 }
13019 }
13020 return D;
13021}
13022
13023ScalarEvolution::BlockDisposition
13024ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) {
13025 switch (S->getSCEVType()) {
13026 case scConstant:
13027 return ProperlyDominatesBlock;
13028 case scPtrToInt:
13029 case scTruncate:
13030 case scZeroExtend:
13031 case scSignExtend:
13032 return getBlockDisposition(cast<SCEVCastExpr>(S)->getOperand(), BB);
13033 case scAddRecExpr: {
13034 // This uses a "dominates" query instead of "properly dominates" query
13035 // to test for proper dominance too, because the instruction which
13036 // produces the addrec's value is a PHI, and a PHI effectively properly
13037 // dominates its entire containing block.
13038 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S);
13039 if (!DT.dominates(AR->getLoop()->getHeader(), BB))
13040 return DoesNotDominateBlock;
13041
13042 // Fall through into SCEVNAryExpr handling.
13043 LLVM_FALLTHROUGH[[gnu::fallthrough]];
13044 }
13045 case scAddExpr:
13046 case scMulExpr:
13047 case scUMaxExpr:
13048 case scSMaxExpr:
13049 case scUMinExpr:
13050 case scSMinExpr:
13051 case scSequentialUMinExpr: {
13052 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(S);
13053 bool Proper = true;
13054 for (const SCEV *NAryOp : NAry->operands()) {
13055 BlockDisposition D = getBlockDisposition(NAryOp, BB);
13056 if (D == DoesNotDominateBlock)
13057 return DoesNotDominateBlock;
13058 if (D == DominatesBlock)
13059 Proper = false;
13060 }
13061 return Proper ? ProperlyDominatesBlock : DominatesBlock;
13062 }
13063 case scUDivExpr: {
13064 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S);
13065 const SCEV *LHS = UDiv->getLHS(), *RHS = UDiv->getRHS();
13066 BlockDisposition LD = getBlockDisposition(LHS, BB);
13067 if (LD == DoesNotDominateBlock)
13068 return DoesNotDominateBlock;
13069 BlockDisposition RD = getBlockDisposition(RHS, BB);
13070 if (RD == DoesNotDominateBlock)
13071 return DoesNotDominateBlock;
13072 return (LD == ProperlyDominatesBlock && RD == ProperlyDominatesBlock) ?
13073 ProperlyDominatesBlock : DominatesBlock;
13074 }
13075 case scUnknown:
13076 if (Instruction *I =
13077 dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) {
13078 if (I->getParent() == BB)
13079 return DominatesBlock;
13080 if (DT.properlyDominates(I->getParent(), BB))
13081 return ProperlyDominatesBlock;
13082 return DoesNotDominateBlock;
13083 }
13084 return ProperlyDominatesBlock;
13085 case scCouldNotCompute:
13086 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!")::llvm::llvm_unreachable_internal("Attempt to use a SCEVCouldNotCompute object!"
, "llvm/lib/Analysis/ScalarEvolution.cpp", 13086)
;
13087 }
13088 llvm_unreachable("Unknown SCEV kind!")::llvm::llvm_unreachable_internal("Unknown SCEV kind!", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 13088)
;
13089}
13090
13091bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) {
13092 return getBlockDisposition(S, BB) >= DominatesBlock;
13093}
13094
13095bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) {
13096 return getBlockDisposition(S, BB) == ProperlyDominatesBlock;
13097}
13098
13099bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const {
13100 return SCEVExprContains(S, [&](const SCEV *Expr) { return Expr == Op; });
13101}
13102
13103void ScalarEvolution::forgetBackedgeTakenCounts(const Loop *L,
13104 bool Predicated) {
13105 auto &BECounts =
13106 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
13107 auto It = BECounts.find(L);
13108 if (It != BECounts.end()) {
13109 for (const ExitNotTakenInfo &ENT : It->second.ExitNotTaken) {
13110 if (!isa<SCEVConstant>(ENT.ExactNotTaken)) {
13111 auto UserIt = BECountUsers.find(ENT.ExactNotTaken);
13112 assert(UserIt != BECountUsers.end())(static_cast <bool> (UserIt != BECountUsers.end()) ? void
(0) : __assert_fail ("UserIt != BECountUsers.end()", "llvm/lib/Analysis/ScalarEvolution.cpp"
, 13112, __extension__ __PRETTY_FUNCTION__))
;
13113 UserIt->second.erase({L, Predicated});
13114 }
13115 }
13116 BECounts.erase(It);
13117 }
13118}
13119
13120void ScalarEvolution::forgetMemoizedResults(ArrayRef<const SCEV *> SCEVs) {
13121 SmallPtrSet<const SCEV *, 8> ToForget(SCEVs.begin(), SCEVs.end());
13122 SmallVector<const SCEV *, 8> Worklist(ToForget.begin(), ToForget.end());
13123
13124 while (!Worklist.empty()) {
13125 const SCEV *Curr = Worklist.pop_back_val();
13126 auto Users = SCEVUsers.find(Curr);
13127 if (Users != SCEVUsers.end())
13128 for (auto *User : Users->second)
13129 if (ToForget.insert(User).second)
13130 Worklist.push_back(User);
13131 }
13132
13133 for (auto *S : ToForget)
13134 forgetMemoizedResultsImpl(S);
13135
13136 for (auto I = PredicatedSCEVRewrites.begin();
13137 I != PredicatedSCEVRewrites.end();) {
13138 std::pair<const SCEV *, const Loop *> Entry = I->first;
13139 if (ToForget.count(Entry.first))
13140 PredicatedSCEVRewrites.erase(I++);
13141 else
13142 ++I;
13143 }
13144}
13145
13146void ScalarEvolution::forgetMemoizedResultsImpl(const SCEV *S) {
13147 LoopDispositions.erase(S);
13148 BlockDispositions.erase(S);
13149 UnsignedRanges.erase(S);
13150 SignedRanges.erase(S);
13151 HasRecMap.erase(S);
13152 MinTrailingZerosCache.erase(S);
13153
13154 auto ExprIt = ExprValueMap.find(S);
13155 if (ExprIt != ExprValueMap.end()) {
13156 for (auto &ValueAndOffset : ExprIt->second) {
13157 if (ValueAndOffset.second == nullptr) {
13158 auto ValueIt = ValueExprMap.find_as(ValueAndOffset.first);
13159 if (ValueIt != ValueExprMap.end())
13160 ValueExprMap.erase(ValueIt);
13161 }
13162 }
13163 ExprValueMap.erase(ExprIt);
13164 }
13165
13166 auto ScopeIt = ValuesAtScopes.find(S);
13167 if (ScopeIt != ValuesAtScopes.end()) {
13168 for (const auto &Pair : ScopeIt->second)
13169 if (!isa_and_nonnull<SCEVConstant>(Pair.second))
13170 erase_value(ValuesAtScopesUsers[Pair.second],
13171 std::make_pair(Pair.first, S));
13172 ValuesAtScopes.erase(ScopeIt);
13173 }
13174
13175 auto ScopeUserIt = ValuesAtScopesUsers.find(S);
13176 if (ScopeUserIt != ValuesAtScopesUsers.end()) {
13177 for (const auto &Pair : ScopeUserIt->second)
13178 erase_value(ValuesAtScopes[Pair.second], std::make_pair(Pair.first, S));
13179 ValuesAtScopesUsers.erase(ScopeUserIt);
13180 }
13181
13182 auto BEUsersIt = BECountUsers.find(S);
13183 if (BEUsersIt != BECountUsers.end()) {
13184 // Work on a copy, as forgetBackedgeTakenCounts() will modify the original.
13185 auto Copy = BEUsersIt->second;
13186 for (const auto &Pair : Copy)
13187 forgetBackedgeTakenCounts(Pair.getPointer(), Pair.getInt());
13188 BECountUsers.erase(BEUsersIt);
13189 }
13190}
13191
13192void
13193ScalarEvolution::getUsedLoops(const SCEV *S,
13194 SmallPtrSetImpl<const Loop *> &LoopsUsed) {
13195 struct FindUsedLoops {
13196 FindUsedLoops(SmallPtrSetImpl<const Loop *> &LoopsUsed)
13197 : LoopsUsed(LoopsUsed) {}
13198 SmallPtrSetImpl<const Loop *> &LoopsUsed;
13199 bool follow(const SCEV *S) {
13200 if (auto *AR = dyn_cast<SCEVAddRecExpr>(S))
13201 LoopsUsed.insert(AR->getLoop());
13202 return true;
13203 }
13204
13205 bool isDone() const { return false; }
13206 };
13207
13208 FindUsedLoops F(LoopsUsed);
13209 SCEVTraversal<FindUsedLoops>(F).visitAll(S);
13210}
13211
13212void ScalarEvolution::verify() const {
13213 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
13214 ScalarEvolution SE2(F, TLI, AC, DT, LI);
13215
13216 SmallVector<Loop *, 8> LoopStack(LI.begin(), LI.end());
13217
13218 // Map's SCEV expressions from one ScalarEvolution "universe" to another.
13219 struct SCEVMapper : public SCEVRewriteVisitor<SCEVMapper> {
13220 SCEVMapper(ScalarEvolution &SE) : SCEVRewriteVisitor<SCEVMapper>(SE) {}
13221
13222 const SCEV *visitConstant(const SCEVConstant *Constant) {
13223 return SE.getConstant(Constant->getAPInt());
13224 }
13225
13226 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
13227 return SE.getUnknown(Expr->getValue());
13228 }
13229
13230 const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) {
13231 return SE.getCouldNotCompute();
13232 }
13233 };
13234
13235 SCEVMapper SCM(SE2);
13236
13237 while (!LoopStack.empty()) {
13238 auto *L = LoopStack.pop_back_val();
13239 llvm::append_range(LoopStack, *L);
13240
13241 auto *CurBECount = SCM.visit(
13242 const_cast<ScalarEvolution *>(this)->getBackedgeTakenCount(L));
13243 auto *NewBECount = SE2.getBackedgeTakenCount(L);
13244
13245 if (CurBECount == SE2.getCouldNotCompute() ||
13246 NewBECount == SE2.getCouldNotCompute()) {
13247 // NB! This situation is legal, but is very suspicious -- whatever pass
13248 // change the loop to make a trip count go from could not compute to
13249 // computable or vice-versa *should have* invalidated SCEV. However, we
13250 // choose not to assert here (for now) since we don't want false
13251 // positives.
13252 continue;
13253 }
13254
13255 if (containsUndefs(CurBECount) || containsUndefs(NewBECount)) {
13256 // SCEV treats "undef" as an unknown but consistent value (i.e. it does
13257 // not propagate undef aggressively). This means we can (and do) fail
13258 // verification in cases where a transform makes the trip count of a loop
13259 // go from "undef" to "undef+1" (say). The transform is fine, since in
13260 // both cases the loop iterates "undef" times, but SCEV thinks we
13261 // increased the trip count of the loop by 1 incorrectly.
13262 continue;
13263 }
13264
13265 if (SE.getTypeSizeInBits(CurBECount->getType()) >
13266 SE.getTypeSizeInBits(NewBECount->getType()))
13267 NewBECount = SE2.getZeroExtendExpr(NewBECount, CurBECount->getType());
13268 else if (SE.getTypeSizeInBits(CurBECount->getType()) <
13269 SE.getTypeSizeInBits(NewBECount->getType()))
13270 CurBECount = SE2.getZeroExtendExpr(CurBECount, NewBECount->getType());
13271
13272 const SCEV *Delta = SE2.getMinusSCEV(CurBECount, NewBECount);
13273
13274 // Unless VerifySCEVStrict is set, we only compare constant deltas.
13275 if ((VerifySCEVStrict || isa<SCEVConstant>(Delta)) && !Delta->isZero()) {
13276 dbgs() << "Trip Count for " << *L << " Changed!\n";
13277 dbgs() << "Old: " << *CurBECount << "\n";
13278 dbgs() << "New: " << *NewBECount << "\n";
13279 dbgs() << "Delta: " << *Delta << "\n";
13280 std::abort();
13281 }
13282 }
13283
13284 // Collect all valid loops currently in LoopInfo.
13285 SmallPtrSet<Loop *, 32> ValidLoops;
13286 SmallVector<Loop *, 32> Worklist(LI.begin(), LI.end());
13287 while (!Worklist.empty()) {
13288 Loop *L = Worklist.pop_back_val();
13289 if (ValidLoops.contains(L))
13290 continue;
13291 ValidLoops.insert(L);
13292 Worklist.append(L->begin(), L->end());
13293 }
13294 for (auto &KV : ValueExprMap) {
13295#ifndef NDEBUG
13296 // Check for SCEV expressions referencing invalid/deleted loops.
13297 if (auto *AR = dyn_cast<SCEVAddRecExpr>(KV.second)) {
13298 assert(ValidLoops.contains(AR->getLoop()) &&(static_cast <bool> (ValidLoops.contains(AR->getLoop
()) && "AddRec references invalid loop") ? void (0) :
__assert_fail ("ValidLoops.contains(AR->getLoop()) && \"AddRec references invalid loop\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 13299, __extension__
__PRETTY_FUNCTION__))
13299 "AddRec references invalid loop")(static_cast <bool> (ValidLoops.contains(AR->getLoop
()) && "AddRec references invalid loop") ? void (0) :
__assert_fail ("ValidLoops.contains(AR->getLoop()) && \"AddRec references invalid loop\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 13299, __extension__
__PRETTY_FUNCTION__))
;
13300 }
13301#endif
13302
13303 // Check that the value is also part of the reverse map.
13304 auto It = ExprValueMap.find(KV.second);
13305 if (It == ExprValueMap.end() || !It->second.contains({KV.first, nullptr})) {
13306 dbgs() << "Value " << *KV.first
13307 << " is in ValueExprMap but not in ExprValueMap\n";
13308 std::abort();
13309 }
13310 }
13311
13312 for (const auto &KV : ExprValueMap) {
13313 for (const auto &ValueAndOffset : KV.second) {
13314 if (ValueAndOffset.second != nullptr)
13315 continue;
13316
13317 auto It = ValueExprMap.find_as(ValueAndOffset.first);
13318 if (It == ValueExprMap.end()) {
13319 dbgs() << "Value " << *ValueAndOffset.first
13320 << " is in ExprValueMap but not in ValueExprMap\n";
13321 std::abort();
13322 }
13323 if (It->second != KV.first) {
13324 dbgs() << "Value " << *ValueAndOffset.first
13325 << " mapped to " << *It->second
13326 << " rather than " << *KV.first << "\n";
13327 std::abort();
13328 }
13329 }
13330 }
13331
13332 // Verify integrity of SCEV users.
13333 for (const auto &S : UniqueSCEVs) {
13334 SmallVector<const SCEV *, 4> Ops;
13335 collectUniqueOps(&S, Ops);
13336 for (const auto *Op : Ops) {
13337 // We do not store dependencies of constants.
13338 if (isa<SCEVConstant>(Op))
13339 continue;
13340 auto It = SCEVUsers.find(Op);
13341 if (It != SCEVUsers.end() && It->second.count(&S))
13342 continue;
13343 dbgs() << "Use of operand " << *Op << " by user " << S
13344 << " is not being tracked!\n";
13345 std::abort();
13346 }
13347 }
13348
13349 // Verify integrity of ValuesAtScopes users.
13350 for (const auto &ValueAndVec : ValuesAtScopes) {
13351 const SCEV *Value = ValueAndVec.first;
13352 for (const auto &LoopAndValueAtScope : ValueAndVec.second) {
13353 const Loop *L = LoopAndValueAtScope.first;
13354 const SCEV *ValueAtScope = LoopAndValueAtScope.second;
13355 if (!isa<SCEVConstant>(ValueAtScope)) {
13356 auto It = ValuesAtScopesUsers.find(ValueAtScope);
13357 if (It != ValuesAtScopesUsers.end() &&
13358 is_contained(It->second, std::make_pair(L, Value)))
13359 continue;
13360 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
13361 << *ValueAtScope << " missing in ValuesAtScopesUsers\n";
13362 std::abort();
13363 }
13364 }
13365 }
13366
13367 for (const auto &ValueAtScopeAndVec : ValuesAtScopesUsers) {
13368 const SCEV *ValueAtScope = ValueAtScopeAndVec.first;
13369 for (const auto &LoopAndValue : ValueAtScopeAndVec.second) {
13370 const Loop *L = LoopAndValue.first;
13371 const SCEV *Value = LoopAndValue.second;
13372 assert(!isa<SCEVConstant>(Value))(static_cast <bool> (!isa<SCEVConstant>(Value)) ?
void (0) : __assert_fail ("!isa<SCEVConstant>(Value)",
"llvm/lib/Analysis/ScalarEvolution.cpp", 13372, __extension__
__PRETTY_FUNCTION__))
;
13373 auto It = ValuesAtScopes.find(Value);
13374 if (It != ValuesAtScopes.end() &&
13375 is_contained(It->second, std::make_pair(L, ValueAtScope)))
13376 continue;
13377 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
13378 << *ValueAtScope << " missing in ValuesAtScopes\n";
13379 std::abort();
13380 }
13381 }
13382
13383 // Verify integrity of BECountUsers.
13384 auto VerifyBECountUsers = [&](bool Predicated) {
13385 auto &BECounts =
13386 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
13387 for (const auto &LoopAndBEInfo : BECounts) {
13388 for (const ExitNotTakenInfo &ENT : LoopAndBEInfo.second.ExitNotTaken) {
13389 if (!isa<SCEVConstant>(ENT.ExactNotTaken)) {
13390 auto UserIt = BECountUsers.find(ENT.ExactNotTaken);
13391 if (UserIt != BECountUsers.end() &&
13392 UserIt->second.contains({ LoopAndBEInfo.first, Predicated }))
13393 continue;
13394 dbgs() << "Value " << *ENT.ExactNotTaken << " for loop "
13395 << *LoopAndBEInfo.first << " missing from BECountUsers\n";
13396 std::abort();
13397 }
13398 }
13399 }
13400 };
13401 VerifyBECountUsers(/* Predicated */ false);
13402 VerifyBECountUsers(/* Predicated */ true);
13403}
13404
13405bool ScalarEvolution::invalidate(
13406 Function &F, const PreservedAnalyses &PA,
13407 FunctionAnalysisManager::Invalidator &Inv) {
13408 // Invalidate the ScalarEvolution object whenever it isn't preserved or one
13409 // of its dependencies is invalidated.
13410 auto PAC = PA.getChecker<ScalarEvolutionAnalysis>();
13411 return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>()) ||
13412 Inv.invalidate<AssumptionAnalysis>(F, PA) ||
13413 Inv.invalidate<DominatorTreeAnalysis>(F, PA) ||
13414 Inv.invalidate<LoopAnalysis>(F, PA);
13415}
13416
13417AnalysisKey ScalarEvolutionAnalysis::Key;
13418
13419ScalarEvolution ScalarEvolutionAnalysis::run(Function &F,
13420 FunctionAnalysisManager &AM) {
13421 return ScalarEvolution(F, AM.getResult<TargetLibraryAnalysis>(F),
13422 AM.getResult<AssumptionAnalysis>(F),
13423 AM.getResult<DominatorTreeAnalysis>(F),
13424 AM.getResult<LoopAnalysis>(F));
13425}
13426
13427PreservedAnalyses
13428ScalarEvolutionVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
13429 AM.getResult<ScalarEvolutionAnalysis>(F).verify();
13430 return PreservedAnalyses::all();
13431}
13432
13433PreservedAnalyses
13434ScalarEvolutionPrinterPass::run(Function &F, FunctionAnalysisManager &AM) {
13435 // For compatibility with opt's -analyze feature under legacy pass manager
13436 // which was not ported to NPM. This keeps tests using
13437 // update_analyze_test_checks.py working.
13438 OS << "Printing analysis 'Scalar Evolution Analysis' for function '"
13439 << F.getName() << "':\n";
13440 AM.getResult<ScalarEvolutionAnalysis>(F).print(OS);
13441 return PreservedAnalyses::all();
13442}
13443
13444INITIALIZE_PASS_BEGIN(ScalarEvolutionWrapperPass, "scalar-evolution",static void *initializeScalarEvolutionWrapperPassPassOnce(PassRegistry
&Registry) {
13445 "Scalar Evolution Analysis", false, true)static void *initializeScalarEvolutionWrapperPassPassOnce(PassRegistry
&Registry) {
13446INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry);
13447INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)initializeLoopInfoWrapperPassPass(Registry);
13448INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry);
13449INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
13450INITIALIZE_PASS_END(ScalarEvolutionWrapperPass, "scalar-evolution",PassInfo *PI = new PassInfo( "Scalar Evolution Analysis", "scalar-evolution"
, &ScalarEvolutionWrapperPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<ScalarEvolutionWrapperPass>), false, true
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeScalarEvolutionWrapperPassPassFlag; void
llvm::initializeScalarEvolutionWrapperPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeScalarEvolutionWrapperPassPassFlag
, initializeScalarEvolutionWrapperPassPassOnce, std::ref(Registry
)); }
13451 "Scalar Evolution Analysis", false, true)PassInfo *PI = new PassInfo( "Scalar Evolution Analysis", "scalar-evolution"
, &ScalarEvolutionWrapperPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<ScalarEvolutionWrapperPass>), false, true
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeScalarEvolutionWrapperPassPassFlag; void
llvm::initializeScalarEvolutionWrapperPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeScalarEvolutionWrapperPassPassFlag
, initializeScalarEvolutionWrapperPassPassOnce, std::ref(Registry
)); }
13452
13453char ScalarEvolutionWrapperPass::ID = 0;
13454
13455ScalarEvolutionWrapperPass::ScalarEvolutionWrapperPass() : FunctionPass(ID) {
13456 initializeScalarEvolutionWrapperPassPass(*PassRegistry::getPassRegistry());
13457}
13458
13459bool ScalarEvolutionWrapperPass::runOnFunction(Function &F) {
13460 SE.reset(new ScalarEvolution(
13461 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F),
13462 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
13463 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
13464 getAnalysis<LoopInfoWrapperPass>().getLoopInfo()));
13465 return false;
13466}
13467
13468void ScalarEvolutionWrapperPass::releaseMemory() { SE.reset(); }
13469
13470void ScalarEvolutionWrapperPass::print(raw_ostream &OS, const Module *) const {
13471 SE->print(OS);
13472}
13473
13474void ScalarEvolutionWrapperPass::verifyAnalysis() const {
13475 if (!VerifySCEV)
13476 return;
13477
13478 SE->verify();
13479}
13480
13481void ScalarEvolutionWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
13482 AU.setPreservesAll();
13483 AU.addRequiredTransitive<AssumptionCacheTracker>();
13484 AU.addRequiredTransitive<LoopInfoWrapperPass>();
13485 AU.addRequiredTransitive<DominatorTreeWrapperPass>();
13486 AU.addRequiredTransitive<TargetLibraryInfoWrapperPass>();
13487}
13488
13489const SCEVPredicate *ScalarEvolution::getEqualPredicate(const SCEV *LHS,
13490 const SCEV *RHS) {
13491 FoldingSetNodeID ID;
13492 assert(LHS->getType() == RHS->getType() &&(static_cast <bool> (LHS->getType() == RHS->getType
() && "Type mismatch between LHS and RHS") ? void (0)
: __assert_fail ("LHS->getType() == RHS->getType() && \"Type mismatch between LHS and RHS\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 13493, __extension__
__PRETTY_FUNCTION__))
13493 "Type mismatch between LHS and RHS")(static_cast <bool> (LHS->getType() == RHS->getType
() && "Type mismatch between LHS and RHS") ? void (0)
: __assert_fail ("LHS->getType() == RHS->getType() && \"Type mismatch between LHS and RHS\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 13493, __extension__
__PRETTY_FUNCTION__))
;
13494 // Unique this node based on the arguments
13495 ID.AddInteger(SCEVPredicate::P_Equal);
13496 ID.AddPointer(LHS);
13497 ID.AddPointer(RHS);
13498 void *IP = nullptr;
13499 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP))
13500 return S;
13501 SCEVEqualPredicate *Eq = new (SCEVAllocator)
13502 SCEVEqualPredicate(ID.Intern(SCEVAllocator), LHS, RHS);
13503 UniquePreds.InsertNode(Eq, IP);
13504 return Eq;
13505}
13506
13507const SCEVPredicate *ScalarEvolution::getWrapPredicate(
13508 const SCEVAddRecExpr *AR,
13509 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
13510 FoldingSetNodeID ID;
13511 // Unique this node based on the arguments
13512 ID.AddInteger(SCEVPredicate::P_Wrap);
13513 ID.AddPointer(AR);
13514 ID.AddInteger(AddedFlags);
13515 void *IP = nullptr;
13516 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP))
13517 return S;
13518 auto *OF = new (SCEVAllocator)
13519 SCEVWrapPredicate(ID.Intern(SCEVAllocator), AR, AddedFlags);
13520 UniquePreds.InsertNode(OF, IP);
13521 return OF;
13522}
13523
13524namespace {
13525
13526class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> {
13527public:
13528
13529 /// Rewrites \p S in the context of a loop L and the SCEV predication
13530 /// infrastructure.
13531 ///
13532 /// If \p Pred is non-null, the SCEV expression is rewritten to respect the
13533 /// equivalences present in \p Pred.
13534 ///
13535 /// If \p NewPreds is non-null, rewrite is free to add further predicates to
13536 /// \p NewPreds such that the result will be an AddRecExpr.
13537 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
13538 SmallPtrSetImpl<const SCEVPredicate *> *NewPreds,
13539 SCEVUnionPredicate *Pred) {
13540 SCEVPredicateRewriter Rewriter(L, SE, NewPreds, Pred);
13541 return Rewriter.visit(S);
13542 }
13543
13544 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
13545 if (Pred) {
13546 auto ExprPreds = Pred->getPredicatesForExpr(Expr);
13547 for (auto *Pred : ExprPreds)
13548 if (const auto *IPred = dyn_cast<SCEVEqualPredicate>(Pred))
13549 if (IPred->getLHS() == Expr)
13550 return IPred->getRHS();
13551 }
13552 return convertToAddRecWithPreds(Expr);
13553 }
13554
13555 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
13556 const SCEV *Operand = visit(Expr->getOperand());
13557 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Operand);
13558 if (AR && AR->getLoop() == L && AR->isAffine()) {
13559 // This couldn't be folded because the operand didn't have the nuw
13560 // flag. Add the nusw flag as an assumption that we could make.
13561 const SCEV *Step = AR->getStepRecurrence(SE);
13562 Type *Ty = Expr->getType();
13563 if (addOverflowAssumption(AR, SCEVWrapPredicate::IncrementNUSW))
13564 return SE.getAddRecExpr(SE.getZeroExtendExpr(AR->getStart(), Ty),
13565 SE.getSignExtendExpr(Step, Ty), L,
13566 AR->getNoWrapFlags());
13567 }
13568 return SE.getZeroExtendExpr(Operand, Expr->getType());
13569 }
13570
13571 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
13572 const SCEV *Operand = visit(Expr->getOperand());
13573 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Operand);
13574 if (AR && AR->getLoop() == L && AR->isAffine()) {
13575 // This couldn't be folded because the operand didn't have the nsw
13576 // flag. Add the nssw flag as an assumption that we could make.
13577 const SCEV *Step = AR->getStepRecurrence(SE);
13578 Type *Ty = Expr->getType();
13579 if (addOverflowAssumption(AR, SCEVWrapPredicate::IncrementNSSW))
13580 return SE.getAddRecExpr(SE.getSignExtendExpr(AR->getStart(), Ty),
13581 SE.getSignExtendExpr(Step, Ty), L,
13582 AR->getNoWrapFlags());
13583 }
13584 return SE.getSignExtendExpr(Operand, Expr->getType());
13585 }
13586
13587private:
13588 explicit SCEVPredicateRewriter(const Loop *L, ScalarEvolution &SE,
13589 SmallPtrSetImpl<const SCEVPredicate *> *NewPreds,
13590 SCEVUnionPredicate *Pred)
13591 : SCEVRewriteVisitor(SE), NewPreds(NewPreds), Pred(Pred), L(L) {}
13592
13593 bool addOverflowAssumption(const SCEVPredicate *P) {
13594 if (!NewPreds) {
13595 // Check if we've already made this assumption.
13596 return Pred && Pred->implies(P);
13597 }
13598 NewPreds->insert(P);
13599 return true;
13600 }
13601
13602 bool addOverflowAssumption(const SCEVAddRecExpr *AR,
13603 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
13604 auto *A = SE.getWrapPredicate(AR, AddedFlags);
13605 return addOverflowAssumption(A);
13606 }
13607
13608 // If \p Expr represents a PHINode, we try to see if it can be represented
13609 // as an AddRec, possibly under a predicate (PHISCEVPred). If it is possible
13610 // to add this predicate as a runtime overflow check, we return the AddRec.
13611 // If \p Expr does not meet these conditions (is not a PHI node, or we
13612 // couldn't create an AddRec for it, or couldn't add the predicate), we just
13613 // return \p Expr.
13614 const SCEV *convertToAddRecWithPreds(const SCEVUnknown *Expr) {
13615 if (!isa<PHINode>(Expr->getValue()))
13616 return Expr;
13617 Optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
13618 PredicatedRewrite = SE.createAddRecFromPHIWithCasts(Expr);
13619 if (!PredicatedRewrite)
13620 return Expr;
13621 for (auto *P : PredicatedRewrite->second){
13622 // Wrap predicates from outer loops are not supported.
13623 if (auto *WP = dyn_cast<const SCEVWrapPredicate>(P)) {
13624 auto *AR = cast<const SCEVAddRecExpr>(WP->getExpr());
13625 if (L != AR->getLoop())
13626 return Expr;
13627 }
13628 if (!addOverflowAssumption(P))
13629 return Expr;
13630 }
13631 return PredicatedRewrite->first;
13632 }
13633
13634 SmallPtrSetImpl<const SCEVPredicate *> *NewPreds;
13635 SCEVUnionPredicate *Pred;
13636 const Loop *L;
13637};
13638
13639} // end anonymous namespace
13640
13641const SCEV *ScalarEvolution::rewriteUsingPredicate(const SCEV *S, const Loop *L,
13642 SCEVUnionPredicate &Preds) {
13643 return SCEVPredicateRewriter::rewrite(S, L, *this, nullptr, &Preds);
13644}
13645
13646const SCEVAddRecExpr *ScalarEvolution::convertSCEVToAddRecWithPredicates(
13647 const SCEV *S, const Loop *L,
13648 SmallPtrSetImpl<const SCEVPredicate *> &Preds) {
13649 SmallPtrSet<const SCEVPredicate *, 4> TransformPreds;
13650 S = SCEVPredicateRewriter::rewrite(S, L, *this, &TransformPreds, nullptr);
13651 auto *AddRec = dyn_cast<SCEVAddRecExpr>(S);
13652
13653 if (!AddRec)
13654 return nullptr;
13655
13656 // Since the transformation was successful, we can now transfer the SCEV
13657 // predicates.
13658 for (auto *P : TransformPreds)
13659 Preds.insert(P);
13660
13661 return AddRec;
13662}
13663
13664/// SCEV predicates
13665SCEVPredicate::SCEVPredicate(const FoldingSetNodeIDRef ID,
13666 SCEVPredicateKind Kind)
13667 : FastID(ID), Kind(Kind) {}
13668
13669SCEVEqualPredicate::SCEVEqualPredicate(const FoldingSetNodeIDRef ID,
13670 const SCEV *LHS, const SCEV *RHS)
13671 : SCEVPredicate(ID, P_Equal), LHS(LHS), RHS(RHS) {
13672 assert(LHS->getType() == RHS->getType() && "LHS and RHS types don't match")(static_cast <bool> (LHS->getType() == RHS->getType
() && "LHS and RHS types don't match") ? void (0) : __assert_fail
("LHS->getType() == RHS->getType() && \"LHS and RHS types don't match\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 13672, __extension__
__PRETTY_FUNCTION__))
;
13673 assert(LHS != RHS && "LHS and RHS are the same SCEV")(static_cast <bool> (LHS != RHS && "LHS and RHS are the same SCEV"
) ? void (0) : __assert_fail ("LHS != RHS && \"LHS and RHS are the same SCEV\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 13673, __extension__
__PRETTY_FUNCTION__))
;
13674}
13675
13676bool SCEVEqualPredicate::implies(const SCEVPredicate *N) const {
13677 const auto *Op = dyn_cast<SCEVEqualPredicate>(N);
13678
13679 if (!Op)
13680 return false;
13681
13682 return Op->LHS == LHS && Op->RHS == RHS;
13683}
13684
13685bool SCEVEqualPredicate::isAlwaysTrue() const { return false; }
13686
13687const SCEV *SCEVEqualPredicate::getExpr() const { return LHS; }
13688
13689void SCEVEqualPredicate::print(raw_ostream &OS, unsigned Depth) const {
13690 OS.indent(Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n";
13691}
13692
13693SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID,
13694 const SCEVAddRecExpr *AR,
13695 IncrementWrapFlags Flags)
13696 : SCEVPredicate(ID, P_Wrap), AR(AR), Flags(Flags) {}
13697
13698const SCEV *SCEVWrapPredicate::getExpr() const { return AR; }
13699
13700bool SCEVWrapPredicate::implies(const SCEVPredicate *N) const {
13701 const auto *Op = dyn_cast<SCEVWrapPredicate>(N);
13702
13703 return Op && Op->AR == AR && setFlags(Flags, Op->Flags) == Flags;
13704}
13705
13706bool SCEVWrapPredicate::isAlwaysTrue() const {
13707 SCEV::NoWrapFlags ScevFlags = AR->getNoWrapFlags();
13708 IncrementWrapFlags IFlags = Flags;
13709
13710 if (ScalarEvolution::setFlags(ScevFlags, SCEV::FlagNSW) == ScevFlags)
13711 IFlags = clearFlags(IFlags, IncrementNSSW);
13712
13713 return IFlags == IncrementAnyWrap;
13714}
13715
13716void SCEVWrapPredicate::print(raw_ostream &OS, unsigned Depth) const {
13717 OS.indent(Depth) << *getExpr() << " Added Flags: ";
13718 if (SCEVWrapPredicate::IncrementNUSW & getFlags())
13719 OS << "<nusw>";
13720 if (SCEVWrapPredicate::IncrementNSSW & getFlags())
13721 OS << "<nssw>";
13722 OS << "\n";
13723}
13724
13725SCEVWrapPredicate::IncrementWrapFlags
13726SCEVWrapPredicate::getImpliedFlags(const SCEVAddRecExpr *AR,
13727 ScalarEvolution &SE) {
13728 IncrementWrapFlags ImpliedFlags = IncrementAnyWrap;
13729 SCEV::NoWrapFlags StaticFlags = AR->getNoWrapFlags();
13730
13731 // We can safely transfer the NSW flag as NSSW.
13732 if (ScalarEvolution::setFlags(StaticFlags, SCEV::FlagNSW) == StaticFlags)
13733 ImpliedFlags = IncrementNSSW;
13734
13735 if (ScalarEvolution::setFlags(StaticFlags, SCEV::FlagNUW) == StaticFlags) {
13736 // If the increment is positive, the SCEV NUW flag will also imply the
13737 // WrapPredicate NUSW flag.
13738 if (const auto *Step = dyn_cast<SCEVConstant>(AR->getStepRecurrence(SE)))
13739 if (Step->getValue()->getValue().isNonNegative())
13740 ImpliedFlags = setFlags(ImpliedFlags, IncrementNUSW);
13741 }
13742
13743 return ImpliedFlags;
13744}
13745
13746/// Union predicates don't get cached so create a dummy set ID for it.
13747SCEVUnionPredicate::SCEVUnionPredicate()
13748 : SCEVPredicate(FoldingSetNodeIDRef(nullptr, 0), P_Union) {}
13749
13750bool SCEVUnionPredicate::isAlwaysTrue() const {
13751 return all_of(Preds,
13752 [](const SCEVPredicate *I) { return I->isAlwaysTrue(); });
13753}
13754
13755ArrayRef<const SCEVPredicate *>
13756SCEVUnionPredicate::getPredicatesForExpr(const SCEV *Expr) {
13757 auto I = SCEVToPreds.find(Expr);
13758 if (I == SCEVToPreds.end())
13759 return ArrayRef<const SCEVPredicate *>();
13760 return I->second;
13761}
13762
13763bool SCEVUnionPredicate::implies(const SCEVPredicate *N) const {
13764 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(N))
13765 return all_of(Set->Preds,
13766 [this](const SCEVPredicate *I) { return this->implies(I); });
13767
13768 auto ScevPredsIt = SCEVToPreds.find(N->getExpr());
13769 if (ScevPredsIt == SCEVToPreds.end())
13770 return false;
13771 auto &SCEVPreds = ScevPredsIt->second;
13772
13773 return any_of(SCEVPreds,
13774 [N](const SCEVPredicate *I) { return I->implies(N); });
13775}
13776
13777const SCEV *SCEVUnionPredicate::getExpr() const { return nullptr; }
13778
13779void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const {
13780 for (auto Pred : Preds)
13781 Pred->print(OS, Depth);
13782}
13783
13784void SCEVUnionPredicate::add(const SCEVPredicate *N) {
13785 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(N)) {
13786 for (auto Pred : Set->Preds)
13787 add(Pred);
13788 return;
13789 }
13790
13791 if (implies(N))
13792 return;
13793
13794 const SCEV *Key = N->getExpr();
13795 assert(Key && "Only SCEVUnionPredicate doesn't have an "(static_cast <bool> (Key && "Only SCEVUnionPredicate doesn't have an "
" associated expression!") ? void (0) : __assert_fail ("Key && \"Only SCEVUnionPredicate doesn't have an \" \" associated expression!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 13796, __extension__
__PRETTY_FUNCTION__))
13796 " associated expression!")(static_cast <bool> (Key && "Only SCEVUnionPredicate doesn't have an "
" associated expression!") ? void (0) : __assert_fail ("Key && \"Only SCEVUnionPredicate doesn't have an \" \" associated expression!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 13796, __extension__
__PRETTY_FUNCTION__))
;
13797
13798 SCEVToPreds[Key].push_back(N);
13799 Preds.push_back(N);
13800}
13801
13802PredicatedScalarEvolution::PredicatedScalarEvolution(ScalarEvolution &SE,
13803 Loop &L)
13804 : SE(SE), L(L) {}
13805
13806void ScalarEvolution::registerUser(const SCEV *User,
13807 ArrayRef<const SCEV *> Ops) {
13808 for (auto *Op : Ops)
13809 // We do not expect that forgetting cached data for SCEVConstants will ever
13810 // open any prospects for sharpening or introduce any correctness issues,
13811 // so we don't bother storing their dependencies.
13812 if (!isa<SCEVConstant>(Op))
13813 SCEVUsers[Op].insert(User);
13814}
13815
13816const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) {
13817 const SCEV *Expr = SE.getSCEV(V);
13818 RewriteEntry &Entry = RewriteMap[Expr];
13819
13820 // If we already have an entry and the version matches, return it.
13821 if (Entry.second && Generation == Entry.first)
13822 return Entry.second;
13823
13824 // We found an entry but it's stale. Rewrite the stale entry
13825 // according to the current predicate.
13826 if (Entry.second)
13827 Expr = Entry.second;
13828
13829 const SCEV *NewSCEV = SE.rewriteUsingPredicate(Expr, &L, Preds);
13830 Entry = {Generation, NewSCEV};
13831
13832 return NewSCEV;
13833}
13834
13835const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() {
13836 if (!BackedgeCount) {
13837 SCEVUnionPredicate BackedgePred;
13838 BackedgeCount = SE.getPredicatedBackedgeTakenCount(&L, BackedgePred);
13839 addPredicate(BackedgePred);
13840 }
13841 return BackedgeCount;
13842}
13843
13844void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) {
13845 if (Preds.implies(&Pred))
13846 return;
13847 Preds.add(&Pred);
13848 updateGeneration();
13849}
13850
13851const SCEVUnionPredicate &PredicatedScalarEvolution::getUnionPredicate() const {
13852 return Preds;
13853}
13854
13855void PredicatedScalarEvolution::updateGeneration() {
13856 // If the generation number wrapped recompute everything.
13857 if (++Generation == 0) {
13858 for (auto &II : RewriteMap) {
13859 const SCEV *Rewritten = II.second.second;
13860 II.second = {Generation, SE.rewriteUsingPredicate(Rewritten, &L, Preds)};
13861 }
13862 }
13863}
13864
13865void PredicatedScalarEvolution::setNoOverflow(
13866 Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags) {
13867 const SCEV *Expr = getSCEV(V);
13868 const auto *AR = cast<SCEVAddRecExpr>(Expr);
13869
13870 auto ImpliedFlags = SCEVWrapPredicate::getImpliedFlags(AR, SE);
13871
13872 // Clear the statically implied flags.
13873 Flags = SCEVWrapPredicate::clearFlags(Flags, ImpliedFlags);
13874 addPredicate(*SE.getWrapPredicate(AR, Flags));
13875
13876 auto II = FlagsMap.insert({V, Flags});
13877 if (!II.second)
13878 II.first->second = SCEVWrapPredicate::setFlags(Flags, II.first->second);
13879}
13880
13881bool PredicatedScalarEvolution::hasNoOverflow(
13882 Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags) {
13883 const SCEV *Expr = getSCEV(V);
13884 const auto *AR = cast<SCEVAddRecExpr>(Expr);
13885
13886 Flags = SCEVWrapPredicate::clearFlags(
13887 Flags, SCEVWrapPredicate::getImpliedFlags(AR, SE));
13888
13889 auto II = FlagsMap.find(V);
13890
13891 if (II != FlagsMap.end())
13892 Flags = SCEVWrapPredicate::clearFlags(Flags, II->second);
13893
13894 return Flags == SCEVWrapPredicate::IncrementAnyWrap;
13895}
13896
13897const SCEVAddRecExpr *PredicatedScalarEvolution::getAsAddRec(Value *V) {
13898 const SCEV *Expr = this->getSCEV(V);
13899 SmallPtrSet<const SCEVPredicate *, 4> NewPreds;
13900 auto *New = SE.convertSCEVToAddRecWithPredicates(Expr, &L, NewPreds);
13901
13902 if (!New)
13903 return nullptr;
13904
13905 for (auto *P : NewPreds)
13906 Preds.add(P);
13907
13908 updateGeneration();
13909 RewriteMap[SE.getSCEV(V)] = {Generation, New};
13910 return New;
13911}
13912
13913PredicatedScalarEvolution::PredicatedScalarEvolution(
13914 const PredicatedScalarEvolution &Init)
13915 : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L), Preds(Init.Preds),
13916 Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) {
13917 for (auto I : Init.FlagsMap)
13918 FlagsMap.insert(I);
13919}
13920
13921void PredicatedScalarEvolution::print(raw_ostream &OS, unsigned Depth) const {
13922 // For each block.
13923 for (auto *BB : L.getBlocks())
13924 for (auto &I : *BB) {
13925 if (!SE.isSCEVable(I.getType()))
13926 continue;
13927
13928 auto *Expr = SE.getSCEV(&I);
13929 auto II = RewriteMap.find(Expr);
13930
13931 if (II == RewriteMap.end())
13932 continue;
13933
13934 // Don't print things that are not interesting.
13935 if (II->second.second == Expr)
13936 continue;
13937
13938 OS.indent(Depth) << "[PSE]" << I << ":\n";
13939 OS.indent(Depth + 2) << *Expr << "\n";
13940 OS.indent(Depth + 2) << "--> " << *II->second.second << "\n";
13941 }
13942}
13943
13944// Match the mathematical pattern A - (A / B) * B, where A and B can be
13945// arbitrary expressions. Also match zext (trunc A to iB) to iY, which is used
13946// for URem with constant power-of-2 second operands.
13947// It's not always easy, as A and B can be folded (imagine A is X / 2, and B is
13948// 4, A / B becomes X / 8).
13949bool ScalarEvolution::matchURem(const SCEV *Expr, const SCEV *&LHS,
13950 const SCEV *&RHS) {
13951 // Try to match 'zext (trunc A to iB) to iY', which is used
13952 // for URem with constant power-of-2 second operands. Make sure the size of
13953 // the operand A matches the size of the whole expressions.
13954 if (const auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Expr))
13955 if (const auto *Trunc = dyn_cast<SCEVTruncateExpr>(ZExt->getOperand(0))) {
13956 LHS = Trunc->getOperand();
13957 // Bail out if the type of the LHS is larger than the type of the
13958 // expression for now.
13959 if (getTypeSizeInBits(LHS->getType()) >
13960 getTypeSizeInBits(Expr->getType()))
13961 return false;
13962 if (LHS->getType() != Expr->getType())
13963 LHS = getZeroExtendExpr(LHS, Expr->getType());
13964 RHS = getConstant(APInt(getTypeSizeInBits(Expr->getType()), 1)
13965 << getTypeSizeInBits(Trunc->getType()));
13966 return true;
13967 }
13968 const auto *Add = dyn_cast<SCEVAddExpr>(Expr);
13969 if (Add == nullptr || Add->getNumOperands() != 2)
13970 return false;
13971
13972 const SCEV *A = Add->getOperand(1);
13973 const auto *Mul = dyn_cast<SCEVMulExpr>(Add->getOperand(0));
13974
13975 if (Mul == nullptr)
13976 return false;
13977
13978 const auto MatchURemWithDivisor = [&](const SCEV *B) {
13979 // (SomeExpr + (-(SomeExpr / B) * B)).
13980 if (Expr == getURemExpr(A, B)) {
13981 LHS = A;
13982 RHS = B;
13983 return true;
13984 }
13985 return false;
13986 };
13987
13988 // (SomeExpr + (-1 * (SomeExpr / B) * B)).
13989 if (Mul->getNumOperands() == 3 && isa<SCEVConstant>(Mul->getOperand(0)))
13990 return MatchURemWithDivisor(Mul->getOperand(1)) ||
13991 MatchURemWithDivisor(Mul->getOperand(2));
13992
13993 // (SomeExpr + ((-SomeExpr / B) * B)) or (SomeExpr + ((SomeExpr / B) * -B)).
13994 if (Mul->getNumOperands() == 2)
13995 return MatchURemWithDivisor(Mul->getOperand(1)) ||
13996 MatchURemWithDivisor(Mul->getOperand(0)) ||
13997 MatchURemWithDivisor(getNegativeSCEV(Mul->getOperand(1))) ||
13998 MatchURemWithDivisor(getNegativeSCEV(Mul->getOperand(0)));
13999 return false;
14000}
14001
14002const SCEV *
14003ScalarEvolution::computeSymbolicMaxBackedgeTakenCount(const Loop *L) {
14004 SmallVector<BasicBlock*, 16> ExitingBlocks;
14005 L->getExitingBlocks(ExitingBlocks);
14006
14007 // Form an expression for the maximum exit count possible for this loop. We
14008 // merge the max and exact information to approximate a version of
14009 // getConstantMaxBackedgeTakenCount which isn't restricted to just constants.
14010 SmallVector<const SCEV*, 4> ExitCounts;
14011 for (BasicBlock *ExitingBB : ExitingBlocks) {
14012 const SCEV *ExitCount = getExitCount(L, ExitingBB);
14013 if (isa<SCEVCouldNotCompute>(ExitCount))
14014 ExitCount = getExitCount(L, ExitingBB,
14015 ScalarEvolution::ConstantMaximum);
14016 if (!isa<SCEVCouldNotCompute>(ExitCount)) {
14017 assert(DT.dominates(ExitingBB, L->getLoopLatch()) &&(static_cast <bool> (DT.dominates(ExitingBB, L->getLoopLatch
()) && "We should only have known counts for exiting blocks that "
"dominate latch!") ? void (0) : __assert_fail ("DT.dominates(ExitingBB, L->getLoopLatch()) && \"We should only have known counts for exiting blocks that \" \"dominate latch!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 14019, __extension__
__PRETTY_FUNCTION__))
14018 "We should only have known counts for exiting blocks that "(static_cast <bool> (DT.dominates(ExitingBB, L->getLoopLatch
()) && "We should only have known counts for exiting blocks that "
"dominate latch!") ? void (0) : __assert_fail ("DT.dominates(ExitingBB, L->getLoopLatch()) && \"We should only have known counts for exiting blocks that \" \"dominate latch!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 14019, __extension__
__PRETTY_FUNCTION__))
14019 "dominate latch!")(static_cast <bool> (DT.dominates(ExitingBB, L->getLoopLatch
()) && "We should only have known counts for exiting blocks that "
"dominate latch!") ? void (0) : __assert_fail ("DT.dominates(ExitingBB, L->getLoopLatch()) && \"We should only have known counts for exiting blocks that \" \"dominate latch!\""
, "llvm/lib/Analysis/ScalarEvolution.cpp", 14019, __extension__
__PRETTY_FUNCTION__))
;
14020 ExitCounts.push_back(ExitCount);
14021 }
14022 }
14023 if (ExitCounts.empty())
14024 return getCouldNotCompute();
14025 return getUMinFromMismatchedTypes(ExitCounts);
14026}
14027
14028/// A rewriter to replace SCEV expressions in Map with the corresponding entry
14029/// in the map. It skips AddRecExpr because we cannot guarantee that the
14030/// replacement is loop invariant in the loop of the AddRec.
14031///
14032/// At the moment only rewriting SCEVUnknown and SCEVZeroExtendExpr is
14033/// supported.
14034class SCEVLoopGuardRewriter : public SCEVRewriteVisitor<SCEVLoopGuardRewriter> {
14035 const DenseMap<const SCEV *, const SCEV *> &Map;
14036
14037public:
14038 SCEVLoopGuardRewriter(ScalarEvolution &SE,
14039 DenseMap<const SCEV *, const SCEV *> &M)
14040 : SCEVRewriteVisitor(SE), Map(M) {}
14041
14042 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
14043
14044 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
14045 auto I = Map.find(Expr);
14046 if (I == Map.end())
14047 return Expr;
14048 return I->second;
14049 }
14050
14051 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
14052 auto I = Map.find(Expr);
14053 if (I == Map.end())
14054 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitZeroExtendExpr(
14055 Expr);
14056 return I->second;
14057 }
14058};
14059
14060const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) {
14061 SmallVector<const SCEV *> ExprsToRewrite;
14062 auto CollectCondition = [&](ICmpInst::Predicate Predicate, const SCEV *LHS,
14063 const SCEV *RHS,
14064 DenseMap<const SCEV *, const SCEV *>
14065 &RewriteMap) {
14066 // WARNING: It is generally unsound to apply any wrap flags to the proposed
14067 // replacement SCEV which isn't directly implied by the structure of that
14068 // SCEV. In particular, using contextual facts to imply flags is *NOT*
14069 // legal. See the scoping rules for flags in the header to understand why.
14070
14071 // If LHS is a constant, apply information to the other expression.
14072 if (isa<SCEVConstant>(LHS)) {
14073 std::swap(LHS, RHS);
14074 Predicate = CmpInst::getSwappedPredicate(Predicate);
14075 }
14076
14077 // Check for a condition of the form (-C1 + X < C2). InstCombine will
14078 // create this form when combining two checks of the form (X u< C2 + C1) and
14079 // (X >=u C1).
14080 auto MatchRangeCheckIdiom = [this, Predicate, LHS, RHS, &RewriteMap,
14081 &ExprsToRewrite]() {
14082 auto *AddExpr = dyn_cast<SCEVAddExpr>(LHS);
14083 if (!AddExpr || AddExpr->getNumOperands() != 2)
14084 return false;
14085
14086 auto *C1 = dyn_cast<SCEVConstant>(AddExpr->getOperand(0));
14087 auto *LHSUnknown = dyn_cast<SCEVUnknown>(AddExpr->getOperand(1));
14088 auto *C2 = dyn_cast<SCEVConstant>(RHS);
14089 if (!C1 || !C2 || !LHSUnknown)
14090 return false;
14091
14092 auto ExactRegion =
14093 ConstantRange::makeExactICmpRegion(Predicate, C2->getAPInt())
14094 .sub(C1->getAPInt());
14095
14096 // Bail out, unless we have a non-wrapping, monotonic range.
14097 if (ExactRegion.isWrappedSet() || ExactRegion.isFullSet())
14098 return false;
14099 auto I = RewriteMap.find(LHSUnknown);
14100 const SCEV *RewrittenLHS = I != RewriteMap.end() ? I->second : LHSUnknown;
14101 RewriteMap[LHSUnknown] = getUMaxExpr(
14102 getConstant(ExactRegion.getUnsignedMin()),
14103 getUMinExpr(RewrittenLHS, getConstant(ExactRegion.getUnsignedMax())));
14104 ExprsToRewrite.push_back(LHSUnknown);
14105 return true;
14106 };
14107 if (MatchRangeCheckIdiom())
14108 return;
14109
14110 // If we have LHS == 0, check if LHS is computing a property of some unknown
14111 // SCEV %v which we can rewrite %v to express explicitly.
14112 const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS);
14113 if (Predicate == CmpInst::ICMP_EQ && RHSC &&
14114 RHSC->getValue()->isNullValue()) {
14115 // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to
14116 // explicitly express that.
14117 const SCEV *URemLHS = nullptr;
14118 const SCEV *URemRHS = nullptr;
14119 if (matchURem(LHS, URemLHS, URemRHS)) {
14120 if (const SCEVUnknown *LHSUnknown = dyn_cast<SCEVUnknown>(URemLHS)) {
14121 auto Multiple = getMulExpr(getUDivExpr(URemLHS, URemRHS), URemRHS);
14122 RewriteMap[LHSUnknown] = Multiple;
14123 ExprsToRewrite.push_back(LHSUnknown);
14124 return;
14125 }
14126 }
14127 }
14128
14129 // Do not apply information for constants or if RHS contains an AddRec.
14130 if (isa<SCEVConstant>(LHS) || containsAddRecurrence(RHS))
14131 return;
14132
14133 // If RHS is SCEVUnknown, make sure the information is applied to it.
14134 if (!isa<SCEVUnknown>(LHS) && isa<SCEVUnknown>(RHS)) {
14135 std::swap(LHS, RHS);
14136 Predicate = CmpInst::getSwappedPredicate(Predicate);
14137 }
14138
14139 // Limit to expressions that can be rewritten.
14140 if (!isa<SCEVUnknown>(LHS) && !isa<SCEVZeroExtendExpr>(LHS))
14141 return;
14142
14143 // Check whether LHS has already been rewritten. In that case we want to
14144 // chain further rewrites onto the already rewritten value.
14145 auto I = RewriteMap.find(LHS);
14146 const SCEV *RewrittenLHS = I != RewriteMap.end() ? I->second : LHS;
14147
14148 const SCEV *RewrittenRHS = nullptr;
14149 switch (Predicate) {
14150 case CmpInst::ICMP_ULT:
14151 RewrittenRHS =
14152 getUMinExpr(RewrittenLHS, getMinusSCEV(RHS, getOne(RHS->getType())));
14153 break;
14154 case CmpInst::ICMP_SLT:
14155 RewrittenRHS =
14156 getSMinExpr(RewrittenLHS, getMinusSCEV(RHS, getOne(RHS->getType())));
14157 break;
14158 case CmpInst::ICMP_ULE:
14159 RewrittenRHS = getUMinExpr(RewrittenLHS, RHS);
14160 break;
14161 case CmpInst::ICMP_SLE:
14162 RewrittenRHS = getSMinExpr(RewrittenLHS, RHS);
14163 break;
14164 case CmpInst::ICMP_UGT:
14165 RewrittenRHS =
14166 getUMaxExpr(RewrittenLHS, getAddExpr(RHS, getOne(RHS->getType())));
14167 break;
14168 case CmpInst::ICMP_SGT:
14169 RewrittenRHS =
14170 getSMaxExpr(RewrittenLHS, getAddExpr(RHS, getOne(RHS->getType())));
14171 break;
14172 case CmpInst::ICMP_UGE:
14173 RewrittenRHS = getUMaxExpr(RewrittenLHS, RHS);
14174 break;
14175 case CmpInst::ICMP_SGE:
14176 RewrittenRHS = getSMaxExpr(RewrittenLHS, RHS);
14177 break;
14178 case CmpInst::ICMP_EQ:
14179 if (isa<SCEVConstant>(RHS))
14180 RewrittenRHS = RHS;
14181 break;
14182 case CmpInst::ICMP_NE:
14183 if (isa<SCEVConstant>(RHS) &&
14184 cast<SCEVConstant>(RHS)->getValue()->isNullValue())
14185 RewrittenRHS = getUMaxExpr(RewrittenLHS, getOne(RHS->getType()));
14186 break;
14187 default:
14188 break;
14189 }
14190
14191 if (RewrittenRHS) {
14192 RewriteMap[LHS] = RewrittenRHS;
14193 if (LHS == RewrittenLHS)
14194 ExprsToRewrite.push_back(LHS);
14195 }
14196 };
14197 // First, collect conditions from dominating branches. Starting at the loop
14198 // predecessor, climb up the predecessor chain, as long as there are
14199 // predecessors that can be found that have unique successors leading to the
14200 // original header.
14201 // TODO: share this logic with isLoopEntryGuardedByCond.
14202 SmallVector<std::pair<Value *, bool>> Terms;
14203 for (std::pair<const BasicBlock *, const BasicBlock *> Pair(
14204 L->getLoopPredecessor(), L->getHeader());
14205 Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
14206
14207 const BranchInst *LoopEntryPredicate =
14208 dyn_cast<BranchInst>(Pair.first->getTerminator());
14209 if (!LoopEntryPredicate || LoopEntryPredicate->isUnconditional())
14210 continue;
14211
14212 Terms.emplace_back(LoopEntryPredicate->getCondition(),
14213 LoopEntryPredicate->getSuccessor(0) == Pair.second);
14214 }
14215
14216 // Now apply the information from the collected conditions to RewriteMap.
14217 // Conditions are processed in reverse order, so the earliest conditions is
14218 // processed first. This ensures the SCEVs with the shortest dependency chains
14219 // are constructed first.
14220 DenseMap<const SCEV *, const SCEV *> RewriteMap;
14221 for (auto &E : reverse(Terms)) {
14222 bool EnterIfTrue = E.second;
14223 SmallVector<Value *, 8> Worklist;
14224 SmallPtrSet<Value *, 8> Visited;
14225 Worklist.push_back(E.first);
14226 while (!Worklist.empty()) {
14227 Value *Cond = Worklist.pop_back_val();
14228 if (!Visited.insert(Cond).second)
14229 continue;
14230
14231 if (auto *Cmp = dyn_cast<ICmpInst>(Cond)) {
14232 auto Predicate =
14233 EnterIfTrue ? Cmp->getPredicate() : Cmp->getInversePredicate();
14234 CollectCondition(Predicate, getSCEV(Cmp->getOperand(0)),
14235 getSCEV(Cmp->getOperand(1)), RewriteMap);
14236 continue;
14237 }
14238
14239 Value *L, *R;
14240 if (EnterIfTrue ? match(Cond, m_LogicalAnd(m_Value(L), m_Value(R)))
14241 : match(Cond, m_LogicalOr(m_Value(L), m_Value(R)))) {
14242 Worklist.push_back(L);
14243 Worklist.push_back(R);
14244 }
14245 }
14246 }
14247
14248 // Also collect information from assumptions dominating the loop.
14249 for (auto &AssumeVH : AC.assumptions()) {
14250 if (!AssumeVH)
14251 continue;
14252 auto *AssumeI = cast<CallInst>(AssumeVH);
14253 auto *Cmp = dyn_cast<ICmpInst>(AssumeI->getOperand(0));
14254 if (!Cmp || !DT.dominates(AssumeI, L->getHeader()))
14255 continue;
14256 CollectCondition(Cmp->getPredicate(), getSCEV(Cmp->getOperand(0)),
14257 getSCEV(Cmp->getOperand(1)), RewriteMap);
14258 }
14259
14260 if (RewriteMap.empty())
14261 return Expr;
14262
14263 // Now that all rewrite information is collect, rewrite the collected
14264 // expressions with the information in the map. This applies information to
14265 // sub-expressions.
14266 if (ExprsToRewrite.size() > 1) {
14267 for (const SCEV *Expr : ExprsToRewrite) {
14268 const SCEV *RewriteTo = RewriteMap[Expr];
14269 RewriteMap.erase(Expr);
14270 SCEVLoopGuardRewriter Rewriter(*this, RewriteMap);
14271 RewriteMap.insert({Expr, Rewriter.visit(RewriteTo)});
14272 }
14273 }
14274
14275 SCEVLoopGuardRewriter Rewriter(*this, RewriteMap);
14276 return Rewriter.visit(Expr);
14277}

/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/llvm/include/llvm/IR/Instructions.h

1//===- llvm/Instructions.h - Instruction subclass definitions ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file exposes the class definitions of all of the subclasses of the
10// Instruction class. This is meant to be an easy way to get access to all
11// instruction subclasses.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_INSTRUCTIONS_H
16#define LLVM_IR_INSTRUCTIONS_H
17
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/Bitfields.h"
20#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/None.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/ADT/iterator.h"
27#include "llvm/ADT/iterator_range.h"
28#include "llvm/IR/Attributes.h"
29#include "llvm/IR/BasicBlock.h"
30#include "llvm/IR/CallingConv.h"
31#include "llvm/IR/CFG.h"
32#include "llvm/IR/Constant.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/Function.h"
35#include "llvm/IR/InstrTypes.h"
36#include "llvm/IR/Instruction.h"
37#include "llvm/IR/OperandTraits.h"
38#include "llvm/IR/Type.h"
39#include "llvm/IR/Use.h"
40#include "llvm/IR/User.h"
41#include "llvm/IR/Value.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include <cassert>
46#include <cstddef>
47#include <cstdint>
48#include <iterator>
49
50namespace llvm {
51
52class APInt;
53class ConstantInt;
54class DataLayout;
55class LLVMContext;
56
57//===----------------------------------------------------------------------===//
58// AllocaInst Class
59//===----------------------------------------------------------------------===//
60
61/// an instruction to allocate memory on the stack
62class AllocaInst : public UnaryInstruction {
63 Type *AllocatedType;
64
65 using AlignmentField = AlignmentBitfieldElementT<0>;
66 using UsedWithInAllocaField = BoolBitfieldElementT<AlignmentField::NextBit>;
67 using SwiftErrorField = BoolBitfieldElementT<UsedWithInAllocaField::NextBit>;
68 static_assert(Bitfield::areContiguous<AlignmentField, UsedWithInAllocaField,
69 SwiftErrorField>(),
70 "Bitfields must be contiguous");
71
72protected:
73 // Note: Instruction needs to be a friend here to call cloneImpl.
74 friend class Instruction;
75
76 AllocaInst *cloneImpl() const;
77
78public:
79 explicit AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
80 const Twine &Name, Instruction *InsertBefore);
81 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
82 const Twine &Name, BasicBlock *InsertAtEnd);
83
84 AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name,
85 Instruction *InsertBefore);
86 AllocaInst(Type *Ty, unsigned AddrSpace,
87 const Twine &Name, BasicBlock *InsertAtEnd);
88
89 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, Align Align,
90 const Twine &Name = "", Instruction *InsertBefore = nullptr);
91 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, Align Align,
92 const Twine &Name, BasicBlock *InsertAtEnd);
93
94 /// Return true if there is an allocation size parameter to the allocation
95 /// instruction that is not 1.
96 bool isArrayAllocation() const;
97
98 /// Get the number of elements allocated. For a simple allocation of a single
99 /// element, this will return a constant 1 value.
100 const Value *getArraySize() const { return getOperand(0); }
101 Value *getArraySize() { return getOperand(0); }
102
103 /// Overload to return most specific pointer type.
104 PointerType *getType() const {
105 return cast<PointerType>(Instruction::getType());
106 }
107
108 /// Return the address space for the allocation.
109 unsigned getAddressSpace() const {
110 return getType()->getAddressSpace();
111 }
112
113 /// Get allocation size in bits. Returns None if size can't be determined,
114 /// e.g. in case of a VLA.
115 Optional<TypeSize> getAllocationSizeInBits(const DataLayout &DL) const;
116
117 /// Return the type that is being allocated by the instruction.
118 Type *getAllocatedType() const { return AllocatedType; }
119 /// for use only in special circumstances that need to generically
120 /// transform a whole instruction (eg: IR linking and vectorization).
121 void setAllocatedType(Type *Ty) { AllocatedType = Ty; }
122
123 /// Return the alignment of the memory that is being allocated by the
124 /// instruction.
125 Align getAlign() const {
126 return Align(1ULL << getSubclassData<AlignmentField>());
127 }
128
129 void setAlignment(Align Align) {
130 setSubclassData<AlignmentField>(Log2(Align));
131 }
132
133 // FIXME: Remove this one transition to Align is over.
134 uint64_t getAlignment() const { return getAlign().value(); }
135
136 /// Return true if this alloca is in the entry block of the function and is a
137 /// constant size. If so, the code generator will fold it into the
138 /// prolog/epilog code, so it is basically free.
139 bool isStaticAlloca() const;
140
141 /// Return true if this alloca is used as an inalloca argument to a call. Such
142 /// allocas are never considered static even if they are in the entry block.
143 bool isUsedWithInAlloca() const {
144 return getSubclassData<UsedWithInAllocaField>();
145 }
146
147 /// Specify whether this alloca is used to represent the arguments to a call.
148 void setUsedWithInAlloca(bool V) {
149 setSubclassData<UsedWithInAllocaField>(V);
150 }
151
152 /// Return true if this alloca is used as a swifterror argument to a call.
153 bool isSwiftError() const { return getSubclassData<SwiftErrorField>(); }
154 /// Specify whether this alloca is used to represent a swifterror.
155 void setSwiftError(bool V) { setSubclassData<SwiftErrorField>(V); }
156
157 // Methods for support type inquiry through isa, cast, and dyn_cast:
158 static bool classof(const Instruction *I) {
159 return (I->getOpcode() == Instruction::Alloca);
160 }
161 static bool classof(const Value *V) {
162 return isa<Instruction>(V) && classof(cast<Instruction>(V));
163 }
164
165private:
166 // Shadow Instruction::setInstructionSubclassData with a private forwarding
167 // method so that subclasses cannot accidentally use it.
168 template <typename Bitfield>
169 void setSubclassData(typename Bitfield::Type Value) {
170 Instruction::setSubclassData<Bitfield>(Value);
171 }
172};
173
174//===----------------------------------------------------------------------===//
175// LoadInst Class
176//===----------------------------------------------------------------------===//
177
178/// An instruction for reading from memory. This uses the SubclassData field in
179/// Value to store whether or not the load is volatile.
180class LoadInst : public UnaryInstruction {
181 using VolatileField = BoolBitfieldElementT<0>;
182 using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
183 using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
184 static_assert(
185 Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
186 "Bitfields must be contiguous");
187
188 void AssertOK();
189
190protected:
191 // Note: Instruction needs to be a friend here to call cloneImpl.
192 friend class Instruction;
193
194 LoadInst *cloneImpl() const;
195
196public:
197 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr,
198 Instruction *InsertBefore);
199 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, BasicBlock *InsertAtEnd);
200 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
201 Instruction *InsertBefore);
202 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
203 BasicBlock *InsertAtEnd);
204 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
205 Align Align, Instruction *InsertBefore = nullptr);
206 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
207 Align Align, BasicBlock *InsertAtEnd);
208 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
209 Align Align, AtomicOrdering Order,
210 SyncScope::ID SSID = SyncScope::System,
211 Instruction *InsertBefore = nullptr);
212 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
213 Align Align, AtomicOrdering Order, SyncScope::ID SSID,
214 BasicBlock *InsertAtEnd);
215
216 /// Return true if this is a load from a volatile memory location.
217 bool isVolatile() const { return getSubclassData<VolatileField>(); }
218
219 /// Specify whether this is a volatile load or not.
220 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
221
222 /// Return the alignment of the access that is being performed.
223 /// FIXME: Remove this function once transition to Align is over.
224 /// Use getAlign() instead.
225 uint64_t getAlignment() const { return getAlign().value(); }
226
227 /// Return the alignment of the access that is being performed.
228 Align getAlign() const {
229 return Align(1ULL << (getSubclassData<AlignmentField>()));
230 }
231
232 void setAlignment(Align Align) {
233 setSubclassData<AlignmentField>(Log2(Align));
234 }
235
236 /// Returns the ordering constraint of this load instruction.
237 AtomicOrdering getOrdering() const {
238 return getSubclassData<OrderingField>();
239 }
240 /// Sets the ordering constraint of this load instruction. May not be Release
241 /// or AcquireRelease.
242 void setOrdering(AtomicOrdering Ordering) {
243 setSubclassData<OrderingField>(Ordering);
244 }
245
246 /// Returns the synchronization scope ID of this load instruction.
247 SyncScope::ID getSyncScopeID() const {
248 return SSID;
249 }
250
251 /// Sets the synchronization scope ID of this load instruction.
252 void setSyncScopeID(SyncScope::ID SSID) {
253 this->SSID = SSID;
254 }
255
256 /// Sets the ordering constraint and the synchronization scope ID of this load
257 /// instruction.
258 void setAtomic(AtomicOrdering Ordering,
259 SyncScope::ID SSID = SyncScope::System) {
260 setOrdering(Ordering);
261 setSyncScopeID(SSID);
262 }
263
264 bool isSimple() const { return !isAtomic() && !isVolatile(); }
265
266 bool isUnordered() const {
267 return (getOrdering() == AtomicOrdering::NotAtomic ||
268 getOrdering() == AtomicOrdering::Unordered) &&
269 !isVolatile();
270 }
271
272 Value *getPointerOperand() { return getOperand(0); }
273 const Value *getPointerOperand() const { return getOperand(0); }
274 static unsigned getPointerOperandIndex() { return 0U; }
275 Type *getPointerOperandType() const { return getPointerOperand()->getType(); }
276
277 /// Returns the address space of the pointer operand.
278 unsigned getPointerAddressSpace() const {
279 return getPointerOperandType()->getPointerAddressSpace();
280 }
281
282 // Methods for support type inquiry through isa, cast, and dyn_cast:
283 static bool classof(const Instruction *I) {
284 return I->getOpcode() == Instruction::Load;
285 }
286 static bool classof(const Value *V) {
287 return isa<Instruction>(V) && classof(cast<Instruction>(V));
288 }
289
290private:
291 // Shadow Instruction::setInstructionSubclassData with a private forwarding
292 // method so that subclasses cannot accidentally use it.
293 template <typename Bitfield>
294 void setSubclassData(typename Bitfield::Type Value) {
295 Instruction::setSubclassData<Bitfield>(Value);
296 }
297
298 /// The synchronization scope ID of this load instruction. Not quite enough
299 /// room in SubClassData for everything, so synchronization scope ID gets its
300 /// own field.
301 SyncScope::ID SSID;
302};
303
304//===----------------------------------------------------------------------===//
305// StoreInst Class
306//===----------------------------------------------------------------------===//
307
308/// An instruction for storing to memory.
309class StoreInst : public Instruction {
310 using VolatileField = BoolBitfieldElementT<0>;
311 using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
312 using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
313 static_assert(
314 Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
315 "Bitfields must be contiguous");
316
317 void AssertOK();
318
319protected:
320 // Note: Instruction needs to be a friend here to call cloneImpl.
321 friend class Instruction;
322
323 StoreInst *cloneImpl() const;
324
325public:
326 StoreInst(Value *Val, Value *Ptr, Instruction *InsertBefore);
327 StoreInst(Value *Val, Value *Ptr, BasicBlock *InsertAtEnd);
328 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Instruction *InsertBefore);
329 StoreInst(Value *Val, Value *Ptr, bool isVolatile, BasicBlock *InsertAtEnd);
330 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
331 Instruction *InsertBefore = nullptr);
332 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
333 BasicBlock *InsertAtEnd);
334 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
335 AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System,
336 Instruction *InsertBefore = nullptr);
337 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
338 AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd);
339
340 // allocate space for exactly two operands
341 void *operator new(size_t S) { return User::operator new(S, 2); }
342 void operator delete(void *Ptr) { User::operator delete(Ptr); }
343
344 /// Return true if this is a store to a volatile memory location.
345 bool isVolatile() const { return getSubclassData<VolatileField>(); }
346
347 /// Specify whether this is a volatile store or not.
348 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
349
350 /// Transparently provide more efficient getOperand methods.
351 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
352
353 /// Return the alignment of the access that is being performed
354 /// FIXME: Remove this function once transition to Align is over.
355 /// Use getAlign() instead.
356 uint64_t getAlignment() const { return getAlign().value(); }
357
358 Align getAlign() const {
359 return Align(1ULL << (getSubclassData<AlignmentField>()));
360 }
361
362 void setAlignment(Align Align) {
363 setSubclassData<AlignmentField>(Log2(Align));
364 }
365
366 /// Returns the ordering constraint of this store instruction.
367 AtomicOrdering getOrdering() const {
368 return getSubclassData<OrderingField>();
369 }
370
371 /// Sets the ordering constraint of this store instruction. May not be
372 /// Acquire or AcquireRelease.
373 void setOrdering(AtomicOrdering Ordering) {
374 setSubclassData<OrderingField>(Ordering);
375 }
376
377 /// Returns the synchronization scope ID of this store instruction.
378 SyncScope::ID getSyncScopeID() const {
379 return SSID;
380 }
381
382 /// Sets the synchronization scope ID of this store instruction.
383 void setSyncScopeID(SyncScope::ID SSID) {
384 this->SSID = SSID;
385 }
386
387 /// Sets the ordering constraint and the synchronization scope ID of this
388 /// store instruction.
389 void setAtomic(AtomicOrdering Ordering,
390 SyncScope::ID SSID = SyncScope::System) {
391 setOrdering(Ordering);
392 setSyncScopeID(SSID);
393 }
394
395 bool isSimple() const { return !isAtomic() && !isVolatile(); }
396
397 bool isUnordered() const {
398 return (getOrdering() == AtomicOrdering::NotAtomic ||
399 getOrdering() == AtomicOrdering::Unordered) &&
400 !isVolatile();
401 }
402
403 Value *getValueOperand() { return getOperand(0); }
404 const Value *getValueOperand() const { return getOperand(0); }
405
406 Value *getPointerOperand() { return getOperand(1); }
407 const Value *getPointerOperand() const { return getOperand(1); }
18
Calling 'StoreInst::getOperand'
22
Returning from 'StoreInst::getOperand'
23
Returning pointer, which participates in a condition later
408 static unsigned getPointerOperandIndex() { return 1U; }
409 Type *getPointerOperandType() const { return getPointerOperand()->getType(); }
410
411 /// Returns the address space of the pointer operand.
412 unsigned getPointerAddressSpace() const {
413 return getPointerOperandType()->getPointerAddressSpace();
414 }
415
416 // Methods for support type inquiry through isa, cast, and dyn_cast:
417 static bool classof(const Instruction *I) {
418 return I->getOpcode() == Instruction::Store;
419 }
420 static bool classof(const Value *V) {
421 return isa<Instruction>(V) && classof(cast<Instruction>(V));
422 }
423
424private:
425 // Shadow Instruction::setInstructionSubclassData with a private forwarding
426 // method so that subclasses cannot accidentally use it.
427 template <typename Bitfield>
428 void setSubclassData(typename Bitfield::Type Value) {
429 Instruction::setSubclassData<Bitfield>(Value);
430 }
431
432 /// The synchronization scope ID of this store instruction. Not quite enough
433 /// room in SubClassData for everything, so synchronization scope ID gets its
434 /// own field.
435 SyncScope::ID SSID;
436};
437
438template <>
439struct OperandTraits<StoreInst> : public FixedNumOperandTraits<StoreInst, 2> {
440};
441
442DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value)StoreInst::op_iterator StoreInst::op_begin() { return OperandTraits
<StoreInst>::op_begin(this); } StoreInst::const_op_iterator
StoreInst::op_begin() const { return OperandTraits<StoreInst
>::op_begin(const_cast<StoreInst*>(this)); } StoreInst
::op_iterator StoreInst::op_end() { return OperandTraits<StoreInst
>::op_end(this); } StoreInst::const_op_iterator StoreInst::
op_end() const { return OperandTraits<StoreInst>::op_end
(const_cast<StoreInst*>(this)); } Value *StoreInst::getOperand
(unsigned i_nocapture) const { (static_cast <bool> (i_nocapture
< OperandTraits<StoreInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<StoreInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 442, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<StoreInst
>::op_begin(const_cast<StoreInst*>(this))[i_nocapture
].get()); } void StoreInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { (static_cast <bool> (i_nocapture <
OperandTraits<StoreInst>::operands(this) && "setOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<StoreInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 442, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<StoreInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned StoreInst::getNumOperands() const
{ return OperandTraits<StoreInst>::operands(this); } template
<int Idx_nocapture> Use &StoreInst::Op() { return this
->OpFrom<Idx_nocapture>(this); } template <int Idx_nocapture
> const Use &StoreInst::Op() const { return this->OpFrom
<Idx_nocapture>(this); }
19
'?' condition is true
20
Assuming the object is a 'Value'
21
Returning pointer, which participates in a condition later
443
444//===----------------------------------------------------------------------===//
445// FenceInst Class
446//===----------------------------------------------------------------------===//
447
448/// An instruction for ordering other memory operations.
449class FenceInst : public Instruction {
450 using OrderingField = AtomicOrderingBitfieldElementT<0>;
451
452 void Init(AtomicOrdering Ordering, SyncScope::ID SSID);
453
454protected:
455 // Note: Instruction needs to be a friend here to call cloneImpl.
456 friend class Instruction;
457
458 FenceInst *cloneImpl() const;
459
460public:
461 // Ordering may only be Acquire, Release, AcquireRelease, or
462 // SequentiallyConsistent.
463 FenceInst(LLVMContext &C, AtomicOrdering Ordering,
464 SyncScope::ID SSID = SyncScope::System,
465 Instruction *InsertBefore = nullptr);
466 FenceInst(LLVMContext &C, AtomicOrdering Ordering, SyncScope::ID SSID,
467 BasicBlock *InsertAtEnd);
468
469 // allocate space for exactly zero operands
470 void *operator new(size_t S) { return User::operator new(S, 0); }
471 void operator delete(void *Ptr) { User::operator delete(Ptr); }
472
473 /// Returns the ordering constraint of this fence instruction.
474 AtomicOrdering getOrdering() const {
475 return getSubclassData<OrderingField>();
476 }
477
478 /// Sets the ordering constraint of this fence instruction. May only be
479 /// Acquire, Release, AcquireRelease, or SequentiallyConsistent.
480 void setOrdering(AtomicOrdering Ordering) {
481 setSubclassData<OrderingField>(Ordering);
482 }
483
484 /// Returns the synchronization scope ID of this fence instruction.
485 SyncScope::ID getSyncScopeID() const {
486 return SSID;
487 }
488
489 /// Sets the synchronization scope ID of this fence instruction.
490 void setSyncScopeID(SyncScope::ID SSID) {
491 this->SSID = SSID;
492 }
493
494 // Methods for support type inquiry through isa, cast, and dyn_cast:
495 static bool classof(const Instruction *I) {
496 return I->getOpcode() == Instruction::Fence;
497 }
498 static bool classof(const Value *V) {
499 return isa<Instruction>(V) && classof(cast<Instruction>(V));
500 }
501
502private:
503 // Shadow Instruction::setInstructionSubclassData with a private forwarding
504 // method so that subclasses cannot accidentally use it.
505 template <typename Bitfield>
506 void setSubclassData(typename Bitfield::Type Value) {
507 Instruction::setSubclassData<Bitfield>(Value);
508 }
509
510 /// The synchronization scope ID of this fence instruction. Not quite enough
511 /// room in SubClassData for everything, so synchronization scope ID gets its
512 /// own field.
513 SyncScope::ID SSID;
514};
515
516//===----------------------------------------------------------------------===//
517// AtomicCmpXchgInst Class
518//===----------------------------------------------------------------------===//
519
520/// An instruction that atomically checks whether a
521/// specified value is in a memory location, and, if it is, stores a new value
522/// there. The value returned by this instruction is a pair containing the
523/// original value as first element, and an i1 indicating success (true) or
524/// failure (false) as second element.
525///
526class AtomicCmpXchgInst : public Instruction {
527 void Init(Value *Ptr, Value *Cmp, Value *NewVal, Align Align,
528 AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
529 SyncScope::ID SSID);
530
531 template <unsigned Offset>
532 using AtomicOrderingBitfieldElement =
533 typename Bitfield::Element<AtomicOrdering, Offset, 3,
534 AtomicOrdering::LAST>;
535
536protected:
537 // Note: Instruction needs to be a friend here to call cloneImpl.
538 friend class Instruction;
539
540 AtomicCmpXchgInst *cloneImpl() const;
541
542public:
543 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, Align Alignment,
544 AtomicOrdering SuccessOrdering,
545 AtomicOrdering FailureOrdering, SyncScope::ID SSID,
546 Instruction *InsertBefore = nullptr);
547 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, Align Alignment,
548 AtomicOrdering SuccessOrdering,
549 AtomicOrdering FailureOrdering, SyncScope::ID SSID,
550 BasicBlock *InsertAtEnd);
551
552 // allocate space for exactly three operands
553 void *operator new(size_t S) { return User::operator new(S, 3); }
554 void operator delete(void *Ptr) { User::operator delete(Ptr); }
555
556 using VolatileField = BoolBitfieldElementT<0>;
557 using WeakField = BoolBitfieldElementT<VolatileField::NextBit>;
558 using SuccessOrderingField =
559 AtomicOrderingBitfieldElementT<WeakField::NextBit>;
560 using FailureOrderingField =
561 AtomicOrderingBitfieldElementT<SuccessOrderingField::NextBit>;
562 using AlignmentField =
563 AlignmentBitfieldElementT<FailureOrderingField::NextBit>;
564 static_assert(
565 Bitfield::areContiguous<VolatileField, WeakField, SuccessOrderingField,
566 FailureOrderingField, AlignmentField>(),
567 "Bitfields must be contiguous");
568
569 /// Return the alignment of the memory that is being allocated by the
570 /// instruction.
571 Align getAlign() const {
572 return Align(1ULL << getSubclassData<AlignmentField>());
573 }
574
575 void setAlignment(Align Align) {
576 setSubclassData<AlignmentField>(Log2(Align));
577 }
578
579 /// Return true if this is a cmpxchg from a volatile memory
580 /// location.
581 ///
582 bool isVolatile() const { return getSubclassData<VolatileField>(); }
583
584 /// Specify whether this is a volatile cmpxchg.
585 ///
586 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
587
588 /// Return true if this cmpxchg may spuriously fail.
589 bool isWeak() const { return getSubclassData<WeakField>(); }
590
591 void setWeak(bool IsWeak) { setSubclassData<WeakField>(IsWeak); }
592
593 /// Transparently provide more efficient getOperand methods.
594 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
595
596 static bool isValidSuccessOrdering(AtomicOrdering Ordering) {
597 return Ordering != AtomicOrdering::NotAtomic &&
598 Ordering != AtomicOrdering::Unordered;
599 }
600
601 static bool isValidFailureOrdering(AtomicOrdering Ordering) {
602 return Ordering != AtomicOrdering::NotAtomic &&
603 Ordering != AtomicOrdering::Unordered &&
604 Ordering != AtomicOrdering::AcquireRelease &&
605 Ordering != AtomicOrdering::Release;
606 }
607
608 /// Returns the success ordering constraint of this cmpxchg instruction.
609 AtomicOrdering getSuccessOrdering() const {
610 return getSubclassData<SuccessOrderingField>();
611 }
612
613 /// Sets the success ordering constraint of this cmpxchg instruction.
614 void setSuccessOrdering(AtomicOrdering Ordering) {
615 assert(isValidSuccessOrdering(Ordering) &&(static_cast <bool> (isValidSuccessOrdering(Ordering) &&
"invalid CmpXchg success ordering") ? void (0) : __assert_fail
("isValidSuccessOrdering(Ordering) && \"invalid CmpXchg success ordering\""
, "llvm/include/llvm/IR/Instructions.h", 616, __extension__ __PRETTY_FUNCTION__
))
616 "invalid CmpXchg success ordering")(static_cast <bool> (isValidSuccessOrdering(Ordering) &&
"invalid CmpXchg success ordering") ? void (0) : __assert_fail
("isValidSuccessOrdering(Ordering) && \"invalid CmpXchg success ordering\""
, "llvm/include/llvm/IR/Instructions.h", 616, __extension__ __PRETTY_FUNCTION__
))
;
617 setSubclassData<SuccessOrderingField>(Ordering);
618 }
619
620 /// Returns the failure ordering constraint of this cmpxchg instruction.
621 AtomicOrdering getFailureOrdering() const {
622 return getSubclassData<FailureOrderingField>();
623 }
624
625 /// Sets the failure ordering constraint of this cmpxchg instruction.
626 void setFailureOrdering(AtomicOrdering Ordering) {
627 assert(isValidFailureOrdering(Ordering) &&(static_cast <bool> (isValidFailureOrdering(Ordering) &&
"invalid CmpXchg failure ordering") ? void (0) : __assert_fail
("isValidFailureOrdering(Ordering) && \"invalid CmpXchg failure ordering\""
, "llvm/include/llvm/IR/Instructions.h", 628, __extension__ __PRETTY_FUNCTION__
))
628 "invalid CmpXchg failure ordering")(static_cast <bool> (isValidFailureOrdering(Ordering) &&
"invalid CmpXchg failure ordering") ? void (0) : __assert_fail
("isValidFailureOrdering(Ordering) && \"invalid CmpXchg failure ordering\""
, "llvm/include/llvm/IR/Instructions.h", 628, __extension__ __PRETTY_FUNCTION__
))
;
629 setSubclassData<FailureOrderingField>(Ordering);
630 }
631
632 /// Returns a single ordering which is at least as strong as both the
633 /// success and failure orderings for this cmpxchg.
634 AtomicOrdering getMergedOrdering() const {
635 if (getFailureOrdering() == AtomicOrdering::SequentiallyConsistent)
636 return AtomicOrdering::SequentiallyConsistent;
637 if (getFailureOrdering() == AtomicOrdering::Acquire) {
638 if (getSuccessOrdering() == AtomicOrdering::Monotonic)
639 return AtomicOrdering::Acquire;
640 if (getSuccessOrdering() == AtomicOrdering::Release)
641 return AtomicOrdering::AcquireRelease;
642 }
643 return getSuccessOrdering();
644 }
645
646 /// Returns the synchronization scope ID of this cmpxchg instruction.
647 SyncScope::ID getSyncScopeID() const {
648 return SSID;
649 }
650
651 /// Sets the synchronization scope ID of this cmpxchg instruction.
652 void setSyncScopeID(SyncScope::ID SSID) {
653 this->SSID = SSID;
654 }
655
656 Value *getPointerOperand() { return getOperand(0); }
657 const Value *getPointerOperand() const { return getOperand(0); }
658 static unsigned getPointerOperandIndex() { return 0U; }
659
660 Value *getCompareOperand() { return getOperand(1); }
661 const Value *getCompareOperand() const { return getOperand(1); }
662
663 Value *getNewValOperand() { return getOperand(2); }
664 const Value *getNewValOperand() const { return getOperand(2); }
665
666 /// Returns the address space of the pointer operand.
667 unsigned getPointerAddressSpace() const {
668 return getPointerOperand()->getType()->getPointerAddressSpace();
669 }
670
671 /// Returns the strongest permitted ordering on failure, given the
672 /// desired ordering on success.
673 ///
674 /// If the comparison in a cmpxchg operation fails, there is no atomic store
675 /// so release semantics cannot be provided. So this function drops explicit
676 /// Release requests from the AtomicOrdering. A SequentiallyConsistent
677 /// operation would remain SequentiallyConsistent.
678 static AtomicOrdering
679 getStrongestFailureOrdering(AtomicOrdering SuccessOrdering) {
680 switch (SuccessOrdering) {
681 default:
682 llvm_unreachable("invalid cmpxchg success ordering")::llvm::llvm_unreachable_internal("invalid cmpxchg success ordering"
, "llvm/include/llvm/IR/Instructions.h", 682)
;
683 case AtomicOrdering::Release:
684 case AtomicOrdering::Monotonic:
685 return AtomicOrdering::Monotonic;
686 case AtomicOrdering::AcquireRelease:
687 case AtomicOrdering::Acquire:
688 return AtomicOrdering::Acquire;
689 case AtomicOrdering::SequentiallyConsistent:
690 return AtomicOrdering::SequentiallyConsistent;
691 }
692 }
693
694 // Methods for support type inquiry through isa, cast, and dyn_cast:
695 static bool classof(const Instruction *I) {
696 return I->getOpcode() == Instruction::AtomicCmpXchg;
697 }
698 static bool classof(const Value *V) {
699 return isa<Instruction>(V) && classof(cast<Instruction>(V));
700 }
701
702private:
703 // Shadow Instruction::setInstructionSubclassData with a private forwarding
704 // method so that subclasses cannot accidentally use it.
705 template <typename Bitfield>
706 void setSubclassData(typename Bitfield::Type Value) {
707 Instruction::setSubclassData<Bitfield>(Value);
708 }
709
710 /// The synchronization scope ID of this cmpxchg instruction. Not quite
711 /// enough room in SubClassData for everything, so synchronization scope ID
712 /// gets its own field.
713 SyncScope::ID SSID;
714};
715
716template <>
717struct OperandTraits<AtomicCmpXchgInst> :
718 public FixedNumOperandTraits<AtomicCmpXchgInst, 3> {
719};
720
721DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicCmpXchgInst, Value)AtomicCmpXchgInst::op_iterator AtomicCmpXchgInst::op_begin() {
return OperandTraits<AtomicCmpXchgInst>::op_begin(this
); } AtomicCmpXchgInst::const_op_iterator AtomicCmpXchgInst::
op_begin() const { return OperandTraits<AtomicCmpXchgInst>
::op_begin(const_cast<AtomicCmpXchgInst*>(this)); } AtomicCmpXchgInst
::op_iterator AtomicCmpXchgInst::op_end() { return OperandTraits
<AtomicCmpXchgInst>::op_end(this); } AtomicCmpXchgInst::
const_op_iterator AtomicCmpXchgInst::op_end() const { return OperandTraits
<AtomicCmpXchgInst>::op_end(const_cast<AtomicCmpXchgInst
*>(this)); } Value *AtomicCmpXchgInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<AtomicCmpXchgInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicCmpXchgInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 721, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<AtomicCmpXchgInst
>::op_begin(const_cast<AtomicCmpXchgInst*>(this))[i_nocapture
].get()); } void AtomicCmpXchgInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<AtomicCmpXchgInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicCmpXchgInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 721, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<AtomicCmpXchgInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned AtomicCmpXchgInst::getNumOperands
() const { return OperandTraits<AtomicCmpXchgInst>::operands
(this); } template <int Idx_nocapture> Use &AtomicCmpXchgInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &AtomicCmpXchgInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
722
723//===----------------------------------------------------------------------===//
724// AtomicRMWInst Class
725//===----------------------------------------------------------------------===//
726
727/// an instruction that atomically reads a memory location,
728/// combines it with another value, and then stores the result back. Returns
729/// the old value.
730///
731class AtomicRMWInst : public Instruction {
732protected:
733 // Note: Instruction needs to be a friend here to call cloneImpl.
734 friend class Instruction;
735
736 AtomicRMWInst *cloneImpl() const;
737
738public:
739 /// This enumeration lists the possible modifications atomicrmw can make. In
740 /// the descriptions, 'p' is the pointer to the instruction's memory location,
741 /// 'old' is the initial value of *p, and 'v' is the other value passed to the
742 /// instruction. These instructions always return 'old'.
743 enum BinOp : unsigned {
744 /// *p = v
745 Xchg,
746 /// *p = old + v
747 Add,
748 /// *p = old - v
749 Sub,
750 /// *p = old & v
751 And,
752 /// *p = ~(old & v)
753 Nand,
754 /// *p = old | v
755 Or,
756 /// *p = old ^ v
757 Xor,
758 /// *p = old >signed v ? old : v
759 Max,
760 /// *p = old <signed v ? old : v
761 Min,
762 /// *p = old >unsigned v ? old : v
763 UMax,
764 /// *p = old <unsigned v ? old : v
765 UMin,
766
767 /// *p = old + v
768 FAdd,
769
770 /// *p = old - v
771 FSub,
772
773 FIRST_BINOP = Xchg,
774 LAST_BINOP = FSub,
775 BAD_BINOP
776 };
777
778private:
779 template <unsigned Offset>
780 using AtomicOrderingBitfieldElement =
781 typename Bitfield::Element<AtomicOrdering, Offset, 3,
782 AtomicOrdering::LAST>;
783
784 template <unsigned Offset>
785 using BinOpBitfieldElement =
786 typename Bitfield::Element<BinOp, Offset, 4, BinOp::LAST_BINOP>;
787
788public:
789 AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, Align Alignment,
790 AtomicOrdering Ordering, SyncScope::ID SSID,
791 Instruction *InsertBefore = nullptr);
792 AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, Align Alignment,
793 AtomicOrdering Ordering, SyncScope::ID SSID,
794 BasicBlock *InsertAtEnd);
795
796 // allocate space for exactly two operands
797 void *operator new(size_t S) { return User::operator new(S, 2); }
798 void operator delete(void *Ptr) { User::operator delete(Ptr); }
799
800 using VolatileField = BoolBitfieldElementT<0>;
801 using AtomicOrderingField =
802 AtomicOrderingBitfieldElementT<VolatileField::NextBit>;
803 using OperationField = BinOpBitfieldElement<AtomicOrderingField::NextBit>;
804 using AlignmentField = AlignmentBitfieldElementT<OperationField::NextBit>;
805 static_assert(Bitfield::areContiguous<VolatileField, AtomicOrderingField,
806 OperationField, AlignmentField>(),
807 "Bitfields must be contiguous");
808
809 BinOp getOperation() const { return getSubclassData<OperationField>(); }
810
811 static StringRef getOperationName(BinOp Op);
812
813 static bool isFPOperation(BinOp Op) {
814 switch (Op) {
815 case AtomicRMWInst::FAdd:
816 case AtomicRMWInst::FSub:
817 return true;
818 default:
819 return false;
820 }
821 }
822
823 void setOperation(BinOp Operation) {
824 setSubclassData<OperationField>(Operation);
825 }
826
827 /// Return the alignment of the memory that is being allocated by the
828 /// instruction.
829 Align getAlign() const {
830 return Align(1ULL << getSubclassData<AlignmentField>());
831 }
832
833 void setAlignment(Align Align) {
834 setSubclassData<AlignmentField>(Log2(Align));
835 }
836
837 /// Return true if this is a RMW on a volatile memory location.
838 ///
839 bool isVolatile() const { return getSubclassData<VolatileField>(); }
840
841 /// Specify whether this is a volatile RMW or not.
842 ///
843 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
844
845 /// Transparently provide more efficient getOperand methods.
846 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
847
848 /// Returns the ordering constraint of this rmw instruction.
849 AtomicOrdering getOrdering() const {
850 return getSubclassData<AtomicOrderingField>();
851 }
852
853 /// Sets the ordering constraint of this rmw instruction.
854 void setOrdering(AtomicOrdering Ordering) {
855 assert(Ordering != AtomicOrdering::NotAtomic &&(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "atomicrmw instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"atomicrmw instructions can only be atomic.\""
, "llvm/include/llvm/IR/Instructions.h", 856, __extension__ __PRETTY_FUNCTION__
))
856 "atomicrmw instructions can only be atomic.")(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "atomicrmw instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"atomicrmw instructions can only be atomic.\""
, "llvm/include/llvm/IR/Instructions.h", 856, __extension__ __PRETTY_FUNCTION__
))
;
857 setSubclassData<AtomicOrderingField>(Ordering);
858 }
859
860 /// Returns the synchronization scope ID of this rmw instruction.
861 SyncScope::ID getSyncScopeID() const {
862 return SSID;
863 }
864
865 /// Sets the synchronization scope ID of this rmw instruction.
866 void setSyncScopeID(SyncScope::ID SSID) {
867 this->SSID = SSID;
868 }
869
870 Value *getPointerOperand() { return getOperand(0); }
871 const Value *getPointerOperand() const { return getOperand(0); }
872 static unsigned getPointerOperandIndex() { return 0U; }
873
874 Value *getValOperand() { return getOperand(1); }
875 const Value *getValOperand() const { return getOperand(1); }
876
877 /// Returns the address space of the pointer operand.
878 unsigned getPointerAddressSpace() const {
879 return getPointerOperand()->getType()->getPointerAddressSpace();
880 }
881
882 bool isFloatingPointOperation() const {
883 return isFPOperation(getOperation());
884 }
885
886 // Methods for support type inquiry through isa, cast, and dyn_cast:
887 static bool classof(const Instruction *I) {
888 return I->getOpcode() == Instruction::AtomicRMW;
889 }
890 static bool classof(const Value *V) {
891 return isa<Instruction>(V) && classof(cast<Instruction>(V));
892 }
893
894private:
895 void Init(BinOp Operation, Value *Ptr, Value *Val, Align Align,
896 AtomicOrdering Ordering, SyncScope::ID SSID);
897
898 // Shadow Instruction::setInstructionSubclassData with a private forwarding
899 // method so that subclasses cannot accidentally use it.
900 template <typename Bitfield>
901 void setSubclassData(typename Bitfield::Type Value) {
902 Instruction::setSubclassData<Bitfield>(Value);
903 }
904
905 /// The synchronization scope ID of this rmw instruction. Not quite enough
906 /// room in SubClassData for everything, so synchronization scope ID gets its
907 /// own field.
908 SyncScope::ID SSID;
909};
910
911template <>
912struct OperandTraits<AtomicRMWInst>
913 : public FixedNumOperandTraits<AtomicRMWInst,2> {
914};
915
916DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicRMWInst, Value)AtomicRMWInst::op_iterator AtomicRMWInst::op_begin() { return
OperandTraits<AtomicRMWInst>::op_begin(this); } AtomicRMWInst
::const_op_iterator AtomicRMWInst::op_begin() const { return OperandTraits
<AtomicRMWInst>::op_begin(const_cast<AtomicRMWInst*>
(this)); } AtomicRMWInst::op_iterator AtomicRMWInst::op_end()
{ return OperandTraits<AtomicRMWInst>::op_end(this); }
AtomicRMWInst::const_op_iterator AtomicRMWInst::op_end() const
{ return OperandTraits<AtomicRMWInst>::op_end(const_cast
<AtomicRMWInst*>(this)); } Value *AtomicRMWInst::getOperand
(unsigned i_nocapture) const { (static_cast <bool> (i_nocapture
< OperandTraits<AtomicRMWInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicRMWInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 916, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<AtomicRMWInst
>::op_begin(const_cast<AtomicRMWInst*>(this))[i_nocapture
].get()); } void AtomicRMWInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<AtomicRMWInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicRMWInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 916, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<AtomicRMWInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned AtomicRMWInst::getNumOperands()
const { return OperandTraits<AtomicRMWInst>::operands(
this); } template <int Idx_nocapture> Use &AtomicRMWInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &AtomicRMWInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
917
918//===----------------------------------------------------------------------===//
919// GetElementPtrInst Class
920//===----------------------------------------------------------------------===//
921
922// checkGEPType - Simple wrapper function to give a better assertion failure
923// message on bad indexes for a gep instruction.
924//
925inline Type *checkGEPType(Type *Ty) {
926 assert(Ty && "Invalid GetElementPtrInst indices for type!")(static_cast <bool> (Ty && "Invalid GetElementPtrInst indices for type!"
) ? void (0) : __assert_fail ("Ty && \"Invalid GetElementPtrInst indices for type!\""
, "llvm/include/llvm/IR/Instructions.h", 926, __extension__ __PRETTY_FUNCTION__
))
;
927 return Ty;
928}
929
930/// an instruction for type-safe pointer arithmetic to
931/// access elements of arrays and structs
932///
933class GetElementPtrInst : public Instruction {
934 Type *SourceElementType;
935 Type *ResultElementType;
936
937 GetElementPtrInst(const GetElementPtrInst &GEPI);
938
939 /// Constructors - Create a getelementptr instruction with a base pointer an
940 /// list of indices. The first ctor can optionally insert before an existing
941 /// instruction, the second appends the new instruction to the specified
942 /// BasicBlock.
943 inline GetElementPtrInst(Type *PointeeType, Value *Ptr,
944 ArrayRef<Value *> IdxList, unsigned Values,
945 const Twine &NameStr, Instruction *InsertBefore);
946 inline GetElementPtrInst(Type *PointeeType, Value *Ptr,
947 ArrayRef<Value *> IdxList, unsigned Values,
948 const Twine &NameStr, BasicBlock *InsertAtEnd);
949
950 void init(Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr);
951
952protected:
953 // Note: Instruction needs to be a friend here to call cloneImpl.
954 friend class Instruction;
955
956 GetElementPtrInst *cloneImpl() const;
957
958public:
959 static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
960 ArrayRef<Value *> IdxList,
961 const Twine &NameStr = "",
962 Instruction *InsertBefore = nullptr) {
963 unsigned Values = 1 + unsigned(IdxList.size());
964 assert(PointeeType && "Must specify element type")(static_cast <bool> (PointeeType && "Must specify element type"
) ? void (0) : __assert_fail ("PointeeType && \"Must specify element type\""
, "llvm/include/llvm/IR/Instructions.h", 964, __extension__ __PRETTY_FUNCTION__
))
;
965 assert(cast<PointerType>(Ptr->getType()->getScalarType())(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "llvm/include/llvm/IR/Instructions.h", 966, __extension__ __PRETTY_FUNCTION__
))
966 ->isOpaqueOrPointeeTypeMatches(PointeeType))(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "llvm/include/llvm/IR/Instructions.h", 966, __extension__ __PRETTY_FUNCTION__
))
;
967 return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
968 NameStr, InsertBefore);
969 }
970
971 static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
972 ArrayRef<Value *> IdxList,
973 const Twine &NameStr,
974 BasicBlock *InsertAtEnd) {
975 unsigned Values = 1 + unsigned(IdxList.size());
976 assert(PointeeType && "Must specify element type")(static_cast <bool> (PointeeType && "Must specify element type"
) ? void (0) : __assert_fail ("PointeeType && \"Must specify element type\""
, "llvm/include/llvm/IR/Instructions.h", 976, __extension__ __PRETTY_FUNCTION__
))
;
977 assert(cast<PointerType>(Ptr->getType()->getScalarType())(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "llvm/include/llvm/IR/Instructions.h", 978, __extension__ __PRETTY_FUNCTION__
))
978 ->isOpaqueOrPointeeTypeMatches(PointeeType))(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "llvm/include/llvm/IR/Instructions.h", 978, __extension__ __PRETTY_FUNCTION__
))
;
979 return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
980 NameStr, InsertAtEnd);
981 }
982
983 /// Create an "inbounds" getelementptr. See the documentation for the
984 /// "inbounds" flag in LangRef.html for details.
985 static GetElementPtrInst *
986 CreateInBounds(Type *PointeeType, Value *Ptr, ArrayRef<Value *> IdxList,
987 const Twine &NameStr = "",
988 Instruction *InsertBefore = nullptr) {
989 GetElementPtrInst *GEP =
990 Create(PointeeType, Ptr, IdxList, NameStr, InsertBefore);
991 GEP->setIsInBounds(true);
992 return GEP;
993 }
994
995 static GetElementPtrInst *CreateInBounds(Type *PointeeType, Value *Ptr,
996 ArrayRef<Value *> IdxList,
997 const Twine &NameStr,
998 BasicBlock *InsertAtEnd) {
999 GetElementPtrInst *GEP =
1000 Create(PointeeType, Ptr, IdxList, NameStr, InsertAtEnd);
1001 GEP->setIsInBounds(true);
1002 return GEP;
1003 }
1004
1005 /// Transparently provide more efficient getOperand methods.
1006 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1007
1008 Type *getSourceElementType() const { return SourceElementType; }
1009
1010 void setSourceElementType(Type *Ty) { SourceElementType = Ty; }
1011 void setResultElementType(Type *Ty) { ResultElementType = Ty; }
1012
1013 Type *getResultElementType() const {
1014 assert(cast<PointerType>(getType()->getScalarType())(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "llvm/include/llvm/IR/Instructions.h", 1015, __extension__ __PRETTY_FUNCTION__
))
1015 ->isOpaqueOrPointeeTypeMatches(ResultElementType))(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "llvm/include/llvm/IR/Instructions.h", 1015, __extension__ __PRETTY_FUNCTION__
))
;
1016 return ResultElementType;
1017 }
1018
1019 /// Returns the address space of this instruction's pointer type.
1020 unsigned getAddressSpace() const {
1021 // Note that this is always the same as the pointer operand's address space
1022 // and that is cheaper to compute, so cheat here.
1023 return getPointerAddressSpace();
1024 }
1025
1026 /// Returns the result type of a getelementptr with the given source
1027 /// element type and indexes.
1028 ///
1029 /// Null is returned if the indices are invalid for the specified
1030 /// source element type.
1031 static Type *getIndexedType(Type *Ty, ArrayRef<Value *> IdxList);
1032 static Type *getIndexedType(Type *Ty, ArrayRef<Constant *> IdxList);
1033 static Type *getIndexedType(Type *Ty, ArrayRef<uint64_t> IdxList);
1034
1035 /// Return the type of the element at the given index of an indexable
1036 /// type. This is equivalent to "getIndexedType(Agg, {Zero, Idx})".
1037 ///
1038 /// Returns null if the type can't be indexed, or the given index is not
1039 /// legal for the given type.
1040 static Type *getTypeAtIndex(Type *Ty, Value *Idx);
1041 static Type *getTypeAtIndex(Type *Ty, uint64_t Idx);
1042
1043 inline op_iterator idx_begin() { return op_begin()+1; }
1044 inline const_op_iterator idx_begin() const { return op_begin()+1; }
1045 inline op_iterator idx_end() { return op_end(); }
1046 inline const_op_iterator idx_end() const { return op_end(); }
1047
1048 inline iterator_range<op_iterator> indices() {
1049 return make_range(idx_begin(), idx_end());
1050 }
1051
1052 inline iterator_range<const_op_iterator> indices() const {
1053 return make_range(idx_begin(), idx_end());
1054 }
1055
1056 Value *getPointerOperand() {
1057 return getOperand(0);
1058 }
1059 const Value *getPointerOperand() const {
1060 return getOperand(0);
1061 }
1062 static unsigned getPointerOperandIndex() {
1063 return 0U; // get index for modifying correct operand.
1064 }
1065
1066 /// Method to return the pointer operand as a
1067 /// PointerType.
1068 Type *getPointerOperandType() const {
1069 return getPointerOperand()->getType();
1070 }
1071
1072 /// Returns the address space of the pointer operand.
1073 unsigned getPointerAddressSpace() const {
1074 return getPointerOperandType()->getPointerAddressSpace();
1075 }
1076
1077 /// Returns the pointer type returned by the GEP
1078 /// instruction, which may be a vector of pointers.
1079 static Type *getGEPReturnType(Type *ElTy, Value *Ptr,
1080 ArrayRef<Value *> IdxList) {
1081 PointerType *OrigPtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
1082 unsigned AddrSpace = OrigPtrTy->getAddressSpace();
1083 Type *ResultElemTy = checkGEPType(getIndexedType(ElTy, IdxList));
1084 Type *PtrTy = OrigPtrTy->isOpaque()
1085 ? PointerType::get(OrigPtrTy->getContext(), AddrSpace)
1086 : PointerType::get(ResultElemTy, AddrSpace);
1087 // Vector GEP
1088 if (auto *PtrVTy = dyn_cast<VectorType>(Ptr->getType())) {
1089 ElementCount EltCount = PtrVTy->getElementCount();
1090 return VectorType::get(PtrTy, EltCount);
1091 }
1092 for (Value *Index : IdxList)
1093 if (auto *IndexVTy = dyn_cast<VectorType>(Index->getType())) {
1094 ElementCount EltCount = IndexVTy->getElementCount();
1095 return VectorType::get(PtrTy, EltCount);
1096 }
1097 // Scalar GEP
1098 return PtrTy;
1099 }
1100
1101 unsigned getNumIndices() const { // Note: always non-negative
1102 return getNumOperands() - 1;
1103 }
1104
1105 bool hasIndices() const {
1106 return getNumOperands() > 1;
1107 }
1108
1109 /// Return true if all of the indices of this GEP are
1110 /// zeros. If so, the result pointer and the first operand have the same
1111 /// value, just potentially different types.
1112 bool hasAllZeroIndices() const;
1113
1114 /// Return true if all of the indices of this GEP are
1115 /// constant integers. If so, the result pointer and the first operand have
1116 /// a constant offset between them.
1117 bool hasAllConstantIndices() const;
1118
1119 /// Set or clear the inbounds flag on this GEP instruction.
1120 /// See LangRef.html for the meaning of inbounds on a getelementptr.
1121 void setIsInBounds(bool b = true);
1122
1123 /// Determine whether the GEP has the inbounds flag.
1124 bool isInBounds() const;
1125
1126 /// Accumulate the constant address offset of this GEP if possible.
1127 ///
1128 /// This routine accepts an APInt into which it will accumulate the constant
1129 /// offset of this GEP if the GEP is in fact constant. If the GEP is not
1130 /// all-constant, it returns false and the value of the offset APInt is
1131 /// undefined (it is *not* preserved!). The APInt passed into this routine
1132 /// must be at least as wide as the IntPtr type for the address space of
1133 /// the base GEP pointer.
1134 bool accumulateConstantOffset(const DataLayout &DL, APInt &Offset) const;
1135 bool collectOffset(const DataLayout &DL, unsigned BitWidth,
1136 MapVector<Value *, APInt> &VariableOffsets,
1137 APInt &ConstantOffset) const;
1138 // Methods for support type inquiry through isa, cast, and dyn_cast:
1139 static bool classof(const Instruction *I) {
1140 return (I->getOpcode() == Instruction::GetElementPtr);
1141 }
1142 static bool classof(const Value *V) {
1143 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1144 }
1145};
1146
1147template <>
1148struct OperandTraits<GetElementPtrInst> :
1149 public VariadicOperandTraits<GetElementPtrInst, 1> {
1150};
1151
1152GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
1153 ArrayRef<Value *> IdxList, unsigned Values,
1154 const Twine &NameStr,
1155 Instruction *InsertBefore)
1156 : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
1157 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
1158 Values, InsertBefore),
1159 SourceElementType(PointeeType),
1160 ResultElementType(getIndexedType(PointeeType, IdxList)) {
1161 assert(cast<PointerType>(getType()->getScalarType())(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "llvm/include/llvm/IR/Instructions.h", 1162, __extension__ __PRETTY_FUNCTION__
))
1162 ->isOpaqueOrPointeeTypeMatches(ResultElementType))(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "llvm/include/llvm/IR/Instructions.h", 1162, __extension__ __PRETTY_FUNCTION__
))
;
1163 init(Ptr, IdxList, NameStr);
1164}
1165
1166GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
1167 ArrayRef<Value *> IdxList, unsigned Values,
1168 const Twine &NameStr,
1169 BasicBlock *InsertAtEnd)
1170 : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
1171 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
1172 Values, InsertAtEnd),
1173 SourceElementType(PointeeType),
1174 ResultElementType(getIndexedType(PointeeType, IdxList)) {
1175 assert(cast<PointerType>(getType()->getScalarType())(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "llvm/include/llvm/IR/Instructions.h", 1176, __extension__ __PRETTY_FUNCTION__
))
1176 ->isOpaqueOrPointeeTypeMatches(ResultElementType))(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "llvm/include/llvm/IR/Instructions.h", 1176, __extension__ __PRETTY_FUNCTION__
))
;
1177 init(Ptr, IdxList, NameStr);
1178}
1179
1180DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GetElementPtrInst, Value)GetElementPtrInst::op_iterator GetElementPtrInst::op_begin() {
return OperandTraits<GetElementPtrInst>::op_begin(this
); } GetElementPtrInst::const_op_iterator GetElementPtrInst::
op_begin() const { return OperandTraits<GetElementPtrInst>
::op_begin(const_cast<GetElementPtrInst*>(this)); } GetElementPtrInst
::op_iterator GetElementPtrInst::op_end() { return OperandTraits
<GetElementPtrInst>::op_end(this); } GetElementPtrInst::
const_op_iterator GetElementPtrInst::op_end() const { return OperandTraits
<GetElementPtrInst>::op_end(const_cast<GetElementPtrInst
*>(this)); } Value *GetElementPtrInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<GetElementPtrInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<GetElementPtrInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1180, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<GetElementPtrInst
>::op_begin(const_cast<GetElementPtrInst*>(this))[i_nocapture
].get()); } void GetElementPtrInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<GetElementPtrInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<GetElementPtrInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1180, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<GetElementPtrInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned GetElementPtrInst::getNumOperands
() const { return OperandTraits<GetElementPtrInst>::operands
(this); } template <int Idx_nocapture> Use &GetElementPtrInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &GetElementPtrInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1181
1182//===----------------------------------------------------------------------===//
1183// ICmpInst Class
1184//===----------------------------------------------------------------------===//
1185
1186/// This instruction compares its operands according to the predicate given
1187/// to the constructor. It only operates on integers or pointers. The operands
1188/// must be identical types.
1189/// Represent an integer comparison operator.
1190class ICmpInst: public CmpInst {
1191 void AssertOK() {
1192 assert(isIntPredicate() &&(static_cast <bool> (isIntPredicate() && "Invalid ICmp predicate value"
) ? void (0) : __assert_fail ("isIntPredicate() && \"Invalid ICmp predicate value\""
, "llvm/include/llvm/IR/Instructions.h", 1193, __extension__ __PRETTY_FUNCTION__
))
1193 "Invalid ICmp predicate value")(static_cast <bool> (isIntPredicate() && "Invalid ICmp predicate value"
) ? void (0) : __assert_fail ("isIntPredicate() && \"Invalid ICmp predicate value\""
, "llvm/include/llvm/IR/Instructions.h", 1193, __extension__ __PRETTY_FUNCTION__
))
;
1194 assert(getOperand(0)->getType() == getOperand(1)->getType() &&(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to ICmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to ICmp instruction are not of the same type!\""
, "llvm/include/llvm/IR/Instructions.h", 1195, __extension__ __PRETTY_FUNCTION__
))
1195 "Both operands to ICmp instruction are not of the same type!")(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to ICmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to ICmp instruction are not of the same type!\""
, "llvm/include/llvm/IR/Instructions.h", 1195, __extension__ __PRETTY_FUNCTION__
))
;
1196 // Check that the operands are the right type
1197 assert((getOperand(0)->getType()->isIntOrIntVectorTy() ||(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "llvm/include/llvm/IR/Instructions.h", 1199, __extension__ __PRETTY_FUNCTION__
))
1198 getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "llvm/include/llvm/IR/Instructions.h", 1199, __extension__ __PRETTY_FUNCTION__
))
1199 "Invalid operand types for ICmp instruction")(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "llvm/include/llvm/IR/Instructions.h", 1199, __extension__ __PRETTY_FUNCTION__
))
;
1200 }
1201
1202protected:
1203 // Note: Instruction needs to be a friend here to call cloneImpl.
1204 friend class Instruction;
1205
1206 /// Clone an identical ICmpInst
1207 ICmpInst *cloneImpl() const;
1208
1209public:
1210 /// Constructor with insert-before-instruction semantics.
1211 ICmpInst(
1212 Instruction *InsertBefore, ///< Where to insert
1213 Predicate pred, ///< The predicate to use for the comparison
1214 Value *LHS, ///< The left-hand-side of the expression
1215 Value *RHS, ///< The right-hand-side of the expression
1216 const Twine &NameStr = "" ///< Name of the instruction
1217 ) : CmpInst(makeCmpResultType(LHS->getType()),
1218 Instruction::ICmp, pred, LHS, RHS, NameStr,
1219 InsertBefore) {
1220#ifndef NDEBUG
1221 AssertOK();
1222#endif
1223 }
1224
1225 /// Constructor with insert-at-end semantics.
1226 ICmpInst(
1227 BasicBlock &InsertAtEnd, ///< Block to insert into.
1228 Predicate pred, ///< The predicate to use for the comparison
1229 Value *LHS, ///< The left-hand-side of the expression
1230 Value *RHS, ///< The right-hand-side of the expression
1231 const Twine &NameStr = "" ///< Name of the instruction
1232 ) : CmpInst(makeCmpResultType(LHS->getType()),
1233 Instruction::ICmp, pred, LHS, RHS, NameStr,
1234 &InsertAtEnd) {
1235#ifndef NDEBUG
1236 AssertOK();
1237#endif
1238 }
1239
1240 /// Constructor with no-insertion semantics
1241 ICmpInst(
1242 Predicate pred, ///< The predicate to use for the comparison
1243 Value *LHS, ///< The left-hand-side of the expression
1244 Value *RHS, ///< The right-hand-side of the expression
1245 const Twine &NameStr = "" ///< Name of the instruction
1246 ) : CmpInst(makeCmpResultType(LHS->getType()),
1247 Instruction::ICmp, pred, LHS, RHS, NameStr) {
1248#ifndef NDEBUG
1249 AssertOK();
1250#endif
1251 }
1252
1253 /// For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
1254 /// @returns the predicate that would be the result if the operand were
1255 /// regarded as signed.
1256 /// Return the signed version of the predicate
1257 Predicate getSignedPredicate() const {
1258 return getSignedPredicate(getPredicate());
1259 }
1260
1261 /// This is a static version that you can use without an instruction.
1262 /// Return the signed version of the predicate.
1263 static Predicate getSignedPredicate(Predicate pred);
1264
1265 /// For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
1266 /// @returns the predicate that would be the result if the operand were
1267 /// regarded as unsigned.
1268 /// Return the unsigned version of the predicate
1269 Predicate getUnsignedPredicate() const {
1270 return getUnsignedPredicate(getPredicate());
1271 }
1272
1273 /// This is a static version that you can use without an instruction.
1274 /// Return the unsigned version of the predicate.
1275 static Predicate getUnsignedPredicate(Predicate pred);
1276
1277 /// Return true if this predicate is either EQ or NE. This also
1278 /// tests for commutativity.
1279 static bool isEquality(Predicate P) {
1280 return P == ICMP_EQ || P == ICMP_NE;
1281 }
1282
1283 /// Return true if this predicate is either EQ or NE. This also
1284 /// tests for commutativity.
1285 bool isEquality() const {
1286 return isEquality(getPredicate());
1287 }
1288
1289 /// @returns true if the predicate of this ICmpInst is commutative
1290 /// Determine if this relation is commutative.
1291 bool isCommutative() const { return isEquality(); }
1292
1293 /// Return true if the predicate is relational (not EQ or NE).
1294 ///
1295 bool isRelational() const {
1296 return !isEquality();
1297 }
1298
1299 /// Return true if the predicate is relational (not EQ or NE).
1300 ///
1301 static bool isRelational(Predicate P) {
1302 return !isEquality(P);
1303 }
1304
1305 /// Return true if the predicate is SGT or UGT.
1306 ///
1307 static bool isGT(Predicate P) {
1308 return P == ICMP_SGT || P == ICMP_UGT;
1309 }
1310
1311 /// Return true if the predicate is SLT or ULT.
1312 ///
1313 static bool isLT(Predicate P) {
1314 return P == ICMP_SLT || P == ICMP_ULT;
1315 }
1316
1317 /// Return true if the predicate is SGE or UGE.
1318 ///
1319 static bool isGE(Predicate P) {
1320 return P == ICMP_SGE || P == ICMP_UGE;
1321 }
1322
1323 /// Return true if the predicate is SLE or ULE.
1324 ///
1325 static bool isLE(Predicate P) {
1326 return P == ICMP_SLE || P == ICMP_ULE;
1327 }
1328
1329 /// Returns the sequence of all ICmp predicates.
1330 ///
1331 static auto predicates() { return ICmpPredicates(); }
1332
1333 /// Exchange the two operands to this instruction in such a way that it does
1334 /// not modify the semantics of the instruction. The predicate value may be
1335 /// changed to retain the same result if the predicate is order dependent
1336 /// (e.g. ult).
1337 /// Swap operands and adjust predicate.
1338 void swapOperands() {
1339 setPredicate(getSwappedPredicate());
1340 Op<0>().swap(Op<1>());
1341 }
1342
1343 /// Return result of `LHS Pred RHS` comparison.
1344 static bool compare(const APInt &LHS, const APInt &RHS,
1345 ICmpInst::Predicate Pred);
1346
1347 // Methods for support type inquiry through isa, cast, and dyn_cast:
1348 static bool classof(const Instruction *I) {
1349 return I->getOpcode() == Instruction::ICmp;
1350 }
1351 static bool classof(const Value *V) {
1352 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1353 }
1354};
1355
1356//===----------------------------------------------------------------------===//
1357// FCmpInst Class
1358//===----------------------------------------------------------------------===//
1359
1360/// This instruction compares its operands according to the predicate given
1361/// to the constructor. It only operates on floating point values or packed
1362/// vectors of floating point values. The operands must be identical types.
1363/// Represents a floating point comparison operator.
1364class FCmpInst: public CmpInst {
1365 void AssertOK() {
1366 assert(isFPPredicate() && "Invalid FCmp predicate value")(static_cast <bool> (isFPPredicate() && "Invalid FCmp predicate value"
) ? void (0) : __assert_fail ("isFPPredicate() && \"Invalid FCmp predicate value\""
, "llvm/include/llvm/IR/Instructions.h", 1366, __extension__ __PRETTY_FUNCTION__
))
;
1367 assert(getOperand(0)->getType() == getOperand(1)->getType() &&(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to FCmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to FCmp instruction are not of the same type!\""
, "llvm/include/llvm/IR/Instructions.h", 1368, __extension__ __PRETTY_FUNCTION__
))
1368 "Both operands to FCmp instruction are not of the same type!")(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to FCmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to FCmp instruction are not of the same type!\""
, "llvm/include/llvm/IR/Instructions.h", 1368, __extension__ __PRETTY_FUNCTION__
))
;
1369 // Check that the operands are the right type
1370 assert(getOperand(0)->getType()->isFPOrFPVectorTy() &&(static_cast <bool> (getOperand(0)->getType()->isFPOrFPVectorTy
() && "Invalid operand types for FCmp instruction") ?
void (0) : __assert_fail ("getOperand(0)->getType()->isFPOrFPVectorTy() && \"Invalid operand types for FCmp instruction\""
, "llvm/include/llvm/IR/Instructions.h", 1371, __extension__ __PRETTY_FUNCTION__
))
1371 "Invalid operand types for FCmp instruction")(static_cast <bool> (getOperand(0)->getType()->isFPOrFPVectorTy
() && "Invalid operand types for FCmp instruction") ?
void (0) : __assert_fail ("getOperand(0)->getType()->isFPOrFPVectorTy() && \"Invalid operand types for FCmp instruction\""
, "llvm/include/llvm/IR/Instructions.h", 1371, __extension__ __PRETTY_FUNCTION__
))
;
1372 }
1373
1374protected:
1375 // Note: Instruction needs to be a friend here to call cloneImpl.
1376 friend class Instruction;
1377
1378 /// Clone an identical FCmpInst
1379 FCmpInst *cloneImpl() const;
1380
1381public:
1382 /// Constructor with insert-before-instruction semantics.
1383 FCmpInst(
1384 Instruction *InsertBefore, ///< Where to insert
1385 Predicate pred, ///< The predicate to use for the comparison
1386 Value *LHS, ///< The left-hand-side of the expression
1387 Value *RHS, ///< The right-hand-side of the expression
1388 const Twine &NameStr = "" ///< Name of the instruction
1389 ) : CmpInst(makeCmpResultType(LHS->getType()),
1390 Instruction::FCmp, pred, LHS, RHS, NameStr,
1391 InsertBefore) {
1392 AssertOK();
1393 }
1394
1395 /// Constructor with insert-at-end semantics.
1396 FCmpInst(
1397 BasicBlock &InsertAtEnd, ///< Block to insert into.
1398 Predicate pred, ///< The predicate to use for the comparison
1399 Value *LHS, ///< The left-hand-side of the expression
1400 Value *RHS, ///< The right-hand-side of the expression
1401 const Twine &NameStr = "" ///< Name of the instruction
1402 ) : CmpInst(makeCmpResultType(LHS->getType()),
1403 Instruction::FCmp, pred, LHS, RHS, NameStr,
1404 &InsertAtEnd) {
1405 AssertOK();
1406 }
1407
1408 /// Constructor with no-insertion semantics
1409 FCmpInst(
1410 Predicate Pred, ///< The predicate to use for the comparison
1411 Value *LHS, ///< The left-hand-side of the expression
1412 Value *RHS, ///< The right-hand-side of the expression
1413 const Twine &NameStr = "", ///< Name of the instruction
1414 Instruction *FlagsSource = nullptr
1415 ) : CmpInst(makeCmpResultType(LHS->getType()), Instruction::FCmp, Pred, LHS,
1416 RHS, NameStr, nullptr, FlagsSource) {
1417 AssertOK();
1418 }
1419
1420 /// @returns true if the predicate of this instruction is EQ or NE.
1421 /// Determine if this is an equality predicate.
1422 static bool isEquality(Predicate Pred) {
1423 return Pred == FCMP_OEQ || Pred == FCMP_ONE || Pred == FCMP_UEQ ||
1424 Pred == FCMP_UNE;
1425 }
1426
1427 /// @returns true if the predicate of this instruction is EQ or NE.
1428 /// Determine if this is an equality predicate.
1429 bool isEquality() const { return isEquality(getPredicate()); }
1430
1431 /// @returns true if the predicate of this instruction is commutative.
1432 /// Determine if this is a commutative predicate.
1433 bool isCommutative() const {
1434 return isEquality() ||
1435 getPredicate() == FCMP_FALSE ||
1436 getPredicate() == FCMP_TRUE ||
1437 getPredicate() == FCMP_ORD ||
1438 getPredicate() == FCMP_UNO;
1439 }
1440
1441 /// @returns true if the predicate is relational (not EQ or NE).
1442 /// Determine if this a relational predicate.
1443 bool isRelational() const { return !isEquality(); }
1444
1445 /// Exchange the two operands to this instruction in such a way that it does
1446 /// not modify the semantics of the instruction. The predicate value may be
1447 /// changed to retain the same result if the predicate is order dependent
1448 /// (e.g. ult).
1449 /// Swap operands and adjust predicate.
1450 void swapOperands() {
1451 setPredicate(getSwappedPredicate());
1452 Op<0>().swap(Op<1>());
1453 }
1454
1455 /// Returns the sequence of all FCmp predicates.
1456 ///
1457 static auto predicates() { return FCmpPredicates(); }
1458
1459 /// Return result of `LHS Pred RHS` comparison.
1460 static bool compare(const APFloat &LHS, const APFloat &RHS,
1461 FCmpInst::Predicate Pred);
1462
1463 /// Methods for support type inquiry through isa, cast, and dyn_cast:
1464 static bool classof(const Instruction *I) {
1465 return I->getOpcode() == Instruction::FCmp;
1466 }
1467 static bool classof(const Value *V) {
1468 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1469 }
1470};
1471
1472//===----------------------------------------------------------------------===//
1473/// This class represents a function call, abstracting a target
1474/// machine's calling convention. This class uses low bit of the SubClassData
1475/// field to indicate whether or not this is a tail call. The rest of the bits
1476/// hold the calling convention of the call.
1477///
1478class CallInst : public CallBase {
1479 CallInst(const CallInst &CI);
1480
1481 /// Construct a CallInst given a range of arguments.
1482 /// Construct a CallInst from a range of arguments
1483 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1484 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1485 Instruction *InsertBefore);
1486
1487 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1488 const Twine &NameStr, Instruction *InsertBefore)
1489 : CallInst(Ty, Func, Args, None, NameStr, InsertBefore) {}
1490
1491 /// Construct a CallInst given a range of arguments.
1492 /// Construct a CallInst from a range of arguments
1493 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1494 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1495 BasicBlock *InsertAtEnd);
1496
1497 explicit CallInst(FunctionType *Ty, Value *F, const Twine &NameStr,
1498 Instruction *InsertBefore);
1499
1500 CallInst(FunctionType *ty, Value *F, const Twine &NameStr,
1501 BasicBlock *InsertAtEnd);
1502
1503 void init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
1504 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
1505 void init(FunctionType *FTy, Value *Func, const Twine &NameStr);
1506
1507 /// Compute the number of operands to allocate.
1508 static int ComputeNumOperands(int NumArgs, int NumBundleInputs = 0) {
1509 // We need one operand for the called function, plus the input operand
1510 // counts provided.
1511 return 1 + NumArgs + NumBundleInputs;
1512 }
1513
1514protected:
1515 // Note: Instruction needs to be a friend here to call cloneImpl.
1516 friend class Instruction;
1517
1518 CallInst *cloneImpl() const;
1519
1520public:
1521 static CallInst *Create(FunctionType *Ty, Value *F, const Twine &NameStr = "",
1522 Instruction *InsertBefore = nullptr) {
1523 return new (ComputeNumOperands(0)) CallInst(Ty, F, NameStr, InsertBefore);
1524 }
1525
1526 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1527 const Twine &NameStr,
1528 Instruction *InsertBefore = nullptr) {
1529 return new (ComputeNumOperands(Args.size()))
1530 CallInst(Ty, Func, Args, None, NameStr, InsertBefore);
1531 }
1532
1533 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1534 ArrayRef<OperandBundleDef> Bundles = None,
1535 const Twine &NameStr = "",
1536 Instruction *InsertBefore = nullptr) {
1537 const int NumOperands =
1538 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
1539 const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
1540
1541 return new (NumOperands, DescriptorBytes)
1542 CallInst(Ty, Func, Args, Bundles, NameStr, InsertBefore);
1543 }
1544
1545 static CallInst *Create(FunctionType *Ty, Value *F, const Twine &NameStr,
1546 BasicBlock *InsertAtEnd) {
1547 return new (ComputeNumOperands(0)) CallInst(Ty, F, NameStr, InsertAtEnd);
1548 }
1549
1550 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1551 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1552 return new (ComputeNumOperands(Args.size()))
1553 CallInst(Ty, Func, Args, None, NameStr, InsertAtEnd);
1554 }
1555
1556 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1557 ArrayRef<OperandBundleDef> Bundles,
1558 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1559 const int NumOperands =
1560 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
1561 const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
1562
1563 return new (NumOperands, DescriptorBytes)
1564 CallInst(Ty, Func, Args, Bundles, NameStr, InsertAtEnd);
1565 }
1566
1567 static CallInst *Create(FunctionCallee Func, const Twine &NameStr = "",
1568 Instruction *InsertBefore = nullptr) {
1569 return Create(Func.getFunctionType(), Func.getCallee(), NameStr,
1570 InsertBefore);
1571 }
1572
1573 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1574 ArrayRef<OperandBundleDef> Bundles = None,
1575 const Twine &NameStr = "",
1576 Instruction *InsertBefore = nullptr) {
1577 return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles,
1578 NameStr, InsertBefore);
1579 }
1580
1581 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1582 const Twine &NameStr,
1583 Instruction *InsertBefore = nullptr) {
1584 return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr,
1585 InsertBefore);
1586 }
1587
1588 static CallInst *Create(FunctionCallee Func, const Twine &NameStr,
1589 BasicBlock *InsertAtEnd) {
1590 return Create(Func.getFunctionType(), Func.getCallee(), NameStr,
1591 InsertAtEnd);
1592 }
1593
1594 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1595 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1596 return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr,
1597 InsertAtEnd);
1598 }
1599
1600 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1601 ArrayRef<OperandBundleDef> Bundles,
1602 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1603 return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles,
1604 NameStr, InsertAtEnd);
1605 }
1606
1607 /// Create a clone of \p CI with a different set of operand bundles and
1608 /// insert it before \p InsertPt.
1609 ///
1610 /// The returned call instruction is identical \p CI in every way except that
1611 /// the operand bundles for the new instruction are set to the operand bundles
1612 /// in \p Bundles.
1613 static CallInst *Create(CallInst *CI, ArrayRef<OperandBundleDef> Bundles,
1614 Instruction *InsertPt = nullptr);
1615
1616 /// Generate the IR for a call to malloc:
1617 /// 1. Compute the malloc call's argument as the specified type's size,
1618 /// possibly multiplied by the array size if the array size is not
1619 /// constant 1.
1620 /// 2. Call malloc with that argument.
1621 /// 3. Bitcast the result of the malloc call to the specified type.
1622 static Instruction *CreateMalloc(Instruction *InsertBefore, Type *IntPtrTy,
1623 Type *AllocTy, Value *AllocSize,
1624 Value *ArraySize = nullptr,
1625 Function *MallocF = nullptr,
1626 const Twine &Name = "");
1627 static Instruction *CreateMalloc(BasicBlock *InsertAtEnd, Type *IntPtrTy,
1628 Type *AllocTy, Value *AllocSize,
1629 Value *ArraySize = nullptr,
1630 Function *MallocF = nullptr,
1631 const Twine &Name = "");
1632 static Instruction *CreateMalloc(Instruction *InsertBefore, Type *IntPtrTy,
1633 Type *AllocTy, Value *AllocSize,
1634 Value *ArraySize = nullptr,
1635 ArrayRef<OperandBundleDef> Bundles = None,
1636 Function *MallocF = nullptr,
1637 const Twine &Name = "");
1638 static Instruction *CreateMalloc(BasicBlock *InsertAtEnd, Type *IntPtrTy,
1639 Type *AllocTy, Value *AllocSize,
1640 Value *ArraySize = nullptr,
1641 ArrayRef<OperandBundleDef> Bundles = None,
1642 Function *MallocF = nullptr,
1643 const Twine &Name = "");
1644 /// Generate the IR for a call to the builtin free function.
1645 static Instruction *CreateFree(Value *Source, Instruction *InsertBefore);
1646 static Instruction *CreateFree(Value *Source, BasicBlock *InsertAtEnd);
1647 static Instruction *CreateFree(Value *Source,
1648 ArrayRef<OperandBundleDef> Bundles,
1649 Instruction *InsertBefore);
1650 static Instruction *CreateFree(Value *Source,
1651 ArrayRef<OperandBundleDef> Bundles,
1652 BasicBlock *InsertAtEnd);
1653
1654 // Note that 'musttail' implies 'tail'.
1655 enum TailCallKind : unsigned {
1656 TCK_None = 0,
1657 TCK_Tail = 1,
1658 TCK_MustTail = 2,
1659 TCK_NoTail = 3,
1660 TCK_LAST = TCK_NoTail
1661 };
1662
1663 using TailCallKindField = Bitfield::Element<TailCallKind, 0, 2, TCK_LAST>;
1664 static_assert(
1665 Bitfield::areContiguous<TailCallKindField, CallBase::CallingConvField>(),
1666 "Bitfields must be contiguous");
1667
1668 TailCallKind getTailCallKind() const {
1669 return getSubclassData<TailCallKindField>();
1670 }
1671
1672 bool isTailCall() const {
1673 TailCallKind Kind = getTailCallKind();
1674 return Kind == TCK_Tail || Kind == TCK_MustTail;
1675 }
1676
1677 bool isMustTailCall() const { return getTailCallKind() == TCK_MustTail; }
1678
1679 bool isNoTailCall() const { return getTailCallKind() == TCK_NoTail; }
1680
1681 void setTailCallKind(TailCallKind TCK) {
1682 setSubclassData<TailCallKindField>(TCK);
1683 }
1684
1685 void setTailCall(bool IsTc = true) {
1686 setTailCallKind(IsTc ? TCK_Tail : TCK_None);
1687 }
1688
1689 /// Return true if the call can return twice
1690 bool canReturnTwice() const { return hasFnAttr(Attribute::ReturnsTwice); }
1691 void setCanReturnTwice() { addFnAttr(Attribute::ReturnsTwice); }
1692
1693 // Methods for support type inquiry through isa, cast, and dyn_cast:
1694 static bool classof(const Instruction *I) {
1695 return I->getOpcode() == Instruction::Call;
1696 }
1697 static bool classof(const Value *V) {
1698 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1699 }
1700
1701 /// Updates profile metadata by scaling it by \p S / \p T.
1702 void updateProfWeight(uint64_t S, uint64_t T);
1703
1704private:
1705 // Shadow Instruction::setInstructionSubclassData with a private forwarding
1706 // method so that subclasses cannot accidentally use it.
1707 template <typename Bitfield>
1708 void setSubclassData(typename Bitfield::Type Value) {
1709 Instruction::setSubclassData<Bitfield>(Value);
1710 }
1711};
1712
1713CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1714 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1715 BasicBlock *InsertAtEnd)
1716 : CallBase(Ty->getReturnType(), Instruction::Call,
1717 OperandTraits<CallBase>::op_end(this) -
1718 (Args.size() + CountBundleInputs(Bundles) + 1),
1719 unsigned(Args.size() + CountBundleInputs(Bundles) + 1),
1720 InsertAtEnd) {
1721 init(Ty, Func, Args, Bundles, NameStr);
1722}
1723
1724CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1725 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1726 Instruction *InsertBefore)
1727 : CallBase(Ty->getReturnType(), Instruction::Call,
1728 OperandTraits<CallBase>::op_end(this) -
1729 (Args.size() + CountBundleInputs(Bundles) + 1),
1730 unsigned(Args.size() + CountBundleInputs(Bundles) + 1),
1731 InsertBefore) {
1732 init(Ty, Func, Args, Bundles, NameStr);
1733}
1734
1735//===----------------------------------------------------------------------===//
1736// SelectInst Class
1737//===----------------------------------------------------------------------===//
1738
1739/// This class represents the LLVM 'select' instruction.
1740///
1741class SelectInst : public Instruction {
1742 SelectInst(Value *C, Value *S1, Value *S2, const Twine &NameStr,
1743 Instruction *InsertBefore)
1744 : Instruction(S1->getType(), Instruction::Select,
1745 &Op<0>(), 3, InsertBefore) {
1746 init(C, S1, S2);
1747 setName(NameStr);
1748 }
1749
1750 SelectInst(Value *C, Value *S1, Value *S2, const Twine &NameStr,
1751 BasicBlock *InsertAtEnd)
1752 : Instruction(S1->getType(), Instruction::Select,
1753 &Op<0>(), 3, InsertAtEnd) {
1754 init(C, S1, S2);
1755 setName(NameStr);
1756 }
1757
1758 void init(Value *C, Value *S1, Value *S2) {
1759 assert(!areInvalidOperands(C, S1, S2) && "Invalid operands for select")(static_cast <bool> (!areInvalidOperands(C, S1, S2) &&
"Invalid operands for select") ? void (0) : __assert_fail ("!areInvalidOperands(C, S1, S2) && \"Invalid operands for select\""
, "llvm/include/llvm/IR/Instructions.h", 1759, __extension__ __PRETTY_FUNCTION__
))
;
1760 Op<0>() = C;
1761 Op<1>() = S1;
1762 Op<2>() = S2;
1763 }
1764
1765protected:
1766 // Note: Instruction needs to be a friend here to call cloneImpl.
1767 friend class Instruction;
1768
1769 SelectInst *cloneImpl() const;
1770
1771public:
1772 static SelectInst *Create(Value *C, Value *S1, Value *S2,
1773 const Twine &NameStr = "",
1774 Instruction *InsertBefore = nullptr,
1775 Instruction *MDFrom = nullptr) {
1776 SelectInst *Sel = new(3) SelectInst(C, S1, S2, NameStr, InsertBefore);
1777 if (MDFrom)
1778 Sel->copyMetadata(*MDFrom);
1779 return Sel;
1780 }
1781
1782 static SelectInst *Create(Value *C, Value *S1, Value *S2,
1783 const Twine &NameStr,
1784 BasicBlock *InsertAtEnd) {
1785 return new(3) SelectInst(C, S1, S2, NameStr, InsertAtEnd);
1786 }
1787
1788 const Value *getCondition() const { return Op<0>(); }
1789 const Value *getTrueValue() const { return Op<1>(); }
1790 const Value *getFalseValue() const { return Op<2>(); }
1791 Value *getCondition() { return Op<0>(); }
1792 Value *getTrueValue() { return Op<1>(); }
1793 Value *getFalseValue() { return Op<2>(); }
1794
1795 void setCondition(Value *V) { Op<0>() = V; }
1796 void setTrueValue(Value *V) { Op<1>() = V; }
1797 void setFalseValue(Value *V) { Op<2>() = V; }
1798
1799 /// Swap the true and false values of the select instruction.
1800 /// This doesn't swap prof metadata.
1801 void swapValues() { Op<1>().swap(Op<2>()); }
1802
1803 /// Return a string if the specified operands are invalid
1804 /// for a select operation, otherwise return null.
1805 static const char *areInvalidOperands(Value *Cond, Value *True, Value *False);
1806
1807 /// Transparently provide more efficient getOperand methods.
1808 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1809
1810 OtherOps getOpcode() const {
1811 return static_cast<OtherOps>(Instruction::getOpcode());
1812 }
1813
1814 // Methods for support type inquiry through isa, cast, and dyn_cast:
1815 static bool classof(const Instruction *I) {
1816 return I->getOpcode() == Instruction::Select;
1817 }
1818 static bool classof(const Value *V) {
1819 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1820 }
1821};
1822
1823template <>
1824struct OperandTraits<SelectInst> : public FixedNumOperandTraits<SelectInst, 3> {
1825};
1826
1827DEFINE_TRANSPARENT_OPERAND_ACCESSORS(SelectInst, Value)SelectInst::op_iterator SelectInst::op_begin() { return OperandTraits
<SelectInst>::op_begin(this); } SelectInst::const_op_iterator
SelectInst::op_begin() const { return OperandTraits<SelectInst
>::op_begin(const_cast<SelectInst*>(this)); } SelectInst
::op_iterator SelectInst::op_end() { return OperandTraits<
SelectInst>::op_end(this); } SelectInst::const_op_iterator
SelectInst::op_end() const { return OperandTraits<SelectInst
>::op_end(const_cast<SelectInst*>(this)); } Value *SelectInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<SelectInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<SelectInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1827, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<SelectInst
>::op_begin(const_cast<SelectInst*>(this))[i_nocapture
].get()); } void SelectInst::setOperand(unsigned i_nocapture,
Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<SelectInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<SelectInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1827, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<SelectInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned SelectInst::getNumOperands() const
{ return OperandTraits<SelectInst>::operands(this); } template
<int Idx_nocapture> Use &SelectInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &SelectInst::Op() const { return
this->OpFrom<Idx_nocapture>(this); }
1828
1829//===----------------------------------------------------------------------===//
1830// VAArgInst Class
1831//===----------------------------------------------------------------------===//
1832
1833/// This class represents the va_arg llvm instruction, which returns
1834/// an argument of the specified type given a va_list and increments that list
1835///
1836class VAArgInst : public UnaryInstruction {
1837protected:
1838 // Note: Instruction needs to be a friend here to call cloneImpl.
1839 friend class Instruction;
1840
1841 VAArgInst *cloneImpl() const;
1842
1843public:
1844 VAArgInst(Value *List, Type *Ty, const Twine &NameStr = "",
1845 Instruction *InsertBefore = nullptr)
1846 : UnaryInstruction(Ty, VAArg, List, InsertBefore) {
1847 setName(NameStr);
1848 }
1849
1850 VAArgInst(Value *List, Type *Ty, const Twine &NameStr,
1851 BasicBlock *InsertAtEnd)
1852 : UnaryInstruction(Ty, VAArg, List, InsertAtEnd) {
1853 setName(NameStr);
1854 }
1855
1856 Value *getPointerOperand() { return getOperand(0); }
1857 const Value *getPointerOperand() const { return getOperand(0); }
1858 static unsigned getPointerOperandIndex() { return 0U; }
1859
1860 // Methods for support type inquiry through isa, cast, and dyn_cast:
1861 static bool classof(const Instruction *I) {
1862 return I->getOpcode() == VAArg;
1863 }
1864 static bool classof(const Value *V) {
1865 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1866 }
1867};
1868
1869//===----------------------------------------------------------------------===//
1870// ExtractElementInst Class
1871//===----------------------------------------------------------------------===//
1872
1873/// This instruction extracts a single (scalar)
1874/// element from a VectorType value
1875///
1876class ExtractElementInst : public Instruction {
1877 ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr = "",
1878 Instruction *InsertBefore = nullptr);
1879 ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr,
1880 BasicBlock *InsertAtEnd);
1881
1882protected:
1883 // Note: Instruction needs to be a friend here to call cloneImpl.
1884 friend class Instruction;
1885
1886 ExtractElementInst *cloneImpl() const;
1887
1888public:
1889 static ExtractElementInst *Create(Value *Vec, Value *Idx,
1890 const Twine &NameStr = "",
1891 Instruction *InsertBefore = nullptr) {
1892 return new(2) ExtractElementInst(Vec, Idx, NameStr, InsertBefore);
1893 }
1894
1895 static ExtractElementInst *Create(Value *Vec, Value *Idx,
1896 const Twine &NameStr,
1897 BasicBlock *InsertAtEnd) {
1898 return new(2) ExtractElementInst(Vec, Idx, NameStr, InsertAtEnd);
1899 }
1900
1901 /// Return true if an extractelement instruction can be
1902 /// formed with the specified operands.
1903 static bool isValidOperands(const Value *Vec, const Value *Idx);
1904
1905 Value *getVectorOperand() { return Op<0>(); }
1906 Value *getIndexOperand() { return Op<1>(); }
1907 const Value *getVectorOperand() const { return Op<0>(); }
1908 const Value *getIndexOperand() const { return Op<1>(); }
1909
1910 VectorType *getVectorOperandType() const {
1911 return cast<VectorType>(getVectorOperand()->getType());
1912 }
1913
1914 /// Transparently provide more efficient getOperand methods.
1915 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1916
1917 // Methods for support type inquiry through isa, cast, and dyn_cast:
1918 static bool classof(const Instruction *I) {
1919 return I->getOpcode() == Instruction::ExtractElement;
1920 }
1921 static bool classof(const Value *V) {
1922 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1923 }
1924};
1925
1926template <>
1927struct OperandTraits<ExtractElementInst> :
1928 public FixedNumOperandTraits<ExtractElementInst, 2> {
1929};
1930
1931DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ExtractElementInst, Value)ExtractElementInst::op_iterator ExtractElementInst::op_begin(
) { return OperandTraits<ExtractElementInst>::op_begin(
this); } ExtractElementInst::const_op_iterator ExtractElementInst
::op_begin() const { return OperandTraits<ExtractElementInst
>::op_begin(const_cast<ExtractElementInst*>(this)); }
ExtractElementInst::op_iterator ExtractElementInst::op_end()
{ return OperandTraits<ExtractElementInst>::op_end(this
); } ExtractElementInst::const_op_iterator ExtractElementInst
::op_end() const { return OperandTraits<ExtractElementInst
>::op_end(const_cast<ExtractElementInst*>(this)); } Value
*ExtractElementInst::getOperand(unsigned i_nocapture) const {
(static_cast <bool> (i_nocapture < OperandTraits<
ExtractElementInst>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ExtractElementInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1931, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<ExtractElementInst
>::op_begin(const_cast<ExtractElementInst*>(this))[i_nocapture
].get()); } void ExtractElementInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ExtractElementInst>::operands(this)
&& "setOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<ExtractElementInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1931, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<ExtractElementInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned ExtractElementInst::getNumOperands
() const { return OperandTraits<ExtractElementInst>::operands
(this); } template <int Idx_nocapture> Use &ExtractElementInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &ExtractElementInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1932
1933//===----------------------------------------------------------------------===//
1934// InsertElementInst Class
1935//===----------------------------------------------------------------------===//
1936
1937/// This instruction inserts a single (scalar)
1938/// element into a VectorType value
1939///
1940class InsertElementInst : public Instruction {
1941 InsertElementInst(Value *Vec, Value *NewElt, Value *Idx,
1942 const Twine &NameStr = "",
1943 Instruction *InsertBefore = nullptr);
1944 InsertElementInst(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr,
1945 BasicBlock *InsertAtEnd);
1946
1947protected:
1948 // Note: Instruction needs to be a friend here to call cloneImpl.
1949 friend class Instruction;
1950
1951 InsertElementInst *cloneImpl() const;
1952
1953public:
1954 static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
1955 const Twine &NameStr = "",
1956 Instruction *InsertBefore = nullptr) {
1957 return new(3) InsertElementInst(Vec, NewElt, Idx, NameStr, InsertBefore);
1958 }
1959
1960 static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
1961 const Twine &NameStr,
1962 BasicBlock *InsertAtEnd) {
1963 return new(3) InsertElementInst(Vec, NewElt, Idx, NameStr, InsertAtEnd);
1964 }
1965
1966 /// Return true if an insertelement instruction can be
1967 /// formed with the specified operands.
1968 static bool isValidOperands(const Value *Vec, const Value *NewElt,
1969 const Value *Idx);
1970
1971 /// Overload to return most specific vector type.
1972 ///
1973 VectorType *getType() const {
1974 return cast<VectorType>(Instruction::getType());
1975 }
1976
1977 /// Transparently provide more efficient getOperand methods.
1978 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1979
1980 // Methods for support type inquiry through isa, cast, and dyn_cast:
1981 static bool classof(const Instruction *I) {
1982 return I->getOpcode() == Instruction::InsertElement;
1983 }
1984 static bool classof(const Value *V) {
1985 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1986 }
1987};
1988
1989template <>
1990struct OperandTraits<InsertElementInst> :
1991 public FixedNumOperandTraits<InsertElementInst, 3> {
1992};
1993
1994DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementInst, Value)InsertElementInst::op_iterator InsertElementInst::op_begin() {
return OperandTraits<InsertElementInst>::op_begin(this
); } InsertElementInst::const_op_iterator InsertElementInst::
op_begin() const { return OperandTraits<InsertElementInst>
::op_begin(const_cast<InsertElementInst*>(this)); } InsertElementInst
::op_iterator InsertElementInst::op_end() { return OperandTraits
<InsertElementInst>::op_end(this); } InsertElementInst::
const_op_iterator InsertElementInst::op_end() const { return OperandTraits
<InsertElementInst>::op_end(const_cast<InsertElementInst
*>(this)); } Value *InsertElementInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<InsertElementInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertElementInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1994, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<InsertElementInst
>::op_begin(const_cast<InsertElementInst*>(this))[i_nocapture
].get()); } void InsertElementInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<InsertElementInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertElementInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1994, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<InsertElementInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned InsertElementInst::getNumOperands
() const { return OperandTraits<InsertElementInst>::operands
(this); } template <int Idx_nocapture> Use &InsertElementInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &InsertElementInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1995
1996//===----------------------------------------------------------------------===//
1997// ShuffleVectorInst Class
1998//===----------------------------------------------------------------------===//
1999
2000constexpr int UndefMaskElem = -1;
2001
2002/// This instruction constructs a fixed permutation of two
2003/// input vectors.
2004///
2005/// For each element of the result vector, the shuffle mask selects an element
2006/// from one of the input vectors to copy to the result. Non-negative elements
2007/// in the mask represent an index into the concatenated pair of input vectors.
2008/// UndefMaskElem (-1) specifies that the result element is undefined.
2009///
2010/// For scalable vectors, all the elements of the mask must be 0 or -1. This
2011/// requirement may be relaxed in the future.
2012class ShuffleVectorInst : public Instruction {
2013 SmallVector<int, 4> ShuffleMask;
2014 Constant *ShuffleMaskForBitcode;
2015
2016protected:
2017 // Note: Instruction needs to be a friend here to call cloneImpl.
2018 friend class Instruction;
2019
2020 ShuffleVectorInst *cloneImpl() const;
2021
2022public:
2023 ShuffleVectorInst(Value *V1, Value *Mask, const Twine &NameStr = "",
2024 Instruction *InsertBefore = nullptr);
2025 ShuffleVectorInst(Value *V1, Value *Mask, const Twine &NameStr,
2026 BasicBlock *InsertAtEnd);
2027 ShuffleVectorInst(Value *V1, ArrayRef<int> Mask, const Twine &NameStr = "",
2028 Instruction *InsertBefore = nullptr);
2029 ShuffleVectorInst(Value *V1, ArrayRef<int> Mask, const Twine &NameStr,
2030 BasicBlock *InsertAtEnd);
2031 ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
2032 const Twine &NameStr = "",
2033 Instruction *InsertBefor = nullptr);
2034 ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
2035 const Twine &NameStr, BasicBlock *InsertAtEnd);
2036 ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
2037 const Twine &NameStr = "",
2038 Instruction *InsertBefor = nullptr);
2039 ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
2040 const Twine &NameStr, BasicBlock *InsertAtEnd);
2041
2042 void *operator new(size_t S) { return User::operator new(S, 2); }
2043 void operator delete(void *Ptr) { return User::operator delete(Ptr); }
2044
2045 /// Swap the operands and adjust the mask to preserve the semantics
2046 /// of the instruction.
2047 void commute();
2048
2049 /// Return true if a shufflevector instruction can be
2050 /// formed with the specified operands.
2051 static bool isValidOperands(const Value *V1, const Value *V2,
2052 const Value *Mask);
2053 static bool isValidOperands(const Value *V1, const Value *V2,
2054 ArrayRef<int> Mask);
2055
2056 /// Overload to return most specific vector type.
2057 ///
2058 VectorType *getType() const {
2059 return cast<VectorType>(Instruction::getType());
2060 }
2061
2062 /// Transparently provide more efficient getOperand methods.
2063 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2064
2065 /// Return the shuffle mask value of this instruction for the given element
2066 /// index. Return UndefMaskElem if the element is undef.
2067 int getMaskValue(unsigned Elt) const { return ShuffleMask[Elt]; }
2068
2069 /// Convert the input shuffle mask operand to a vector of integers. Undefined
2070 /// elements of the mask are returned as UndefMaskElem.
2071 static void getShuffleMask(const Constant *Mask,
2072 SmallVectorImpl<int> &Result);
2073
2074 /// Return the mask for this instruction as a vector of integers. Undefined
2075 /// elements of the mask are returned as UndefMaskElem.
2076 void getShuffleMask(SmallVectorImpl<int> &Result) const {
2077 Result.assign(ShuffleMask.begin(), ShuffleMask.end());
2078 }
2079
2080 /// Return the mask for this instruction, for use in bitcode.
2081 ///
2082 /// TODO: This is temporary until we decide a new bitcode encoding for
2083 /// shufflevector.
2084 Constant *getShuffleMaskForBitcode() const { return ShuffleMaskForBitcode; }
2085
2086 static Constant *convertShuffleMaskForBitcode(ArrayRef<int> Mask,
2087 Type *ResultTy);
2088
2089 void setShuffleMask(ArrayRef<int> Mask);
2090
2091 ArrayRef<int> getShuffleMask() const { return ShuffleMask; }
2092
2093 /// Return true if this shuffle returns a vector with a different number of
2094 /// elements than its source vectors.
2095 /// Examples: shufflevector <4 x n> A, <4 x n> B, <1,2,3>
2096 /// shufflevector <4 x n> A, <4 x n> B, <1,2,3,4,5>
2097 bool changesLength() const {
2098 unsigned NumSourceElts = cast<VectorType>(Op<0>()->getType())
2099 ->getElementCount()
2100 .getKnownMinValue();
2101 unsigned NumMaskElts = ShuffleMask.size();
2102 return NumSourceElts != NumMaskElts;
2103 }
2104
2105 /// Return true if this shuffle returns a vector with a greater number of
2106 /// elements than its source vectors.
2107 /// Example: shufflevector <2 x n> A, <2 x n> B, <1,2,3>
2108 bool increasesLength() const {
2109 unsigned NumSourceElts = cast<VectorType>(Op<0>()->getType())
2110 ->getElementCount()
2111 .getKnownMinValue();
2112 unsigned NumMaskElts = ShuffleMask.size();
2113 return NumSourceElts < NumMaskElts;
2114 }
2115
2116 /// Return true if this shuffle mask chooses elements from exactly one source
2117 /// vector.
2118 /// Example: <7,5,undef,7>
2119 /// This assumes that vector operands are the same length as the mask.
2120 static bool isSingleSourceMask(ArrayRef<int> Mask);
2121 static bool isSingleSourceMask(const Constant *Mask) {
2122 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2122, __extension__ __PRETTY_FUNCTION__
))
;
2123 SmallVector<int, 16> MaskAsInts;
2124 getShuffleMask(Mask, MaskAsInts);
2125 return isSingleSourceMask(MaskAsInts);
2126 }
2127
2128 /// Return true if this shuffle chooses elements from exactly one source
2129 /// vector without changing the length of that vector.
2130 /// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
2131 /// TODO: Optionally allow length-changing shuffles.
2132 bool isSingleSource() const {
2133 return !changesLength() && isSingleSourceMask(ShuffleMask);
2134 }
2135
2136 /// Return true if this shuffle mask chooses elements from exactly one source
2137 /// vector without lane crossings. A shuffle using this mask is not
2138 /// necessarily a no-op because it may change the number of elements from its
2139 /// input vectors or it may provide demanded bits knowledge via undef lanes.
2140 /// Example: <undef,undef,2,3>
2141 static bool isIdentityMask(ArrayRef<int> Mask);
2142 static bool isIdentityMask(const Constant *Mask) {
2143 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2143, __extension__ __PRETTY_FUNCTION__
))
;
2144 SmallVector<int, 16> MaskAsInts;
2145 getShuffleMask(Mask, MaskAsInts);
2146 return isIdentityMask(MaskAsInts);
2147 }
2148
2149 /// Return true if this shuffle chooses elements from exactly one source
2150 /// vector without lane crossings and does not change the number of elements
2151 /// from its input vectors.
2152 /// Example: shufflevector <4 x n> A, <4 x n> B, <4,undef,6,undef>
2153 bool isIdentity() const {
2154 return !changesLength() && isIdentityMask(ShuffleMask);
2155 }
2156
2157 /// Return true if this shuffle lengthens exactly one source vector with
2158 /// undefs in the high elements.
2159 bool isIdentityWithPadding() const;
2160
2161 /// Return true if this shuffle extracts the first N elements of exactly one
2162 /// source vector.
2163 bool isIdentityWithExtract() const;
2164
2165 /// Return true if this shuffle concatenates its 2 source vectors. This
2166 /// returns false if either input is undefined. In that case, the shuffle is
2167 /// is better classified as an identity with padding operation.
2168 bool isConcat() const;
2169
2170 /// Return true if this shuffle mask chooses elements from its source vectors
2171 /// without lane crossings. A shuffle using this mask would be
2172 /// equivalent to a vector select with a constant condition operand.
2173 /// Example: <4,1,6,undef>
2174 /// This returns false if the mask does not choose from both input vectors.
2175 /// In that case, the shuffle is better classified as an identity shuffle.
2176 /// This assumes that vector operands are the same length as the mask
2177 /// (a length-changing shuffle can never be equivalent to a vector select).
2178 static bool isSelectMask(ArrayRef<int> Mask);
2179 static bool isSelectMask(const Constant *Mask) {
2180 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2180, __extension__ __PRETTY_FUNCTION__
))
;
2181 SmallVector<int, 16> MaskAsInts;
2182 getShuffleMask(Mask, MaskAsInts);
2183 return isSelectMask(MaskAsInts);
2184 }
2185
2186 /// Return true if this shuffle chooses elements from its source vectors
2187 /// without lane crossings and all operands have the same number of elements.
2188 /// In other words, this shuffle is equivalent to a vector select with a
2189 /// constant condition operand.
2190 /// Example: shufflevector <4 x n> A, <4 x n> B, <undef,1,6,3>
2191 /// This returns false if the mask does not choose from both input vectors.
2192 /// In that case, the shuffle is better classified as an identity shuffle.
2193 /// TODO: Optionally allow length-changing shuffles.
2194 bool isSelect() const {
2195 return !changesLength() && isSelectMask(ShuffleMask);
2196 }
2197
2198 /// Return true if this shuffle mask swaps the order of elements from exactly
2199 /// one source vector.
2200 /// Example: <7,6,undef,4>
2201 /// This assumes that vector operands are the same length as the mask.
2202 static bool isReverseMask(ArrayRef<int> Mask);
2203 static bool isReverseMask(const Constant *Mask) {
2204 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2204, __extension__ __PRETTY_FUNCTION__
))
;
2205 SmallVector<int, 16> MaskAsInts;
2206 getShuffleMask(Mask, MaskAsInts);
2207 return isReverseMask(MaskAsInts);
2208 }
2209
2210 /// Return true if this shuffle swaps the order of elements from exactly
2211 /// one source vector.
2212 /// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
2213 /// TODO: Optionally allow length-changing shuffles.
2214 bool isReverse() const {
2215 return !changesLength() && isReverseMask(ShuffleMask);
2216 }
2217
2218 /// Return true if this shuffle mask chooses all elements with the same value
2219 /// as the first element of exactly one source vector.
2220 /// Example: <4,undef,undef,4>
2221 /// This assumes that vector operands are the same length as the mask.
2222 static bool isZeroEltSplatMask(ArrayRef<int> Mask);
2223 static bool isZeroEltSplatMask(const Constant *Mask) {
2224 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2224, __extension__ __PRETTY_FUNCTION__
))
;
2225 SmallVector<int, 16> MaskAsInts;
2226 getShuffleMask(Mask, MaskAsInts);
2227 return isZeroEltSplatMask(MaskAsInts);
2228 }
2229
2230 /// Return true if all elements of this shuffle are the same value as the
2231 /// first element of exactly one source vector without changing the length
2232 /// of that vector.
2233 /// Example: shufflevector <4 x n> A, <4 x n> B, <undef,0,undef,0>
2234 /// TODO: Optionally allow length-changing shuffles.
2235 /// TODO: Optionally allow splats from other elements.
2236 bool isZeroEltSplat() const {
2237 return !changesLength() && isZeroEltSplatMask(ShuffleMask);
2238 }
2239
2240 /// Return true if this shuffle mask is a transpose mask.
2241 /// Transpose vector masks transpose a 2xn matrix. They read corresponding
2242 /// even- or odd-numbered vector elements from two n-dimensional source
2243 /// vectors and write each result into consecutive elements of an
2244 /// n-dimensional destination vector. Two shuffles are necessary to complete
2245 /// the transpose, one for the even elements and another for the odd elements.
2246 /// This description closely follows how the TRN1 and TRN2 AArch64
2247 /// instructions operate.
2248 ///
2249 /// For example, a simple 2x2 matrix can be transposed with:
2250 ///
2251 /// ; Original matrix
2252 /// m0 = < a, b >
2253 /// m1 = < c, d >
2254 ///
2255 /// ; Transposed matrix
2256 /// t0 = < a, c > = shufflevector m0, m1, < 0, 2 >
2257 /// t1 = < b, d > = shufflevector m0, m1, < 1, 3 >
2258 ///
2259 /// For matrices having greater than n columns, the resulting nx2 transposed
2260 /// matrix is stored in two result vectors such that one vector contains
2261 /// interleaved elements from all the even-numbered rows and the other vector
2262 /// contains interleaved elements from all the odd-numbered rows. For example,
2263 /// a 2x4 matrix can be transposed with:
2264 ///
2265 /// ; Original matrix
2266 /// m0 = < a, b, c, d >
2267 /// m1 = < e, f, g, h >
2268 ///
2269 /// ; Transposed matrix
2270 /// t0 = < a, e, c, g > = shufflevector m0, m1 < 0, 4, 2, 6 >
2271 /// t1 = < b, f, d, h > = shufflevector m0, m1 < 1, 5, 3, 7 >
2272 static bool isTransposeMask(ArrayRef<int> Mask);
2273 static bool isTransposeMask(const Constant *Mask) {
2274 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2274, __extension__ __PRETTY_FUNCTION__
))
;
2275 SmallVector<int, 16> MaskAsInts;
2276 getShuffleMask(Mask, MaskAsInts);
2277 return isTransposeMask(MaskAsInts);
2278 }
2279
2280 /// Return true if this shuffle transposes the elements of its inputs without
2281 /// changing the length of the vectors. This operation may also be known as a
2282 /// merge or interleave. See the description for isTransposeMask() for the
2283 /// exact specification.
2284 /// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
2285 bool isTranspose() const {
2286 return !changesLength() && isTransposeMask(ShuffleMask);
2287 }
2288
2289 /// Return true if this shuffle mask is an extract subvector mask.
2290 /// A valid extract subvector mask returns a smaller vector from a single
2291 /// source operand. The base extraction index is returned as well.
2292 static bool isExtractSubvectorMask(ArrayRef<int> Mask, int NumSrcElts,
2293 int &Index);
2294 static bool isExtractSubvectorMask(const Constant *Mask, int NumSrcElts,
2295 int &Index) {
2296 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2296, __extension__ __PRETTY_FUNCTION__
))
;
2297 // Not possible to express a shuffle mask for a scalable vector for this
2298 // case.
2299 if (isa<ScalableVectorType>(Mask->getType()))
2300 return false;
2301 SmallVector<int, 16> MaskAsInts;
2302 getShuffleMask(Mask, MaskAsInts);
2303 return isExtractSubvectorMask(MaskAsInts, NumSrcElts, Index);
2304 }
2305
2306 /// Return true if this shuffle mask is an extract subvector mask.
2307 bool isExtractSubvectorMask(int &Index) const {
2308 // Not possible to express a shuffle mask for a scalable vector for this
2309 // case.
2310 if (isa<ScalableVectorType>(getType()))
2311 return false;
2312
2313 int NumSrcElts =
2314 cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
2315 return isExtractSubvectorMask(ShuffleMask, NumSrcElts, Index);
2316 }
2317
2318 /// Return true if this shuffle mask is an insert subvector mask.
2319 /// A valid insert subvector mask inserts the lowest elements of a second
2320 /// source operand into an in-place first source operand operand.
2321 /// Both the sub vector width and the insertion index is returned.
2322 static bool isInsertSubvectorMask(ArrayRef<int> Mask, int NumSrcElts,
2323 int &NumSubElts, int &Index);
2324 static bool isInsertSubvectorMask(const Constant *Mask, int NumSrcElts,
2325 int &NumSubElts, int &Index) {
2326 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2326, __extension__ __PRETTY_FUNCTION__
))
;
2327 // Not possible to express a shuffle mask for a scalable vector for this
2328 // case.
2329 if (isa<ScalableVectorType>(Mask->getType()))
2330 return false;
2331 SmallVector<int, 16> MaskAsInts;
2332 getShuffleMask(Mask, MaskAsInts);
2333 return isInsertSubvectorMask(MaskAsInts, NumSrcElts, NumSubElts, Index);
2334 }
2335
2336 /// Return true if this shuffle mask is an insert subvector mask.
2337 bool isInsertSubvectorMask(int &NumSubElts, int &Index) const {
2338 // Not possible to express a shuffle mask for a scalable vector for this
2339 // case.
2340 if (isa<ScalableVectorType>(getType()))
2341 return false;
2342
2343 int NumSrcElts =
2344 cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
2345 return isInsertSubvectorMask(ShuffleMask, NumSrcElts, NumSubElts, Index);
2346 }
2347
2348 /// Return true if this shuffle mask replicates each of the \p VF elements
2349 /// in a vector \p ReplicationFactor times.
2350 /// For example, the mask for \p ReplicationFactor=3 and \p VF=4 is:
2351 /// <0,0,0,1,1,1,2,2,2,3,3,3>
2352 static bool isReplicationMask(ArrayRef<int> Mask, int &ReplicationFactor,
2353 int &VF);
2354 static bool isReplicationMask(const Constant *Mask, int &ReplicationFactor,
2355 int &VF) {
2356 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2356, __extension__ __PRETTY_FUNCTION__
))
;
2357 // Not possible to express a shuffle mask for a scalable vector for this
2358 // case.
2359 if (isa<ScalableVectorType>(Mask->getType()))
2360 return false;
2361 SmallVector<int, 16> MaskAsInts;
2362 getShuffleMask(Mask, MaskAsInts);
2363 return isReplicationMask(MaskAsInts, ReplicationFactor, VF);
2364 }
2365
2366 /// Return true if this shuffle mask is a replication mask.
2367 bool isReplicationMask(int &ReplicationFactor, int &VF) const;
2368
2369 /// Change values in a shuffle permute mask assuming the two vector operands
2370 /// of length InVecNumElts have swapped position.
2371 static void commuteShuffleMask(MutableArrayRef<int> Mask,
2372 unsigned InVecNumElts) {
2373 for (int &Idx : Mask) {
2374 if (Idx == -1)
2375 continue;
2376 Idx = Idx < (int)InVecNumElts ? Idx + InVecNumElts : Idx - InVecNumElts;
2377 assert(Idx >= 0 && Idx < (int)InVecNumElts * 2 &&(static_cast <bool> (Idx >= 0 && Idx < (int
)InVecNumElts * 2 && "shufflevector mask index out of range"
) ? void (0) : __assert_fail ("Idx >= 0 && Idx < (int)InVecNumElts * 2 && \"shufflevector mask index out of range\""
, "llvm/include/llvm/IR/Instructions.h", 2378, __extension__ __PRETTY_FUNCTION__
))
2378 "shufflevector mask index out of range")(static_cast <bool> (Idx >= 0 && Idx < (int
)InVecNumElts * 2 && "shufflevector mask index out of range"
) ? void (0) : __assert_fail ("Idx >= 0 && Idx < (int)InVecNumElts * 2 && \"shufflevector mask index out of range\""
, "llvm/include/llvm/IR/Instructions.h", 2378, __extension__ __PRETTY_FUNCTION__
))
;
2379 }
2380 }
2381
2382 // Methods for support type inquiry through isa, cast, and dyn_cast:
2383 static bool classof(const Instruction *I) {
2384 return I->getOpcode() == Instruction::ShuffleVector;
2385 }
2386 static bool classof(const Value *V) {
2387 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2388 }
2389};
2390
2391template <>
2392struct OperandTraits<ShuffleVectorInst>
2393 : public FixedNumOperandTraits<ShuffleVectorInst, 2> {};
2394
2395DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorInst, Value)ShuffleVectorInst::op_iterator ShuffleVectorInst::op_begin() {
return OperandTraits<ShuffleVectorInst>::op_begin(this
); } ShuffleVectorInst::const_op_iterator ShuffleVectorInst::
op_begin() const { return OperandTraits<ShuffleVectorInst>
::op_begin(const_cast<ShuffleVectorInst*>(this)); } ShuffleVectorInst
::op_iterator ShuffleVectorInst::op_end() { return OperandTraits
<ShuffleVectorInst>::op_end(this); } ShuffleVectorInst::
const_op_iterator ShuffleVectorInst::op_end() const { return OperandTraits
<ShuffleVectorInst>::op_end(const_cast<ShuffleVectorInst
*>(this)); } Value *ShuffleVectorInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<ShuffleVectorInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ShuffleVectorInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 2395, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<ShuffleVectorInst
>::op_begin(const_cast<ShuffleVectorInst*>(this))[i_nocapture
].get()); } void ShuffleVectorInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ShuffleVectorInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ShuffleVectorInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 2395, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<ShuffleVectorInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned ShuffleVectorInst::getNumOperands
() const { return OperandTraits<ShuffleVectorInst>::operands
(this); } template <int Idx_nocapture> Use &ShuffleVectorInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &ShuffleVectorInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
2396
2397//===----------------------------------------------------------------------===//
2398// ExtractValueInst Class
2399//===----------------------------------------------------------------------===//
2400
2401/// This instruction extracts a struct member or array
2402/// element value from an aggregate value.
2403///
2404class ExtractValueInst : public UnaryInstruction {
2405 SmallVector<unsigned, 4> Indices;
2406
2407 ExtractValueInst(const ExtractValueInst &EVI);
2408
2409 /// Constructors - Create a extractvalue instruction with a base aggregate
2410 /// value and a list of indices. The first ctor can optionally insert before
2411 /// an existing instruction, the second appends the new instruction to the
2412 /// specified BasicBlock.
2413 inline ExtractValueInst(Value *Agg,
2414 ArrayRef<unsigned> Idxs,
2415 const Twine &NameStr,
2416 Instruction *InsertBefore);
2417 inline ExtractValueInst(Value *Agg,
2418 ArrayRef<unsigned> Idxs,
2419 const Twine &NameStr, BasicBlock *InsertAtEnd);
2420
2421 void init(ArrayRef<unsigned> Idxs, const Twine &NameStr);
2422
2423protected:
2424 // Note: Instruction needs to be a friend here to call cloneImpl.
2425 friend class Instruction;
2426
2427 ExtractValueInst *cloneImpl() const;
2428
2429public:
2430 static ExtractValueInst *Create(Value *Agg,
2431 ArrayRef<unsigned> Idxs,
2432 const Twine &NameStr = "",
2433 Instruction *InsertBefore = nullptr) {
2434 return new
2435 ExtractValueInst(Agg, Idxs, NameStr, InsertBefore);
2436 }
2437
2438 static ExtractValueInst *Create(Value *Agg,
2439 ArrayRef<unsigned> Idxs,
2440 const Twine &NameStr,
2441 BasicBlock *InsertAtEnd) {
2442 return new ExtractValueInst(Agg, Idxs, NameStr, InsertAtEnd);
2443 }
2444
2445 /// Returns the type of the element that would be extracted
2446 /// with an extractvalue instruction with the specified parameters.
2447 ///
2448 /// Null is returned if the indices are invalid for the specified type.
2449 static Type *getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs);
2450
2451 using idx_iterator = const unsigned*;
2452
2453 inline idx_iterator idx_begin() const { return Indices.begin(); }
2454 inline idx_iterator idx_end() const { return Indices.end(); }
2455 inline iterator_range<idx_iterator> indices() const {
2456 return make_range(idx_begin(), idx_end());
2457 }
2458
2459 Value *getAggregateOperand() {
2460 return getOperand(0);
2461 }
2462 const Value *getAggregateOperand() const {
2463 return getOperand(0);
2464 }
2465 static unsigned getAggregateOperandIndex() {
2466 return 0U; // get index for modifying correct operand
2467 }
2468
2469 ArrayRef<unsigned> getIndices() const {
2470 return Indices;
2471 }
2472
2473 unsigned getNumIndices() const {
2474 return (unsigned)Indices.size();
2475 }
2476
2477 bool hasIndices() const {
2478 return true;
2479 }
2480
2481 // Methods for support type inquiry through isa, cast, and dyn_cast:
2482 static bool classof(const Instruction *I) {
2483 return I->getOpcode() == Instruction::ExtractValue;
2484 }
2485 static bool classof(const Value *V) {
2486 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2487 }
2488};
2489
2490ExtractValueInst::ExtractValueInst(Value *Agg,
2491 ArrayRef<unsigned> Idxs,
2492 const Twine &NameStr,
2493 Instruction *InsertBefore)
2494 : UnaryInstruction(checkGEPType(getIndexedType(Agg->getType(), Idxs)),
2495 ExtractValue, Agg, InsertBefore) {
2496 init(Idxs, NameStr);
2497}
2498
2499ExtractValueInst::ExtractValueInst(Value *Agg,
2500 ArrayRef<unsigned> Idxs,
2501 const Twine &NameStr,
2502 BasicBlock *InsertAtEnd)
2503 : UnaryInstruction(checkGEPType(getIndexedType(Agg->getType(), Idxs)),
2504 ExtractValue, Agg, InsertAtEnd) {
2505 init(Idxs, NameStr);
2506}
2507
2508//===----------------------------------------------------------------------===//
2509// InsertValueInst Class
2510//===----------------------------------------------------------------------===//
2511
2512/// This instruction inserts a struct field of array element
2513/// value into an aggregate value.
2514///
2515class InsertValueInst : public Instruction {
2516 SmallVector<unsigned, 4> Indices;
2517
2518 InsertValueInst(const InsertValueInst &IVI);
2519
2520 /// Constructors - Create a insertvalue instruction with a base aggregate
2521 /// value, a value to insert, and a list of indices. The first ctor can
2522 /// optionally insert before an existing instruction, the second appends
2523 /// the new instruction to the specified BasicBlock.
2524 inline InsertValueInst(Value *Agg, Value *Val,
2525 ArrayRef<unsigned> Idxs,
2526 const Twine &NameStr,
2527 Instruction *InsertBefore);
2528 inline InsertValueInst(Value *Agg, Value *Val,
2529 ArrayRef<unsigned> Idxs,
2530 const Twine &NameStr, BasicBlock *InsertAtEnd);
2531
2532 /// Constructors - These two constructors are convenience methods because one
2533 /// and two index insertvalue instructions are so common.
2534 InsertValueInst(Value *Agg, Value *Val, unsigned Idx,
2535 const Twine &NameStr = "",
2536 Instruction *InsertBefore = nullptr);
2537 InsertValueInst(Value *Agg, Value *Val, unsigned Idx, const Twine &NameStr,
2538 BasicBlock *InsertAtEnd);
2539
2540 void init(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
2541 const Twine &NameStr);
2542
2543protected:
2544 // Note: Instruction needs to be a friend here to call cloneImpl.
2545 friend class Instruction;
2546
2547 InsertValueInst *cloneImpl() const;
2548
2549public:
2550 // allocate space for exactly two operands
2551 void *operator new(size_t S) { return User::operator new(S, 2); }
2552 void operator delete(void *Ptr) { User::operator delete(Ptr); }
2553
2554 static InsertValueInst *Create(Value *Agg, Value *Val,
2555 ArrayRef<unsigned> Idxs,
2556 const Twine &NameStr = "",
2557 Instruction *InsertBefore = nullptr) {
2558 return new InsertValueInst(Agg, Val, Idxs, NameStr, InsertBefore);
2559 }
2560
2561 static InsertValueInst *Create(Value *Agg, Value *Val,
2562 ArrayRef<unsigned> Idxs,
2563 const Twine &NameStr,
2564 BasicBlock *InsertAtEnd) {
2565 return new InsertValueInst(Agg, Val, Idxs, NameStr, InsertAtEnd);
2566 }
2567
2568 /// Transparently provide more efficient getOperand methods.
2569 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2570
2571 using idx_iterator = const unsigned*;
2572
2573 inline idx_iterator idx_begin() const { return Indices.begin(); }
2574 inline idx_iterator idx_end() const { return Indices.end(); }
2575 inline iterator_range<idx_iterator> indices() const {
2576 return make_range(idx_begin(), idx_end());
2577 }
2578
2579 Value *getAggregateOperand() {
2580 return getOperand(0);
2581 }
2582 const Value *getAggregateOperand() const {
2583 return getOperand(0);
2584 }
2585 static unsigned getAggregateOperandIndex() {
2586 return 0U; // get index for modifying correct operand
2587 }
2588
2589 Value *getInsertedValueOperand() {
2590 return getOperand(1);
2591 }
2592 const Value *getInsertedValueOperand() const {
2593 return getOperand(1);
2594 }
2595 static unsigned getInsertedValueOperandIndex() {
2596 return 1U; // get index for modifying correct operand
2597 }
2598
2599 ArrayRef<unsigned> getIndices() const {
2600 return Indices;
2601 }
2602
2603 unsigned getNumIndices() const {
2604 return (unsigned)Indices.size();
2605 }
2606
2607 bool hasIndices() const {
2608 return true;
2609 }
2610
2611 // Methods for support type inquiry through isa, cast, and dyn_cast:
2612 static bool classof(const Instruction *I) {
2613 return I->getOpcode() == Instruction::InsertValue;
2614 }
2615 static bool classof(const Value *V) {
2616 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2617 }
2618};
2619
2620template <>
2621struct OperandTraits<InsertValueInst> :
2622 public FixedNumOperandTraits<InsertValueInst, 2> {
2623};
2624
2625InsertValueInst::InsertValueInst(Value *Agg,
2626 Value *Val,
2627 ArrayRef<unsigned> Idxs,
2628 const Twine &NameStr,
2629 Instruction *InsertBefore)
2630 : Instruction(Agg->getType(), InsertValue,
2631 OperandTraits<InsertValueInst>::op_begin(this),
2632 2, InsertBefore) {
2633 init(Agg, Val, Idxs, NameStr);
2634}
2635
2636InsertValueInst::InsertValueInst(Value *Agg,
2637 Value *Val,
2638 ArrayRef<unsigned> Idxs,
2639 const Twine &NameStr,
2640 BasicBlock *InsertAtEnd)
2641 : Instruction(Agg->getType(), InsertValue,
2642 OperandTraits<InsertValueInst>::op_begin(this),
2643 2, InsertAtEnd) {
2644 init(Agg, Val, Idxs, NameStr);
2645}
2646
2647DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertValueInst, Value)InsertValueInst::op_iterator InsertValueInst::op_begin() { return
OperandTraits<InsertValueInst>::op_begin(this); } InsertValueInst
::const_op_iterator InsertValueInst::op_begin() const { return
OperandTraits<InsertValueInst>::op_begin(const_cast<
InsertValueInst*>(this)); } InsertValueInst::op_iterator InsertValueInst
::op_end() { return OperandTraits<InsertValueInst>::op_end
(this); } InsertValueInst::const_op_iterator InsertValueInst::
op_end() const { return OperandTraits<InsertValueInst>::
op_end(const_cast<InsertValueInst*>(this)); } Value *InsertValueInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<InsertValueInst>::
operands(this) && "getOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<InsertValueInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 2647, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<InsertValueInst
>::op_begin(const_cast<InsertValueInst*>(this))[i_nocapture
].get()); } void InsertValueInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<InsertValueInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertValueInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 2647, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<InsertValueInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned InsertValueInst::getNumOperands
() const { return OperandTraits<InsertValueInst>::operands
(this); } template <int Idx_nocapture> Use &InsertValueInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &InsertValueInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
2648
2649//===----------------------------------------------------------------------===//
2650// PHINode Class
2651//===----------------------------------------------------------------------===//
2652
2653// PHINode - The PHINode class is used to represent the magical mystical PHI
2654// node, that can not exist in nature, but can be synthesized in a computer
2655// scientist's overactive imagination.
2656//
2657class PHINode : public Instruction {
2658 /// The number of operands actually allocated. NumOperands is
2659 /// the number actually in use.
2660 unsigned ReservedSpace;
2661
2662 PHINode(const PHINode &PN);
2663
2664 explicit PHINode(Type *Ty, unsigned NumReservedValues,
2665 const Twine &NameStr = "",
2666 Instruction *InsertBefore = nullptr)
2667 : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertBefore),
2668 ReservedSpace(NumReservedValues) {
2669 assert(!Ty->isTokenTy() && "PHI nodes cannot have token type!")(static_cast <bool> (!Ty->isTokenTy() && "PHI nodes cannot have token type!"
) ? void (0) : __assert_fail ("!Ty->isTokenTy() && \"PHI nodes cannot have token type!\""
, "llvm/include/llvm/IR/Instructions.h", 2669, __extension__ __PRETTY_FUNCTION__
))
;
2670 setName(NameStr);
2671 allocHungoffUses(ReservedSpace);
2672 }
2673
2674 PHINode(Type *Ty, unsigned NumReservedValues, const Twine &NameStr,
2675 BasicBlock *InsertAtEnd)
2676 : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertAtEnd),
2677 ReservedSpace(NumReservedValues) {
2678 assert(!Ty->isTokenTy() && "PHI nodes cannot have token type!")(static_cast <bool> (!Ty->isTokenTy() && "PHI nodes cannot have token type!"
) ? void (0) : __assert_fail ("!Ty->isTokenTy() && \"PHI nodes cannot have token type!\""
, "llvm/include/llvm/IR/Instructions.h", 2678, __extension__ __PRETTY_FUNCTION__
))
;
2679 setName(NameStr);
2680 allocHungoffUses(ReservedSpace);
2681 }
2682
2683protected:
2684 // Note: Instruction needs to be a friend here to call cloneImpl.
2685 friend class Instruction;
2686
2687 PHINode *cloneImpl() const;
2688
2689 // allocHungoffUses - this is more complicated than the generic
2690 // User::allocHungoffUses, because we have to allocate Uses for the incoming
2691 // values and pointers to the incoming blocks, all in one allocation.
2692 void allocHungoffUses(unsigned N) {
2693 User::allocHungoffUses(N, /* IsPhi */ true);
2694 }
2695
2696public:
2697 /// Constructors - NumReservedValues is a hint for the number of incoming
2698 /// edges that this phi node will have (use 0 if you really have no idea).
2699 static PHINode *Create(Type *Ty, unsigned NumReservedValues,
2700 const Twine &NameStr = "",
2701 Instruction *InsertBefore = nullptr) {
2702 return new PHINode(Ty, NumReservedValues, NameStr, InsertBefore);
2703 }
2704
2705 static PHINode *Create(Type *Ty, unsigned NumReservedValues,
2706 const Twine &NameStr, BasicBlock *InsertAtEnd) {
2707 return new PHINode(Ty, NumReservedValues, NameStr, InsertAtEnd);
2708 }
2709
2710 /// Provide fast operand accessors
2711 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2712
2713 // Block iterator interface. This provides access to the list of incoming
2714 // basic blocks, which parallels the list of incoming values.
2715
2716 using block_iterator = BasicBlock **;
2717 using const_block_iterator = BasicBlock * const *;
2718
2719 block_iterator block_begin() {
2720 return reinterpret_cast<block_iterator>(op_begin() + ReservedSpace);
2721 }
2722
2723 const_block_iterator block_begin() const {
2724 return reinterpret_cast<const_block_iterator>(op_begin() + ReservedSpace);
2725 }
2726
2727 block_iterator block_end() {
2728 return block_begin() + getNumOperands();
2729 }
2730
2731 const_block_iterator block_end() const {
2732 return block_begin() + getNumOperands();
2733 }
2734
2735 iterator_range<block_iterator> blocks() {
2736 return make_range(block_begin(), block_end());
2737 }
2738
2739 iterator_range<const_block_iterator> blocks() const {
2740 return make_range(block_begin(), block_end());
2741 }
2742
2743 op_range incoming_values() { return operands(); }
2744
2745 const_op_range incoming_values() const { return operands(); }
2746
2747 /// Return the number of incoming edges
2748 ///
2749 unsigned getNumIncomingValues() const { return getNumOperands(); }
2750
2751 /// Return incoming value number x
2752 ///
2753 Value *getIncomingValue(unsigned i) const {
2754 return getOperand(i);
2755 }
2756 void setIncomingValue(unsigned i, Value *V) {
2757 assert(V && "PHI node got a null value!")(static_cast <bool> (V && "PHI node got a null value!"
) ? void (0) : __assert_fail ("V && \"PHI node got a null value!\""
, "llvm/include/llvm/IR/Instructions.h", 2757, __extension__ __PRETTY_FUNCTION__
))
;
2758 assert(getType() == V->getType() &&(static_cast <bool> (getType() == V->getType() &&
"All operands to PHI node must be the same type as the PHI node!"
) ? void (0) : __assert_fail ("getType() == V->getType() && \"All operands to PHI node must be the same type as the PHI node!\""
, "llvm/include/llvm/IR/Instructions.h", 2759, __extension__ __PRETTY_FUNCTION__
))
2759 "All operands to PHI node must be the same type as the PHI node!")(static_cast <bool> (getType() == V->getType() &&
"All operands to PHI node must be the same type as the PHI node!"
) ? void (0) : __assert_fail ("getType() == V->getType() && \"All operands to PHI node must be the same type as the PHI node!\""
, "llvm/include/llvm/IR/Instructions.h", 2759, __extension__ __PRETTY_FUNCTION__
))
;
2760 setOperand(i, V);
2761 }
2762
2763 static unsigned getOperandNumForIncomingValue(unsigned i) {
2764 return i;
2765 }
2766
2767 static unsigned getIncomingValueNumForOperand(unsigned i) {
2768 return i;
2769 }
2770
2771 /// Return incoming basic block number @p i.
2772 ///
2773 BasicBlock *getIncomingBlock(unsigned i) const {
2774 return block_begin()[i];
2775 }
2776
2777 /// Return incoming basic block corresponding
2778 /// to an operand of the PHI.
2779 ///
2780 BasicBlock *getIncomingBlock(const Use &U) const {
2781 assert(this == U.getUser() && "Iterator doesn't point to PHI's Uses?")(static_cast <bool> (this == U.getUser() && "Iterator doesn't point to PHI's Uses?"
) ? void (0) : __assert_fail ("this == U.getUser() && \"Iterator doesn't point to PHI's Uses?\""
, "llvm/include/llvm/IR/Instructions.h", 2781, __extension__ __PRETTY_FUNCTION__
))
;
2782 return getIncomingBlock(unsigned(&U - op_begin()));
2783 }
2784
2785 /// Return incoming basic block corresponding
2786 /// to value use iterator.
2787 ///
2788 BasicBlock *getIncomingBlock(Value::const_user_iterator I) const {
2789 return getIncomingBlock(I.getUse());
2790 }
2791
2792 void setIncomingBlock(unsigned i, BasicBlock *BB) {
2793 assert(BB && "PHI node got a null basic block!")(static_cast <bool> (BB && "PHI node got a null basic block!"
) ? void (0) : __assert_fail ("BB && \"PHI node got a null basic block!\""
, "llvm/include/llvm/IR/Instructions.h", 2793, __extension__ __PRETTY_FUNCTION__
))
;
2794 block_begin()[i] = BB;
2795 }
2796
2797 /// Replace every incoming basic block \p Old to basic block \p New.
2798 void replaceIncomingBlockWith(const BasicBlock *Old, BasicBlock *New) {
2799 assert(New && Old && "PHI node got a null basic block!")(static_cast <bool> (New && Old && "PHI node got a null basic block!"
) ? void (0) : __assert_fail ("New && Old && \"PHI node got a null basic block!\""
, "llvm/include/llvm/IR/Instructions.h", 2799, __extension__ __PRETTY_FUNCTION__
))
;
2800 for (unsigned Op = 0, NumOps = getNumOperands(); Op != NumOps; ++Op)
2801 if (getIncomingBlock(Op) == Old)
2802 setIncomingBlock(Op, New);
2803 }
2804
2805 /// Add an incoming value to the end of the PHI list
2806 ///
2807 void addIncoming(Value *V, BasicBlock *BB) {
2808 if (getNumOperands() == ReservedSpace)
2809 growOperands(); // Get more space!
2810 // Initialize some new operands.
2811 setNumHungOffUseOperands(getNumOperands() + 1);
2812 setIncomingValue(getNumOperands() - 1, V);
2813 setIncomingBlock(getNumOperands() - 1, BB);
2814 }
2815
2816 /// Remove an incoming value. This is useful if a
2817 /// predecessor basic block is deleted. The value removed is returned.
2818 ///
2819 /// If the last incoming value for a PHI node is removed (and DeletePHIIfEmpty
2820 /// is true), the PHI node is destroyed and any uses of it are replaced with
2821 /// dummy values. The only time there should be zero incoming values to a PHI
2822 /// node is when the block is dead, so this strategy is sound.
2823 ///
2824 Value *removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty = true);
2825
2826 Value *removeIncomingValue(const BasicBlock *BB, bool DeletePHIIfEmpty=true) {
2827 int Idx = getBasicBlockIndex(BB);
2828 assert(Idx >= 0 && "Invalid basic block argument to remove!")(static_cast <bool> (Idx >= 0 && "Invalid basic block argument to remove!"
) ? void (0) : __assert_fail ("Idx >= 0 && \"Invalid basic block argument to remove!\""
, "llvm/include/llvm/IR/Instructions.h", 2828, __extension__ __PRETTY_FUNCTION__
))
;
2829 return removeIncomingValue(Idx, DeletePHIIfEmpty);
2830 }
2831
2832 /// Return the first index of the specified basic
2833 /// block in the value list for this PHI. Returns -1 if no instance.
2834 ///
2835 int getBasicBlockIndex(const BasicBlock *BB) const {
2836 for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
2837 if (block_begin()[i] == BB)
2838 return i;
2839 return -1;
2840 }
2841
2842 Value *getIncomingValueForBlock(const BasicBlock *BB) const {
2843 int Idx = getBasicBlockIndex(BB);
2844 assert(Idx >= 0 && "Invalid basic block argument!")(static_cast <bool> (Idx >= 0 && "Invalid basic block argument!"
) ? void (0) : __assert_fail ("Idx >= 0 && \"Invalid basic block argument!\""
, "llvm/include/llvm/IR/Instructions.h", 2844, __extension__ __PRETTY_FUNCTION__
))
;
2845 return getIncomingValue(Idx);
2846 }
2847
2848 /// Set every incoming value(s) for block \p BB to \p V.
2849 void setIncomingValueForBlock(const BasicBlock *BB, Value *V) {
2850 assert(BB && "PHI node got a null basic block!")(static_cast <bool> (BB && "PHI node got a null basic block!"
) ? void (0) : __assert_fail ("BB && \"PHI node got a null basic block!\""
, "llvm/include/llvm/IR/Instructions.h", 2850, __extension__ __PRETTY_FUNCTION__
))
;
2851 bool Found = false;
2852 for (unsigned Op = 0, NumOps = getNumOperands(); Op != NumOps; ++Op)
2853 if (getIncomingBlock(Op) == BB) {
2854 Found = true;
2855 setIncomingValue(Op, V);
2856 }
2857 (void)Found;
2858 assert(Found && "Invalid basic block argument to set!")(static_cast <bool> (Found && "Invalid basic block argument to set!"
) ? void (0) : __assert_fail ("Found && \"Invalid basic block argument to set!\""
, "llvm/include/llvm/IR/Instructions.h", 2858, __extension__ __PRETTY_FUNCTION__
))
;
2859 }
2860
2861 /// If the specified PHI node always merges together the
2862 /// same value, return the value, otherwise return null.
2863 Value *hasConstantValue() const;
2864
2865 /// Whether the specified PHI node always merges
2866 /// together the same value, assuming undefs are equal to a unique
2867 /// non-undef value.
2868 bool hasConstantOrUndefValue() const;
2869
2870 /// If the PHI node is complete which means all of its parent's predecessors
2871 /// have incoming value in this PHI, return true, otherwise return false.
2872 bool isComplete() const {
2873 return llvm::all_of(predecessors(getParent()),
2874 [this](const BasicBlock *Pred) {
2875 return getBasicBlockIndex(Pred) >= 0;
2876 });
2877 }
2878
2879 /// Methods for support type inquiry through isa, cast, and dyn_cast:
2880 static bool classof(const Instruction *I) {
2881 return I->getOpcode() == Instruction::PHI;
2882 }
2883 static bool classof(const Value *V) {
2884 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2885 }
2886
2887private:
2888 void growOperands();
2889};
2890
2891template <>
2892struct OperandTraits<PHINode> : public HungoffOperandTraits<2> {
2893};
2894
2895DEFINE_TRANSPARENT_OPERAND_ACCESSORS(PHINode, Value)PHINode::op_iterator PHINode::op_begin() { return OperandTraits
<PHINode>::op_begin(this); } PHINode::const_op_iterator
PHINode::op_begin() const { return OperandTraits<PHINode>
::op_begin(const_cast<PHINode*>(this)); } PHINode::op_iterator
PHINode::op_end() { return OperandTraits<PHINode>::op_end
(this); } PHINode::const_op_iterator PHINode::op_end() const {
return OperandTraits<PHINode>::op_end(const_cast<PHINode
*>(this)); } Value *PHINode::getOperand(unsigned i_nocapture
) const { (static_cast <bool> (i_nocapture < OperandTraits
<PHINode>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<PHINode>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 2895, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<PHINode
>::op_begin(const_cast<PHINode*>(this))[i_nocapture]
.get()); } void PHINode::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { (static_cast <bool> (i_nocapture <
OperandTraits<PHINode>::operands(this) && "setOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<PHINode>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 2895, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<PHINode>::op_begin(this)[i_nocapture]
= Val_nocapture; } unsigned PHINode::getNumOperands() const {
return OperandTraits<PHINode>::operands(this); } template
<int Idx_nocapture> Use &PHINode::Op() { return this
->OpFrom<Idx_nocapture>(this); } template <int Idx_nocapture
> const Use &PHINode::Op() const { return this->OpFrom
<Idx_nocapture>(this); }
2896
2897//===----------------------------------------------------------------------===//
2898// LandingPadInst Class
2899//===----------------------------------------------------------------------===//
2900
2901//===---------------------------------------------------------------------------
2902/// The landingpad instruction holds all of the information
2903/// necessary to generate correct exception handling. The landingpad instruction
2904/// cannot be moved from the top of a landing pad block, which itself is
2905/// accessible only from the 'unwind' edge of an invoke. This uses the
2906/// SubclassData field in Value to store whether or not the landingpad is a
2907/// cleanup.
2908///
2909class LandingPadInst : public Instruction {
2910 using CleanupField = BoolBitfieldElementT<0>;
2911
2912 /// The number of operands actually allocated. NumOperands is
2913 /// the number actually in use.
2914 unsigned ReservedSpace;
2915
2916 LandingPadInst(const LandingPadInst &LP);
2917
2918public:
2919 enum ClauseType { Catch, Filter };
2920
2921private:
2922 explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
2923 const Twine &NameStr, Instruction *InsertBefore);
2924 explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
2925 const Twine &NameStr, BasicBlock *InsertAtEnd);
2926
2927 // Allocate space for exactly zero operands.
2928 void *operator new(size_t S) { return User::operator new(S); }
2929
2930 void growOperands(unsigned Size);
2931 void init(unsigned NumReservedValues, const Twine &NameStr);
2932
2933protected:
2934 // Note: Instruction needs to be a friend here to call cloneImpl.
2935 friend class Instruction;
2936
2937 LandingPadInst *cloneImpl() const;
2938
2939public:
2940 void operator delete(void *Ptr) { User::operator delete(Ptr); }
2941
2942 /// Constructors - NumReservedClauses is a hint for the number of incoming
2943 /// clauses that this landingpad will have (use 0 if you really have no idea).
2944 static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
2945 const Twine &NameStr = "",
2946 Instruction *InsertBefore = nullptr);
2947 static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
2948 const Twine &NameStr, BasicBlock *InsertAtEnd);
2949
2950 /// Provide fast operand accessors
2951 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2952
2953 /// Return 'true' if this landingpad instruction is a
2954 /// cleanup. I.e., it should be run when unwinding even if its landing pad
2955 /// doesn't catch the exception.
2956 bool isCleanup() const { return getSubclassData<CleanupField>(); }
2957
2958 /// Indicate that this landingpad instruction is a cleanup.
2959 void setCleanup(bool V) { setSubclassData<CleanupField>(V); }
2960
2961 /// Add a catch or filter clause to the landing pad.
2962 void addClause(Constant *ClauseVal);
2963
2964 /// Get the value of the clause at index Idx. Use isCatch/isFilter to
2965 /// determine what type of clause this is.
2966 Constant *getClause(unsigned Idx) const {
2967 return cast<Constant>(getOperandList()[Idx]);
2968 }
2969
2970 /// Return 'true' if the clause and index Idx is a catch clause.
2971 bool isCatch(unsigned Idx) const {
2972 return !isa<ArrayType>(getOperandList()[Idx]->getType());
2973 }
2974
2975 /// Return 'true' if the clause and index Idx is a filter clause.
2976 bool isFilter(unsigned Idx) const {
2977 return isa<ArrayType>(getOperandList()[Idx]->getType());
2978 }
2979
2980 /// Get the number of clauses for this landing pad.
2981 unsigned getNumClauses() const { return getNumOperands(); }
2982
2983 /// Grow the size of the operand list to accommodate the new
2984 /// number of clauses.
2985 void reserveClauses(unsigned Size) { growOperands(Size); }
2986
2987 // Methods for support type inquiry through isa, cast, and dyn_cast:
2988 static bool classof(const Instruction *I) {
2989 return I->getOpcode() == Instruction::LandingPad;
2990 }
2991 static bool classof(const Value *V) {
2992 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2993 }
2994};
2995
2996template <>
2997struct OperandTraits<LandingPadInst> : public HungoffOperandTraits<1> {
2998};
2999
3000DEFINE_TRANSPARENT_OPERAND_ACCESSORS(LandingPadInst, Value)LandingPadInst::op_iterator LandingPadInst::op_begin() { return
OperandTraits<LandingPadInst>::op_begin(this); } LandingPadInst
::const_op_iterator LandingPadInst::op_begin() const { return
OperandTraits<LandingPadInst>::op_begin(const_cast<
LandingPadInst*>(this)); } LandingPadInst::op_iterator LandingPadInst
::op_end() { return OperandTraits<LandingPadInst>::op_end
(this); } LandingPadInst::const_op_iterator LandingPadInst::op_end
() const { return OperandTraits<LandingPadInst>::op_end
(const_cast<LandingPadInst*>(this)); } Value *LandingPadInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<LandingPadInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<LandingPadInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 3000, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<LandingPadInst
>::op_begin(const_cast<LandingPadInst*>(this))[i_nocapture
].get()); } void LandingPadInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<LandingPadInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<LandingPadInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 3000, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<LandingPadInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned LandingPadInst::getNumOperands(
) const { return OperandTraits<LandingPadInst>::operands
(this); } template <int Idx_nocapture> Use &LandingPadInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &LandingPadInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
3001
3002//===----------------------------------------------------------------------===//
3003// ReturnInst Class
3004//===----------------------------------------------------------------------===//
3005
3006//===---------------------------------------------------------------------------
3007/// Return a value (possibly void), from a function. Execution
3008/// does not continue in this function any longer.
3009///
3010class ReturnInst : public Instruction {
3011 ReturnInst(const ReturnInst &RI);
3012
3013private:
3014 // ReturnInst constructors:
3015 // ReturnInst() - 'ret void' instruction
3016 // ReturnInst( null) - 'ret void' instruction
3017 // ReturnInst(Value* X) - 'ret X' instruction
3018 // ReturnInst( null, Inst *I) - 'ret void' instruction, insert before I
3019 // ReturnInst(Value* X, Inst *I) - 'ret X' instruction, insert before I
3020 // ReturnInst( null, BB *B) - 'ret void' instruction, insert @ end of B
3021 // ReturnInst(Value* X, BB *B) - 'ret X' instruction, insert @ end of B
3022 //
3023 // NOTE: If the Value* passed is of type void then the constructor behaves as
3024 // if it was passed NULL.
3025 explicit ReturnInst(LLVMContext &C, Value *retVal = nullptr,
3026 Instruction *InsertBefore = nullptr);
3027 ReturnInst(LLVMContext &C, Value *retVal, BasicBlock *InsertAtEnd);
3028 explicit ReturnInst(LLVMContext &C, BasicBlock *InsertAtEnd);
3029
3030protected:
3031 // Note: Instruction needs to be a friend here to call cloneImpl.
3032 friend class Instruction;
3033
3034 ReturnInst *cloneImpl() const;
3035
3036public:
3037 static ReturnInst* Create(LLVMContext &C, Value *retVal = nullptr,
3038 Instruction *InsertBefore = nullptr) {
3039 return new(!!retVal) ReturnInst(C, retVal, InsertBefore);
3040 }
3041
3042 static ReturnInst* Create(LLVMContext &C, Value *retVal,
3043 BasicBlock *InsertAtEnd) {
3044 return new(!!retVal) ReturnInst(C, retVal, InsertAtEnd);
3045 }
3046
3047 static ReturnInst* Create(LLVMContext &C, BasicBlock *InsertAtEnd) {
3048 return new(0) ReturnInst(C, InsertAtEnd);
3049 }
3050
3051 /// Provide fast operand accessors
3052 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
3053
3054 /// Convenience accessor. Returns null if there is no return value.
3055 Value *getReturnValue() const {
3056 return getNumOperands() != 0 ? getOperand(0) : nullptr;
3057 }
3058
3059 unsigned getNumSuccessors() const { return 0; }
3060
3061 // Methods for support type inquiry through isa, cast, and dyn_cast:
3062 static bool classof(const Instruction *I) {
3063 return (I->getOpcode() == Instruction::Ret);
3064 }
3065 static bool classof(const Value *V) {
3066 return isa<Instruction>(V) && classof(cast<Instruction>(V));
3067 }
3068
3069private:
3070 BasicBlock *getSuccessor(unsigned idx) const {
3071 llvm_unreachable("ReturnInst has no successors!")::llvm::llvm_unreachable_internal("ReturnInst has no successors!"
, "llvm/include/llvm/IR/Instructions.h", 3071)
;
3072 }
3073
3074 void setSuccessor(unsigned idx, BasicBlock *B) {
3075 llvm_unreachable("ReturnInst has no successors!")::llvm::llvm_unreachable_internal("ReturnInst has no successors!"
, "llvm/include/llvm/IR/Instructions.h", 3075)
;
3076 }
3077};
3078
3079template <>
3080struct OperandTraits<ReturnInst> : public VariadicOperandTraits<ReturnInst> {
3081};
3082
3083DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ReturnInst, Value)ReturnInst::op_iterator ReturnInst::op_begin() { return OperandTraits
<ReturnInst>::op_begin(this); } ReturnInst::const_op_iterator
ReturnInst::op_begin() const { return OperandTraits<ReturnInst
>::op_begin(const_cast<ReturnInst*>(this)); } ReturnInst
::op_iterator ReturnInst::op_end() { return OperandTraits<
ReturnInst>::op_end(this); } ReturnInst::const_op_iterator
ReturnInst::op_end() const { return OperandTraits<ReturnInst
>::op_end(const_cast<ReturnInst*>(this)); } Value *ReturnInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<ReturnInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<ReturnInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 3083, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<ReturnInst
>::op_begin(const_cast<ReturnInst*>(this))[i_nocapture
].get()); } void ReturnInst::setOperand(unsigned i_nocapture,
Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ReturnInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ReturnInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 3083, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<ReturnInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned ReturnInst::getNumOperands() const
{ return OperandTraits<ReturnInst>::operands(this); } template
<int Idx_nocapture> Use &ReturnInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &ReturnInst::Op() const { return
this->OpFrom<Idx_nocapture>(this); }
3084
3085//===----------------------------------------------------------------------===//
3086// BranchInst Class
3087//===----------------------------------------------------------------------===//
3088
3089//===---------------------------------------------------------------------------
3090/// Conditional or Unconditional Branch instruction.
3091///
3092class BranchInst : public Instruction {
3093 /// Ops list - Branches are strange. The operands are ordered:
3094 /// [Cond, FalseDest,] TrueDest. This makes some accessors faster because
3095 /// they don't have to check for cond/uncond branchness. These are mostly
3096 /// accessed relative from op_end().
3097 BranchInst(const BranchInst &BI);
3098 // BranchInst constructors (where {B, T, F} are blocks, and C is a condition):
3099 // BranchInst(BB *B) - 'br B'
3100 // BranchInst(BB* T, BB *F, Value *C) - 'br C, T, F'
3101 // BranchInst(BB* B, Inst *I) - 'br B' insert before I
3102 // BranchInst(BB* T, BB *F, Value *C, Inst *I) - 'br C, T, F', insert before I
3103 // BranchInst(BB* B, BB *I) - 'br B' insert at end
3104 // BranchInst(BB* T, BB *F, Value *C, BB *I) - 'br C, T, F', insert at end
3105 explicit BranchInst(BasicBlock *IfTrue, Instruction *InsertBefore = nullptr);
3106 BranchInst(BasicBlock *IfTrue, BasicBlock *IfFalse, Value *Cond,
3107 Instruction *InsertBefore = nullptr);
3108 BranchInst(BasicBlock *IfTrue, BasicBlock *InsertAtEnd);
3109 BranchInst(BasicBlock *IfTrue, BasicBlock *IfFalse, Value *Cond,
3110 BasicBlock *InsertAtEnd);
3111
3112 void AssertOK();
3113
3114protected:
3115 // Note: Instruction needs to be a friend here to call cloneImpl.
3116 friend class Instruction;
3117
3118 BranchInst *cloneImpl() const;
3119
3120public:
3121 /// Iterator type that casts an operand to a basic block.
3122 ///
3123 /// This only makes sense because the successors are stored as adjacent
3124 /// operands for branch instructions.
3125 struct succ_op_iterator
3126 : iterator_adaptor_base<succ_op_iterator, value_op_iterator,
3127 std::random_access_iterator_tag, BasicBlock *,
3128 ptrdiff_t, BasicBlock *, BasicBlock *> {
3129 explicit succ_op_iterator(value_op_iterator I) : iterator_adaptor_base(I) {}
3130
3131 BasicBlock *operator*() const { return cast<BasicBlock>(*I); }
3132 BasicBlock *operator->() const { return operator*(); }
3133 };
3134
3135 /// The const version of `succ_op_iterator`.
3136 struct const_succ_op_iterator
3137 : iterator_adaptor_base<const_succ_op_iterator, const_value_op_iterator,
3138 std::random_access_iterator_tag,
3139 const BasicBlock *, ptrdiff_t, const BasicBlock *,
3140 const BasicBlock *> {
3141 explicit const_succ_op_iterator(const_value_op_iterator I)
3142 : iterator_adaptor_base(I) {}
3143
3144 const BasicBlock *operator*() const { return cast<BasicBlock>(*I); }
3145 const BasicBlock *operator->() const { return operator*(); }
3146 };
3147
3148 static BranchInst *Create(BasicBlock *IfTrue,
3149 Instruction *InsertBefore = nullptr) {
3150 return new(1) BranchInst(IfTrue, InsertBefore);
3151 }
3152
3153 static BranchInst *Create(BasicBlock *IfTrue, BasicBlock *IfFalse,
3154 Value *Cond, Instruction *InsertBefore = nullptr) {
3155 return new(3) BranchInst(IfTrue, IfFalse, Cond, InsertBefore);
3156 }
3157
3158 static BranchInst *Create(BasicBlock *IfTrue, BasicBlock *InsertAtEnd) {
3159 return new(1) BranchInst(IfTrue, InsertAtEnd);
3160 }
3161
3162 static BranchInst *Create(BasicBlock *IfTrue, BasicBlock *IfFalse,
3163 Value *Cond, BasicBlock *InsertAtEnd) {
3164 return new(3) BranchInst(IfTrue, IfFalse, Cond, InsertAtEnd);
3165 }
3166
3167 /// Transparently provide more efficient getOperand methods.
3168 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
3169
3170 bool isUnconditional() const { return getNumOperands() == 1; }
3171 bool isConditional() const { return getNumOperands() == 3; }
3172
3173 Value *getCondition() const {
3174 assert(isConditional() && "Cannot get condition of an uncond branch!")(static_cast <bool> (isConditional() && "Cannot get condition of an uncond branch!"
) ? void (0) : __assert_fail ("isConditional() && \"Cannot get condition of an uncond branch!\""
, "llvm/include/llvm/IR/Instructions.h", 3174, __extension__ __PRETTY_FUNCTION__
))
;
3175 return Op<-3>();
3176 }
3177
3178 void setCondition(Value *V) {
3179 assert(isConditional() && "Cannot set condition of unconditional branch!")(static_cast <bool> (isConditional() && "Cannot set condition of unconditional branch!"
) ? void (0) : __assert_fail ("isConditional() && \"Cannot set condition of unconditional branch!\""
, "llvm/include/llvm/IR/Instructions.h", 3179, __extension__ __PRETTY_FUNCTION__
))
;
3180 Op<-3>() = V;
3181 }
3182
3183 unsigned getNumSuccessors() const { return 1+isConditional(); }
3184
3185 BasicBlock *getSuccessor(unsigned i) const {
3186 assert(i < getNumSuccessors() && "Successor # out of range for Branch!")(static_cast <bool> (i < getNumSuccessors() &&
"Successor # out of range for Branch!") ? void (0) : __assert_fail
("i < getNumSuccessors() && \"Successor # out of range for Branch!\""
, "llvm/include/llvm/IR/Instructions.h", 3186, __extension__ __PRETTY_FUNCTION__
))
;
3187 return cast_or_null<BasicBlock>((&Op<-1>() - i)->get());
3188 }
3189
3190 void setSuccessor(unsigned idx, BasicBlock *NewSucc) {
3191 assert(idx < getNumSuccessors() && "Successor # out of range for Branch!")(static_cast <bool> (idx < getNumSuccessors() &&
"Successor # out of range for Branch!") ? void (0) : __assert_fail
("idx < getNumSuccessors() && \"Successor # out of range for Branch!\""
, "llvm/include/llvm/IR/Instructions.h", 3191, __extension__ __PRETTY_FUNCTION__
))
;
3192 *(&Op<-1>() - idx) = NewSucc;
3193 }
3194
3195 /// Swap the successors of this branch instruction.
3196 ///
3197 /// Swaps the successors of the branch instruction. This also swaps any
3198 /// branch weight metadata associated with the instruction so that it
3199 /// continues to map correctly to each operand.
3200 void swapSuccessors();
3201
3202 iterator_range<succ_op_iterator> successors() {
3203 return make_range(
3204 succ_op_iterator(std::next(value_op_begin(), isConditional() ? 1 : 0)),
3205 succ_op_iterator(value_op_end()));
3206 }
3207
3208 iterator_range<const_succ_op_iterator> successors() const {
3209 return make_range(const_succ_op_iterator(
3210 std::next(value_op_begin(), isConditional() ? 1 : 0)),
3211 const_succ_op_iterator(value_op_end()));
3212 }
3213
3214 // Methods for support type inquiry through isa, cast, and dyn_cast:
3215 static bool classof(const Instruction *I) {
3216 return (I->getOpcode() == Instruction::Br);
3217 }
3218 static bool classof(const Value *V) {
3219 return isa<Instruction>(V) && classof(cast<Instruction>(V));
3220 }
3221};
3222
3223template <>
3224struct OperandTraits<BranchInst> : public VariadicOperandTraits<BranchInst, 1> {
3225};
3226
3227DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BranchInst, Value)BranchInst::op_iterator BranchInst::op_begin() { return OperandTraits
<BranchInst>::op_begin(this); } BranchInst::const_op_iterator
BranchInst::op_begin() const { return OperandTraits<BranchInst
>::op_begin(const_cast<BranchInst*>(this)); } BranchInst
::op_iterator BranchInst::op_end() { return OperandTraits<
BranchInst>::op_end(this); } BranchInst::const_op_iterator
BranchInst::op_end() const { return OperandTraits<BranchInst
>::op_end(const_cast<BranchInst*>(this)); } Value *BranchInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<BranchInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<BranchInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 3227, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<BranchInst
>::op_begin(const_cast<BranchInst*>(this))[i_nocapture
].get()); } void BranchInst::setOperand(unsigned i_nocapture,
Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<BranchInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<BranchInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 3227, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<BranchInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned BranchInst::getNumOperands() const
{ return OperandTraits<BranchInst>::operands(this); } template
<int Idx_nocapture> Use &BranchInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &BranchInst::Op() const { return
this->OpFrom<Idx_nocapture>(this); }
3228
3229//===----------------------------------------------------------------------===//
3230// SwitchInst Class
3231//===----------------------------------------------------------------------===//
3232
3233//===---------------------------------------------------------------------------
3234/// Multiway switch
3235///
3236class SwitchInst : public Instruction {
3237 unsigned ReservedSpace;
3238
3239 // Operand[0] = Value to switch on
3240 // Operand[1] = Default basic block destination
3241 // Operand[2n ] = Value to match
3242 // Operand[2n+1] = BasicBlock to go to on match
3243 SwitchInst(const SwitchInst &SI);
3244
3245 /// Create a new switch instruction, specifying a value to switch on and a
3246 /// default destination. The number of additional cases can be specified here
3247 /// to make memory allocation more efficient. This constructor can also
3248 /// auto-insert before another instruction.
3249 SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases,
3250 Instruction *InsertBefore);
3251
3252 /// Create a new switch instruction, specifying a value to switch on and a
3253 /// default destination. The number of additional cases can be specified here
3254 /// to make memory allocation more efficient. This constructor also
3255 /// auto-inserts at the end of the specified BasicBlock.
3256 SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases,
3257 BasicBlock *InsertAtEnd);
3258
3259 // allocate space for exactly zero operands
3260 void *operator new(size_t S) { return User::operator new(S); }
3261
3262 void init(Value *Value, BasicBlock *Default, unsigned NumReserved);
3263 void growOperands();
3264
3265protected:
3266 // Note: Instruction needs to be a friend here to call cloneImpl.
3267 friend class Instruction;
3268
3269 SwitchInst *cloneImpl() const;
3270
3271public:
3272 void operator delete(void *Ptr) { User::operator delete(Ptr); }
3273
3274 // -2
3275 static const unsigned DefaultPseudoIndex = static_cast<unsigned>(~0L-1);
3276
3277 template <typename CaseHandleT> class CaseIteratorImpl;
3278
3279 /// A handle to a particular switch case. It exposes a convenient interface
3280 /// to both the case value and the successor block.
3281 ///
3282 /// We define this as a template and instantiate it to form both a const and
3283 /// non-const handle.
3284 template <typename SwitchInstT, typename ConstantIntT, typename BasicBlockT>
3285 class CaseHandleImpl {
3286 // Directly befriend both const and non-const iterators.
3287 friend class SwitchInst::CaseIteratorImpl<
3288 CaseHandleImpl<SwitchInstT, ConstantIntT, BasicBlockT>>;
3289
3290 protected:
3291 // Expose the switch type we're parameterized with to the iterator.
3292 using SwitchInstType = SwitchInstT;
3293
3294 SwitchInstT *SI;
3295 ptrdiff_t Index;
3296
3297 CaseHandleImpl() = default;
3298 CaseHandleImpl(SwitchInstT *SI, ptrdiff_t Index) : SI(SI), Index(Index) {}
3299
3300 public:
3301 /// Resolves case value for current case.
3302 ConstantIntT *getCaseValue() const {
3303 assert((unsigned)Index < SI->getNumCases() &&(static_cast <bool> ((unsigned)Index < SI->getNumCases
() && "Index out the number of cases.") ? void (0) : __assert_fail
("(unsigned)Index < SI->getNumCases() && \"Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3304, __extension__ __PRETTY_FUNCTION__
))
3304 "Index out the number of cases.")(static_cast <bool> ((unsigned)Index < SI->getNumCases
() && "Index out the number of cases.") ? void (0) : __assert_fail
("(unsigned)Index < SI->getNumCases() && \"Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3304, __extension__ __PRETTY_FUNCTION__
))
;
3305 return reinterpret_cast<ConstantIntT *>(SI->getOperand(2 + Index * 2));
3306 }
3307
3308 /// Resolves successor for current case.
3309 BasicBlockT *getCaseSuccessor() const {
3310 assert(((unsigned)Index < SI->getNumCases() ||(static_cast <bool> (((unsigned)Index < SI->getNumCases
() || (unsigned)Index == DefaultPseudoIndex) && "Index out the number of cases."
) ? void (0) : __assert_fail ("((unsigned)Index < SI->getNumCases() || (unsigned)Index == DefaultPseudoIndex) && \"Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3312, __extension__ __PRETTY_FUNCTION__
))
3311 (unsigned)Index == DefaultPseudoIndex) &&(static_cast <bool> (((unsigned)Index < SI->getNumCases
() || (unsigned)Index == DefaultPseudoIndex) && "Index out the number of cases."
) ? void (0) : __assert_fail ("((unsigned)Index < SI->getNumCases() || (unsigned)Index == DefaultPseudoIndex) && \"Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3312, __extension__ __PRETTY_FUNCTION__
))
3312 "Index out the number of cases.")(static_cast <bool> (((unsigned)Index < SI->getNumCases
() || (unsigned)Index == DefaultPseudoIndex) && "Index out the number of cases."
) ? void (0) : __assert_fail ("((unsigned)Index < SI->getNumCases() || (unsigned)Index == DefaultPseudoIndex) && \"Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3312, __extension__ __PRETTY_FUNCTION__
))
;
3313 return SI->getSuccessor(getSuccessorIndex());
3314 }
3315
3316 /// Returns number of current case.
3317 unsigned getCaseIndex() const { return Index; }
3318
3319 /// Returns successor index for current case successor.
3320 unsigned getSuccessorIndex() const {
3321 assert(((unsigned)Index == DefaultPseudoIndex ||(static_cast <bool> (((unsigned)Index == DefaultPseudoIndex
|| (unsigned)Index < SI->getNumCases()) && "Index out the number of cases."
) ? void (0) : __assert_fail ("((unsigned)Index == DefaultPseudoIndex || (unsigned)Index < SI->getNumCases()) && \"Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3323, __extension__ __PRETTY_FUNCTION__
))
3322 (unsigned)Index < SI->getNumCases()) &&(static_cast <bool> (((unsigned)Index == DefaultPseudoIndex
|| (unsigned)Index < SI->getNumCases()) && "Index out the number of cases."
) ? void (0) : __assert_fail ("((unsigned)Index == DefaultPseudoIndex || (unsigned)Index < SI->getNumCases()) && \"Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3323, __extension__ __PRETTY_FUNCTION__
))
3323 "Index out the number of cases.")(static_cast <bool> (((unsigned)Index == DefaultPseudoIndex
|| (unsigned)Index < SI->getNumCases()) && "Index out the number of cases."
) ? void (0) : __assert_fail ("((unsigned)Index == DefaultPseudoIndex || (unsigned)Index < SI->getNumCases()) && \"Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3323, __extension__ __PRETTY_FUNCTION__
))
;
3324 return (unsigned)Index != DefaultPseudoIndex ? Index + 1 : 0;
3325 }
3326
3327 bool operator==(const CaseHandleImpl &RHS) const {
3328 assert(SI == RHS.SI && "Incompatible operators.")(static_cast <bool> (SI == RHS.SI && "Incompatible operators."
) ? void (0) : __assert_fail ("SI == RHS.SI && \"Incompatible operators.\""
, "llvm/include/llvm/IR/Instructions.h", 3328, __extension__ __PRETTY_FUNCTION__
))
;
3329 return Index == RHS.Index;
3330 }
3331 };
3332
3333 using ConstCaseHandle =
3334 CaseHandleImpl<const SwitchInst, const ConstantInt, const BasicBlock>;
3335
3336 class CaseHandle
3337 : public CaseHandleImpl<SwitchInst, ConstantInt, BasicBlock> {
3338 friend class SwitchInst::CaseIteratorImpl<CaseHandle>;
3339
3340 public:
3341 CaseHandle(SwitchInst *SI, ptrdiff_t Index) : CaseHandleImpl(SI, Index) {}
3342
3343 /// Sets the new value for current case.
3344 void setValue(ConstantInt *V) const {
3345 assert((unsigned)Index < SI->getNumCases() &&(static_cast <bool> ((unsigned)Index < SI->getNumCases
() && "Index out the number of cases.") ? void (0) : __assert_fail
("(unsigned)Index < SI->getNumCases() && \"Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3346, __extension__ __PRETTY_FUNCTION__
))
3346 "Index out the number of cases.")(static_cast <bool> ((unsigned)Index < SI->getNumCases
() && "Index out the number of cases.") ? void (0) : __assert_fail
("(unsigned)Index < SI->getNumCases() && \"Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3346, __extension__ __PRETTY_FUNCTION__
))
;
3347 SI->setOperand(2 + Index*2, reinterpret_cast<Value*>(V));
3348 }
3349
3350 /// Sets the new successor for current case.
3351 void setSuccessor(BasicBlock *S) const {
3352 SI->setSuccessor(getSuccessorIndex(), S);
3353 }
3354 };
3355
3356 template <typename CaseHandleT>
3357 class CaseIteratorImpl
3358 : public iterator_facade_base<CaseIteratorImpl<CaseHandleT>,
3359 std::random_access_iterator_tag,
3360 const CaseHandleT> {
3361 using SwitchInstT = typename CaseHandleT::SwitchInstType;
3362
3363 CaseHandleT Case;
3364
3365 public:
3366 /// Default constructed iterator is in an invalid state until assigned to
3367 /// a case for a particular switch.
3368 CaseIteratorImpl() = default;
3369
3370 /// Initializes case iterator for given SwitchInst and for given
3371 /// case number.
3372 CaseIteratorImpl(SwitchInstT *SI, unsigned CaseNum) : Case(SI, CaseNum) {}
3373
3374 /// Initializes case iterator for given SwitchInst and for given
3375 /// successor index.
3376 static CaseIteratorImpl fromSuccessorIndex(SwitchInstT *SI,
3377 unsigned SuccessorIndex) {
3378 assert(SuccessorIndex < SI->getNumSuccessors() &&(static_cast <bool> (SuccessorIndex < SI->getNumSuccessors
() && "Successor index # out of range!") ? void (0) :
__assert_fail ("SuccessorIndex < SI->getNumSuccessors() && \"Successor index # out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 3379, __extension__ __PRETTY_FUNCTION__
))
3379 "Successor index # out of range!")(static_cast <bool> (SuccessorIndex < SI->getNumSuccessors
() && "Successor index # out of range!") ? void (0) :
__assert_fail ("SuccessorIndex < SI->getNumSuccessors() && \"Successor index # out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 3379, __extension__ __PRETTY_FUNCTION__
))
;
3380 return SuccessorIndex != 0 ? CaseIteratorImpl(SI, SuccessorIndex - 1)
3381 : CaseIteratorImpl(SI, DefaultPseudoIndex);
3382 }
3383
3384 /// Support converting to the const variant. This will be a no-op for const
3385 /// variant.
3386 operator CaseIteratorImpl<ConstCaseHandle>() const {
3387 return CaseIteratorImpl<ConstCaseHandle>(Case.SI, Case.Index);
3388 }
3389
3390 CaseIteratorImpl &operator+=(ptrdiff_t N) {
3391 // Check index correctness after addition.
3392 // Note: Index == getNumCases() means end().
3393 assert(Case.Index + N >= 0 &&(static_cast <bool> (Case.Index + N >= 0 && (
unsigned)(Case.Index + N) <= Case.SI->getNumCases() &&
"Case.Index out the number of cases.") ? void (0) : __assert_fail
("Case.Index + N >= 0 && (unsigned)(Case.Index + N) <= Case.SI->getNumCases() && \"Case.Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3395, __extension__ __PRETTY_FUNCTION__
))
3394 (unsigned)(Case.Index + N) <= Case.SI->getNumCases() &&(static_cast <bool> (Case.Index + N >= 0 && (
unsigned)(Case.Index + N) <= Case.SI->getNumCases() &&
"Case.Index out the number of cases.") ? void (0) : __assert_fail
("Case.Index + N >= 0 && (unsigned)(Case.Index + N) <= Case.SI->getNumCases() && \"Case.Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3395, __extension__ __PRETTY_FUNCTION__
))
3395 "Case.Index out the number of cases.")(static_cast <bool> (Case.Index + N >= 0 && (
unsigned)(Case.Index + N) <= Case.SI->getNumCases() &&
"Case.Index out the number of cases.") ? void (0) : __assert_fail
("Case.Index + N >= 0 && (unsigned)(Case.Index + N) <= Case.SI->getNumCases() && \"Case.Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3395, __extension__ __PRETTY_FUNCTION__
))
;
3396 Case.Index += N;
3397 return *this;
3398 }
3399 CaseIteratorImpl &operator-=(ptrdiff_t N) {
3400 // Check index correctness after subtraction.
3401 // Note: Case.Index == getNumCases() means end().
3402 assert(Case.Index - N >= 0 &&(static_cast <bool> (Case.Index - N >= 0 && (
unsigned)(Case.Index - N) <= Case.SI->getNumCases() &&
"Case.Index out the number of cases.") ? void (0) : __assert_fail
("Case.Index - N >= 0 && (unsigned)(Case.Index - N) <= Case.SI->getNumCases() && \"Case.Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3404, __extension__ __PRETTY_FUNCTION__
))
3403 (unsigned)(Case.Index - N) <= Case.SI->getNumCases() &&(static_cast <bool> (Case.Index - N >= 0 && (
unsigned)(Case.Index - N) <= Case.SI->getNumCases() &&
"Case.Index out the number of cases.") ? void (0) : __assert_fail
("Case.Index - N >= 0 && (unsigned)(Case.Index - N) <= Case.SI->getNumCases() && \"Case.Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3404, __extension__ __PRETTY_FUNCTION__
))
3404 "Case.Index out the number of cases.")(static_cast <bool> (Case.Index - N >= 0 && (
unsigned)(Case.Index - N) <= Case.SI->getNumCases() &&
"Case.Index out the number of cases.") ? void (0) : __assert_fail
("Case.Index - N >= 0 && (unsigned)(Case.Index - N) <= Case.SI->getNumCases() && \"Case.Index out the number of cases.\""
, "llvm/include/llvm/IR/Instructions.h", 3404, __extension__ __PRETTY_FUNCTION__
))
;
3405 Case.Index -= N;
3406 return *this;
3407 }
3408 ptrdiff_t operator-(const CaseIteratorImpl &RHS) const {
3409 assert(Case.SI == RHS.Case.SI && "Incompatible operators.")(static_cast <bool> (Case.SI == RHS.Case.SI && "Incompatible operators."
) ? void (0) : __assert_fail ("Case.SI == RHS.Case.SI && \"Incompatible operators.\""
, "llvm/include/llvm/IR/Instructions.h", 3409, __extension__ __PRETTY_FUNCTION__
))
;
3410 return Case.Index - RHS.Case.Index;
3411 }
3412 bool operator==(const CaseIteratorImpl &RHS) const {
3413 return Case == RHS.Case;
3414 }
3415 bool operator<(const CaseIteratorImpl &RHS) const {
3416 assert(Case.SI == RHS.Case.SI && "Incompatible operators.")(static_cast <bool> (Case.SI == RHS.Case.SI && "Incompatible operators."
) ? void (0) : __assert_fail ("Case.SI == RHS.Case.SI && \"Incompatible operators.\""
, "llvm/include/llvm/IR/Instructions.h", 3416, __extension__ __PRETTY_FUNCTION__
))
;
3417 return Case.Index < RHS.Case.Index;
3418 }
3419 const CaseHandleT &operator*() const { return Case; }
3420 };
3421
3422 using CaseIt = CaseIteratorImpl<CaseHandle>;
3423 using ConstCaseIt = CaseIteratorImpl<ConstCaseHandle>;
3424
3425 static SwitchInst *Create(Value *Value, BasicBlock *Default,
3426 unsigned NumCases,
3427 Instruction *InsertBefore = nullptr) {
3428 return new SwitchInst(Value, Default, NumCases, InsertBefore);
3429 }
3430
3431 static SwitchInst *Create(Value *Value, BasicBlock *Default,
3432 unsigned NumCases, BasicBlock *InsertAtEnd) {
3433 return new SwitchInst(Value, Default, NumCases, InsertAtEnd);
3434 }
3435
3436 /// Provide fast operand accessors
3437 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
3438
3439 // Accessor Methods for Switch stmt
3440 Value *getCondition() const { return getOperand(0); }
3441 void setCondition(Value *V) { setOperand(0, V); }
3442
3443 BasicBlock *getDefaultDest() const {
3444 return cast<BasicBlock>(getOperand(1));
3445 }
3446
3447 void setDefaultDest(BasicBlock *DefaultCase) {
3448 setOperand(1, reinterpret_cast<Value*>(DefaultCase));
3449 }
3450
3451 /// Return the number of 'cases' in this switch instruction, excluding the
3452 /// default case.
3453 unsigned getNumCases() const {
3454 return getNumOperands()/2 - 1;
3455 }
3456
3457 /// Returns a read/write iterator that points to the first case in the
3458 /// SwitchInst.
3459 CaseIt case_begin() {
3460 return CaseIt(this, 0);
3461 }
3462
3463 /// Returns a read-only iterator that points to the first case in the
3464 /// SwitchInst.
3465 ConstCaseIt case_begin() const {
3466 return ConstCaseIt(this, 0);
3467 }
3468
3469 /// Returns a read/write iterator that points one past the last in the
3470 /// SwitchInst.
3471 CaseIt case_end() {
3472 return CaseIt(this, getNumCases());
3473 }
3474
3475 /// Returns a read-only iterator that points one past the last in the
3476 /// SwitchInst.
3477 ConstCaseIt case_end() const {
3478 return ConstCaseIt(this, getNumCases());
3479 }
3480
3481 /// Iteration adapter for range-for loops.
3482 iterator_range<CaseIt> cases() {
3483 return make_range(case_begin(), case_end());
3484 }
3485
3486 /// Constant iteration adapter for range-for loops.
3487 iterator_range<ConstCaseIt> cases() const {
3488 return make_range(case_begin(), case_end());
3489 }
3490
3491 /// Returns an iterator that points to the default case.
3492 /// Note: this iterator allows to resolve successor only. Attempt
3493 /// to resolve case value causes an assertion.
3494 /// Also note, that increment and decrement also causes an assertion and
3495 /// makes iterator invalid.
3496 CaseIt case_default() {
3497 return CaseIt(this, DefaultPseudoIndex);
3498 }
3499 ConstCaseIt case_default() const {
3500 return ConstCaseIt(this, DefaultPseudoIndex);
3501 }
3502
3503 /// Search all of the case values for the specified constant. If it is
3504 /// explicitly handled, return the case iterator of it, otherwise return
3505 /// default case iterator to indicate that it is handled by the default
3506 /// handler.
3507 CaseIt findCaseValue(const ConstantInt *C) {
3508 return CaseIt(
3509 this,
3510 const_cast<const SwitchInst *>(this)->findCaseValue(C)->getCaseIndex());
3511 }
3512 ConstCaseIt findCaseValue(const ConstantInt *C) const {
3513 ConstCaseIt I = llvm::find_if(cases(), [C](const ConstCaseHandle &Case) {
3514 return Case.getCaseValue() == C;
3515 });
3516 if (I != case_end())
3517 return I;
3518
3519 return case_default();
3520 }
3521
3522 /// Finds the unique case value for a given successor. Returns null if the
3523 /// successor is not found, not unique, or is the default case.
3524 ConstantInt *findCaseDest(BasicBlock *BB) {
3525 if (BB == getDefaultDest())
3526 return nullptr;
3527
3528 ConstantInt *CI = nullptr;
3529 for (auto Case : cases()) {
3530 if (Case.getCaseSuccessor() != BB)
3531 continue;
3532
3533 if (CI)
3534 return nullptr; // Multiple cases lead to BB.
3535
3536 CI = Case.getCaseValue();
3537 }
3538
3539 return CI;
3540 }
3541
3542 /// Add an entry to the switch instruction.
3543 /// Note:
3544 /// This action invalidates case_end(). Old case_end() iterator will
3545 /// point to the added case.
3546 void addCase(ConstantInt *OnVal, BasicBlock *Dest);
3547
3548 /// This method removes the specified case and its successor from the switch
3549 /// instruction. Note that this operation may reorder the remaining cases at
3550 /// index idx and above.
3551 /// Note:
3552 /// This action invalidates iterators for all cases following the one removed,
3553 /// including the case_end() iterator. It returns an iterator for the next
3554 /// case.
3555 CaseIt removeCase(CaseIt I);
3556
3557 unsigned getNumSuccessors() const { return getNumOperands()/2; }
3558 BasicBlock *getSuccessor(unsigned idx) const {
3559 assert(idx < getNumSuccessors() &&"Successor idx out of range for switch!")(static_cast <bool> (idx < getNumSuccessors() &&
"Successor idx out of range for switch!") ? void (0) : __assert_fail
("idx < getNumSuccessors() &&\"Successor idx out of range for switch!\""
, "llvm/include/llvm/IR/Instructions.h", 3559, __extension__ __PRETTY_FUNCTION__
))
;
3560 return cast<BasicBlock>(getOperand(idx*2+1));
3561 }
3562 void setSuccessor(unsigned idx, BasicBlock *NewSucc) {
3563 assert(idx < getNumSuccessors() && "Successor # out of range for switch!")(static_cast <bool> (idx < getNumSuccessors() &&
"Successor # out of range for switch!") ? void (0) : __assert_fail
("idx < getNumSuccessors() && \"Successor # out of range for switch!\""
, "llvm/include/llvm/IR/Instructions.h", 3563, __extension__ __PRETTY_FUNCTION__
))
;
3564 setOperand(idx * 2 + 1, NewSucc);
3565 }
3566
3567 // Methods for support type inquiry through isa, cast, and dyn_cast:
3568 static bool classof(const Instruction *I) {
3569 return I->getOpcode() == Instruction::Switch;
3570 }
3571 static bool classof(const Value *V) {
3572 return isa<Instruction>(V) && classof(cast<Instruction>(V));
3573 }
3574};
3575
3576/// A wrapper class to simplify modification of SwitchInst cases along with
3577/// their prof branch_weights metadata.
3578class SwitchInstProfUpdateWrapper {
3579 SwitchInst &SI;
3580 Optional<SmallVector<uint32_t, 8> > Weights = None;
3581 bool Changed = false;
3582
3583protected:
3584 static MDNode *getProfBranchWeightsMD(const SwitchInst &SI);
3585
3586 MDNode *buildProfBranchWeightsMD();
3587
3588 void init();
3589
3590public:
3591 using CaseWeightOpt = Optional<uint32_t>;
3592 SwitchInst *operator->() { return &SI; }
3593 SwitchInst &operator*() { return SI; }
3594 operator SwitchInst *() { return &SI; }
3595
3596 SwitchInstProfUpdateWrapper(SwitchInst &SI) : SI(SI) { init(); }
3597
3598 ~SwitchInstProfUpdateWrapper() {
3599 if (Changed)
3600 SI.setMetadata(LLVMContext::MD_prof, buildProfBranchWeightsMD());
3601 }
3602
3603 /// Delegate the call to the underlying SwitchInst::removeCase() and remove
3604 /// correspondent branch weight.
3605 SwitchInst::CaseIt removeCase(SwitchInst::CaseIt I);
3606
3607 /// Delegate the call to the underlying SwitchInst::addCase() and set the
3608 /// specified branch weight for the added case.
3609 void addCase(ConstantInt *OnVal, BasicBlock *Dest, CaseWeightOpt W);
3610
3611 /// Delegate the call to the underlying SwitchInst::eraseFromParent() and mark
3612 /// this object to not touch the underlying SwitchInst in destructor.
3613 SymbolTableList<Instruction>::iterator eraseFromParent();
3614
3615 void setSuccessorWeight(unsigned idx, CaseWeightOpt W);
3616 CaseWeightOpt getSuccessorWeight(unsigned idx);
3617
3618 static CaseWeightOpt getSuccessorWeight(const SwitchInst &SI, unsigned idx);
3619};
3620
3621template <>
3622struct OperandTraits<SwitchInst> : public HungoffOperandTraits<2> {
3623};
3624
3625DEFINE_TRANSPARENT_OPERAND_ACCESSORS(SwitchInst, Value)SwitchInst::op_iterator SwitchInst::op_begin() { return OperandTraits
<SwitchInst>::op_begin(this); } SwitchInst::const_op_iterator
SwitchInst::op_begin() const { return OperandTraits<SwitchInst
>::op_begin(const_cast<SwitchInst*>(this)); } SwitchInst
::op_iterator SwitchInst::op_end() { return OperandTraits<
SwitchInst>::op_end(this); } SwitchInst::const_op_iterator
SwitchInst::op_end() const { return OperandTraits<SwitchInst
>::op_end(const_cast<SwitchInst*>(this)); } Value *SwitchInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<SwitchInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<SwitchInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 3625, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<SwitchInst
>::op_begin(const_cast<SwitchInst*>(this))[i_nocapture
].get()); } void SwitchInst::setOperand(unsigned i_nocapture,
Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<SwitchInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<SwitchInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 3625, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<SwitchInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned SwitchInst::getNumOperands() const
{ return OperandTraits<SwitchInst>::operands(this); } template
<int Idx_nocapture> Use &SwitchInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &SwitchInst::Op() const { return
this->OpFrom<Idx_nocapture>(this); }
3626
3627//===----------------------------------------------------------------------===//
3628// IndirectBrInst Class
3629//===----------------------------------------------------------------------===//
3630
3631//===---------------------------------------------------------------------------
3632/// Indirect Branch Instruction.
3633///
3634class IndirectBrInst : public Instruction {
3635 unsigned ReservedSpace;
3636
3637 // Operand[0] = Address to jump to
3638 // Operand[n+1] = n-th destination
3639 IndirectBrInst(const IndirectBrInst &IBI);
3640
3641 /// Create a new indirectbr instruction, specifying an
3642 /// Address to jump to. The number of expected destinations can be specified
3643 /// here to make memory allocation more efficient. This constructor can also
3644 /// autoinsert before another instruction.
3645 IndirectBrInst(Value *Address, unsigned NumDests, Instruction *InsertBefore);
3646
3647 /// Create a new indirectbr instruction, specifying an
3648 /// Address to jump to. The number of expected destinations can be specified
3649 /// here to make memory allocation more efficient. This constructor also
3650 /// autoinserts at the end of the specified BasicBlock.
3651 IndirectBrInst(Value *Address, unsigned NumDests, BasicBlock *InsertAtEnd);
3652
3653 // allocate space for exactly zero operands
3654 void *operator new(size_t S) { return User::operator new(S); }
3655
3656 void init(Value *Address, unsigned NumDests);
3657 void growOperands();
3658
3659protected:
3660 // Note: Instruction needs to be a friend here to call cloneImpl.
3661 friend class Instruction;
3662
3663 IndirectBrInst *cloneImpl() const;
3664
3665public:
3666 void operator delete(void *Ptr) { User::operator delete(Ptr); }
3667
3668 /// Iterator type that casts an operand to a basic block.
3669 ///
3670 /// This only makes sense because the successors are stored as adjacent
3671 /// operands for indirectbr instructions.
3672 struct succ_op_iterator
3673 : iterator_adaptor_base<succ_op_iterator, value_op_iterator,
3674 std::random_access_iterator_tag, BasicBlock *,
3675 ptrdiff_t, BasicBlock *, BasicBlock *> {
3676 explicit succ_op_iterator(value_op_iterator I) : iterator_adaptor_base(I) {}
3677
3678 BasicBlock *operator*() const { return cast<BasicBlock>(*I); }
3679 BasicBlock *operator->() const { return operator*(); }
3680 };
3681
3682 /// The const version of `succ_op_iterator`.
3683 struct const_succ_op_iterator
3684 : iterator_adaptor_base<const_succ_op_iterator, const_value_op_iterator,
3685 std::random_access_iterator_tag,
3686 const BasicBlock *, ptrdiff_t, const BasicBlock *,
3687 const BasicBlock *> {
3688 explicit const_succ_op_iterator(const_value_op_iterator I)
3689 : iterator_adaptor_base(I) {}
3690
3691 const BasicBlock *operator*() const { return cast<BasicBlock>(*I); }
3692 const BasicBlock *operator->() const { return operator*(); }
3693 };
3694
3695 static IndirectBrInst *Create(Value *Address, unsigned NumDests,
3696 Instruction *InsertBefore = nullptr) {
3697 return new IndirectBrInst(Address, NumDests, InsertBefore);
3698 }
3699
3700 static IndirectBrInst *Create(Value *Address, unsigned NumDests,
3701 BasicBlock *InsertAtEnd) {
3702 return new IndirectBrInst(Address, NumDests, InsertAtEnd);
3703 }
3704
3705 /// Provide fast operand accessors.
3706 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
3707
3708 // Accessor Methods for IndirectBrInst instruction.
3709 Value *getAddress() { return getOperand(0); }
3710 const Value *getAddress() const { return getOperand(0); }
3711 void setAddress(Value *V) { setOperand(0, V); }
3712
3713 /// return the number of possible destinations in this
3714 /// indirectbr instruction.
3715 unsigned getNumDestinations() const { return getNumOperands()-1; }
3716
3717 /// Return the specified destination.
3718 BasicBlock *getDestination(unsigned i) { return getSuccessor(i); }
3719 const BasicBlock *getDestination(unsigned i) const { return getSuccessor(i); }
3720
3721 /// Add a destination.
3722 ///
3723 void addDestination(BasicBlock *Dest);
3724
3725 /// This method removes the specified successor from the
3726 /// indirectbr instruction.
3727 void removeDestination(unsigned i);
3728
3729 unsigned getNumSuccessors() const { return getNumOperands()-1; }
3730 BasicBlock *getSuccessor(unsigned i) const {
3731 return cast<BasicBlock>(getOperand(i+1));
3732 }
3733 void setSuccessor(unsigned i, BasicBlock *NewSucc) {
3734 setOperand(i + 1, NewSucc);
3735 }
3736
3737 iterator_range<succ_op_iterator> successors() {
3738 return make_range(succ_op_iterator(std::next(value_op_begin())),
3739 succ_op_iterator(value_op_end()));
3740 }
3741
3742 iterator_range<const_succ_op_iterator> successors() const {
3743 return make_range(const_succ_op_iterator(std::next(value_op_begin())),
3744 const_succ_op_iterator(value_op_end()));
3745 }
3746
3747 // Methods for support type inquiry through isa, cast, and dyn_cast:
3748 static bool classof(const Instruction *I) {
3749 return I->getOpcode() == Instruction::IndirectBr;
3750 }
3751 static bool classof(const Value *V) {
3752 return isa<Instruction>(V) && classof(cast<Instruction>(V));
3753 }
3754};
3755
3756template <>
3757struct OperandTraits<IndirectBrInst> : public HungoffOperandTraits<1> {
3758};
3759
3760DEFINE_TRANSPARENT_OPERAND_ACCESSORS(IndirectBrInst, Value)IndirectBrInst::op_iterator IndirectBrInst::op_begin() { return
OperandTraits<IndirectBrInst>::op_begin(this); } IndirectBrInst
::const_op_iterator IndirectBrInst::op_begin() const { return
OperandTraits<IndirectBrInst>::op_begin(const_cast<
IndirectBrInst*>(this)); } IndirectBrInst::op_iterator IndirectBrInst
::op_end() { return OperandTraits<IndirectBrInst>::op_end
(this); } IndirectBrInst::const_op_iterator IndirectBrInst::op_end
() const { return OperandTraits<IndirectBrInst>::op_end
(const_cast<IndirectBrInst*>(this)); } Value *IndirectBrInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<IndirectBrInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<IndirectBrInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 3760, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<IndirectBrInst
>::op_begin(const_cast<IndirectBrInst*>(this))[i_nocapture
].get()); } void IndirectBrInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<IndirectBrInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<IndirectBrInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 3760, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<IndirectBrInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned IndirectBrInst::getNumOperands(
) const { return OperandTraits<IndirectBrInst>::operands
(this); } template <int Idx_nocapture> Use &IndirectBrInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &IndirectBrInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
3761
3762//===----------------------------------------------------------------------===//
3763// InvokeInst Class
3764//===----------------------------------------------------------------------===//
3765
3766/// Invoke instruction. The SubclassData field is used to hold the
3767/// calling convention of the call.
3768///
3769class InvokeInst : public CallBase {
3770 /// The number of operands for this call beyond the called function,
3771 /// arguments, and operand bundles.
3772 static constexpr int NumExtraOperands = 2;
3773
3774 /// The index from the end of the operand array to the normal destination.
3775 static constexpr int NormalDestOpEndIdx = -3;
3776
3777 /// The index from the end of the operand array to the unwind destination.
3778 static constexpr int UnwindDestOpEndIdx = -2;
3779
3780 InvokeInst(const InvokeInst &BI);
3781
3782 /// Construct an InvokeInst given a range of arguments.
3783 ///
3784 /// Construct an InvokeInst from a range of arguments
3785 inline InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3786 BasicBlock *IfException, ArrayRef<Value *> Args,
3787 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
3788 const Twine &NameStr, Instruction *InsertBefore);
3789
3790 inline InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3791 BasicBlock *IfException, ArrayRef<Value *> Args,
3792 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
3793 const Twine &NameStr, BasicBlock *InsertAtEnd);
3794
3795 void init(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3796 BasicBlock *IfException, ArrayRef<Value *> Args,
3797 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
3798
3799 /// Compute the number of operands to allocate.
3800 static int ComputeNumOperands(int NumArgs, int NumBundleInputs = 0) {
3801 // We need one operand for the called function, plus our extra operands and
3802 // the input operand counts provided.
3803 return 1 + NumExtraOperands + NumArgs + NumBundleInputs;
3804 }
3805
3806protected:
3807 // Note: Instruction needs to be a friend here to call cloneImpl.
3808 friend class Instruction;
3809
3810 InvokeInst *cloneImpl() const;
3811
3812public:
3813 static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3814 BasicBlock *IfException, ArrayRef<Value *> Args,
3815 const Twine &NameStr,
3816 Instruction *InsertBefore = nullptr) {
3817 int NumOperands = ComputeNumOperands(Args.size());
3818 return new (NumOperands)
3819 InvokeInst(Ty, Func, IfNormal, IfException, Args, None, NumOperands,
3820 NameStr, InsertBefore);
3821 }
3822
3823 static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3824 BasicBlock *IfException, ArrayRef<Value *> Args,
3825 ArrayRef<OperandBundleDef> Bundles = None,
3826 const Twine &NameStr = "",
3827 Instruction *InsertBefore = nullptr) {
3828 int NumOperands =
3829 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
3830 unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
3831
3832 return new (NumOperands, DescriptorBytes)
3833 InvokeInst(Ty, Func, IfNormal, IfException, Args, Bundles, NumOperands,
3834 NameStr, InsertBefore);
3835 }
3836
3837 static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3838 BasicBlock *IfException, ArrayRef<Value *> Args,
3839 const Twine &NameStr, BasicBlock *InsertAtEnd) {
3840 int NumOperands = ComputeNumOperands(Args.size());
3841 return new (NumOperands)
3842 InvokeInst(Ty, Func, IfNormal, IfException, Args, None, NumOperands,
3843 NameStr, InsertAtEnd);
3844 }
3845
3846 static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3847 BasicBlock *IfException, ArrayRef<Value *> Args,
3848 ArrayRef<OperandBundleDef> Bundles,
3849 const Twine &NameStr, BasicBlock *InsertAtEnd) {
3850 int NumOperands =
3851 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
3852 unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
3853
3854 return new (NumOperands, DescriptorBytes)
3855 InvokeInst(Ty, Func, IfNormal, IfException, Args, Bundles, NumOperands,
3856 NameStr, InsertAtEnd);
3857 }
3858
3859 static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal,
3860 BasicBlock *IfException, ArrayRef<Value *> Args,
3861 const Twine &NameStr,
3862 Instruction *InsertBefore = nullptr) {
3863 return Create(Func.getFunctionType(), Func.getCallee(), IfNormal,
3864 IfException, Args, None, NameStr, InsertBefore);
3865 }
3866
3867 static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal,
3868 BasicBlock *IfException, ArrayRef<Value *> Args,
3869 ArrayRef<OperandBundleDef> Bundles = None,
3870 const Twine &NameStr = "",
3871 Instruction *InsertBefore = nullptr) {
3872 return Create(Func.getFunctionType(), Func.getCallee(), IfNormal,
3873 IfException, Args, Bundles, NameStr, InsertBefore);
3874 }
3875
3876 static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal,
3877 BasicBlock *IfException, ArrayRef<Value *> Args,
3878 const Twine &NameStr, BasicBlock *InsertAtEnd) {
3879 return Create(Func.getFunctionType(), Func.getCallee(), IfNormal,
3880 IfException, Args, NameStr, InsertAtEnd);
3881 }
3882
3883 static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal,
3884 BasicBlock *IfException, ArrayRef<Value *> Args,
3885 ArrayRef<OperandBundleDef> Bundles,
3886 const Twine &NameStr, BasicBlock *InsertAtEnd) {
3887 return Create(Func.getFunctionType(), Func.getCallee(), IfNormal,
3888 IfException, Args, Bundles, NameStr, InsertAtEnd);
3889 }
3890
3891 /// Create a clone of \p II with a different set of operand bundles and
3892 /// insert it before \p InsertPt.
3893 ///
3894 /// The returned invoke instruction is identical to \p II in every way except
3895 /// that the operand bundles for the new instruction are set to the operand
3896 /// bundles in \p Bundles.
3897 static InvokeInst *Create(InvokeInst *II, ArrayRef<OperandBundleDef> Bundles,
3898 Instruction *InsertPt = nullptr);
3899
3900 // get*Dest - Return the destination basic blocks...
3901 BasicBlock *getNormalDest() const {
3902 return cast<BasicBlock>(Op<NormalDestOpEndIdx>());
3903 }
3904 BasicBlock *getUnwindDest() const {
3905 return cast<BasicBlock>(Op<UnwindDestOpEndIdx>());
3906 }
3907 void setNormalDest(BasicBlock *B) {
3908 Op<NormalDestOpEndIdx>() = reinterpret_cast<Value *>(B);
3909 }
3910 void setUnwindDest(BasicBlock *B) {
3911 Op<UnwindDestOpEndIdx>() = reinterpret_cast<Value *>(B);
3912 }
3913
3914 /// Get the landingpad instruction from the landing pad
3915 /// block (the unwind destination).
3916 LandingPadInst *getLandingPadInst() const;
3917
3918 BasicBlock *getSuccessor(unsigned i) const {
3919 assert(i < 2 && "Successor # out of range for invoke!")(static_cast <bool> (i < 2 && "Successor # out of range for invoke!"
) ? void (0) : __assert_fail ("i < 2 && \"Successor # out of range for invoke!\""
, "llvm/include/llvm/IR/Instructions.h", 3919, __extension__ __PRETTY_FUNCTION__
))
;
3920 return i == 0 ? getNormalDest() : getUnwindDest();
3921 }
3922
3923 void setSuccessor(unsigned i, BasicBlock *NewSucc) {
3924 assert(i < 2 && "Successor # out of range for invoke!")(static_cast <bool> (i < 2 && "Successor # out of range for invoke!"
) ? void (0) : __assert_fail ("i < 2 && \"Successor # out of range for invoke!\""
, "llvm/include/llvm/IR/Instructions.h", 3924, __extension__ __PRETTY_FUNCTION__
))
;
3925 if (i == 0)
3926 setNormalDest(NewSucc);
3927 else
3928 setUnwindDest(NewSucc);
3929 }
3930
3931 unsigned getNumSuccessors() const { return 2; }
3932
3933 // Methods for support type inquiry through isa, cast, and dyn_cast:
3934 static bool classof(const Instruction *I) {
3935 return (I->getOpcode() == Instruction::Invoke);
3936 }
3937 static bool classof(const Value *V) {
3938 return isa<Instruction>(V) && classof(cast<Instruction>(V));
3939 }
3940
3941private:
3942 // Shadow Instruction::setInstructionSubclassData with a private forwarding
3943 // method so that subclasses cannot accidentally use it.
3944 template <typename Bitfield>
3945 void setSubclassData(typename Bitfield::Type Value) {
3946 Instruction::setSubclassData<Bitfield>(Value);
3947 }
3948};
3949
3950InvokeInst::InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3951 BasicBlock *IfException, ArrayRef<Value *> Args,
3952 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
3953 const Twine &NameStr, Instruction *InsertBefore)
3954 : CallBase(Ty->getReturnType(), Instruction::Invoke,
3955 OperandTraits<CallBase>::op_end(this) - NumOperands, NumOperands,
3956 InsertBefore) {
3957 init(Ty, Func, IfNormal, IfException, Args, Bundles, NameStr);
3958}
3959
3960InvokeInst::InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3961 BasicBlock *IfException, ArrayRef<Value *> Args,
3962 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
3963 const Twine &NameStr, BasicBlock *InsertAtEnd)
3964 : CallBase(Ty->getReturnType(), Instruction::Invoke,
3965 OperandTraits<CallBase>::op_end(this) - NumOperands, NumOperands,
3966 InsertAtEnd) {
3967 init(Ty, Func, IfNormal, IfException, Args, Bundles, NameStr);
3968}
3969
3970//===----------------------------------------------------------------------===//
3971// CallBrInst Class
3972//===----------------------------------------------------------------------===//
3973
3974/// CallBr instruction, tracking function calls that may not return control but
3975/// instead transfer it to a third location. The SubclassData field is used to
3976/// hold the calling convention of the call.
3977///
3978class CallBrInst : public CallBase {
3979
3980 unsigned NumIndirectDests;
3981
3982 CallBrInst(const CallBrInst &BI);
3983
3984 /// Construct a CallBrInst given a range of arguments.
3985 ///
3986 /// Construct a CallBrInst from a range of arguments
3987 inline CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest,
3988 ArrayRef<BasicBlock *> IndirectDests,
3989 ArrayRef<Value *> Args,
3990 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
3991 const Twine &NameStr, Instruction *InsertBefore);
3992
3993 inline CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest,
3994 ArrayRef<BasicBlock *> IndirectDests,
3995 ArrayRef<Value *> Args,
3996 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
3997 const Twine &NameStr, BasicBlock *InsertAtEnd);
3998
3999 void init(FunctionType *FTy, Value *Func, BasicBlock *DefaultDest,
4000 ArrayRef<BasicBlock *> IndirectDests, ArrayRef<Value *> Args,
4001 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
4002
4003 /// Should the Indirect Destinations change, scan + update the Arg list.
4004 void updateArgBlockAddresses(unsigned i, BasicBlock *B);
4005
4006 /// Compute the number of operands to allocate.
4007 static int ComputeNumOperands(int NumArgs, int NumIndirectDests,
4008 int NumBundleInputs = 0) {
4009 // We need one operand for the called function, plus our extra operands and
4010 // the input operand counts provided.
4011 return 2 + NumIndirectDests + NumArgs + NumBundleInputs;
4012 }
4013
4014protected:
4015 // Note: Instruction needs to be a friend here to call cloneImpl.
4016 friend class Instruction;
4017
4018 CallBrInst *cloneImpl() const;
4019
4020public:
4021 static CallBrInst *Create(FunctionType *Ty, Value *Func,
4022 BasicBlock *DefaultDest,
4023 ArrayRef<BasicBlock *> IndirectDests,
4024 ArrayRef<Value *> Args, const Twine &NameStr,
4025 Instruction *InsertBefore = nullptr) {
4026 int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size());
4027 return new (NumOperands)
4028 CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, None,
4029 NumOperands, NameStr, InsertBefore);
4030 }
4031
4032 static CallBrInst *Create(FunctionType *Ty, Value *Func,
4033 BasicBlock *DefaultDest,
4034 ArrayRef<BasicBlock *> IndirectDests,
4035 ArrayRef<Value *> Args,
4036 ArrayRef<OperandBundleDef> Bundles = None,
4037 const Twine &NameStr = "",
4038 Instruction *InsertBefore = nullptr) {
4039 int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size(),
4040 CountBundleInputs(Bundles));
4041 unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
4042
4043 return new (NumOperands, DescriptorBytes)
4044 CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, Bundles,
4045 NumOperands, NameStr, InsertBefore);
4046 }
4047
4048 static CallBrInst *Create(FunctionType *Ty, Value *Func,
4049 BasicBlock *DefaultDest,
4050 ArrayRef<BasicBlock *> IndirectDests,
4051 ArrayRef<Value *> Args, const Twine &NameStr,
4052 BasicBlock *InsertAtEnd) {
4053 int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size());
4054 return new (NumOperands)
4055 CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, None,
4056 NumOperands, NameStr, InsertAtEnd);
4057 }
4058
4059 static CallBrInst *Create(FunctionType *Ty, Value *Func,
4060 BasicBlock *DefaultDest,
4061 ArrayRef<BasicBlock *> IndirectDests,
4062 ArrayRef<Value *> Args,
4063 ArrayRef<OperandBundleDef> Bundles,
4064 const Twine &NameStr, BasicBlock *InsertAtEnd) {
4065 int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size(),
4066 CountBundleInputs(Bundles));
4067 unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
4068
4069 return new (NumOperands, DescriptorBytes)
4070 CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, Bundles,
4071 NumOperands, NameStr, InsertAtEnd);
4072 }
4073
4074 static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest,
4075 ArrayRef<BasicBlock *> IndirectDests,
4076 ArrayRef<Value *> Args, const Twine &NameStr,
4077 Instruction *InsertBefore = nullptr) {
4078 return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest,
4079 IndirectDests, Args, NameStr, InsertBefore);
4080 }
4081
4082 static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest,
4083 ArrayRef<BasicBlock *> IndirectDests,
4084 ArrayRef<Value *> Args,
4085 ArrayRef<OperandBundleDef> Bundles = None,
4086 const Twine &NameStr = "",
4087 Instruction *InsertBefore = nullptr) {
4088 return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest,
4089 IndirectDests, Args, Bundles, NameStr, InsertBefore);
4090 }
4091
4092 static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest,
4093 ArrayRef<BasicBlock *> IndirectDests,
4094 ArrayRef<Value *> Args, const Twine &NameStr,
4095 BasicBlock *InsertAtEnd) {
4096 return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest,
4097 IndirectDests, Args, NameStr, InsertAtEnd);
4098 }
4099
4100 static CallBrInst *Create(FunctionCallee Func,
4101 BasicBlock *DefaultDest,
4102 ArrayRef<BasicBlock *> IndirectDests,
4103 ArrayRef<Value *> Args,
4104 ArrayRef<OperandBundleDef> Bundles,
4105 const Twine &NameStr, BasicBlock *InsertAtEnd) {
4106 return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest,
4107 IndirectDests, Args, Bundles, NameStr, InsertAtEnd);
4108 }
4109
4110 /// Create a clone of \p CBI with a different set of operand bundles and
4111 /// insert it before \p InsertPt.
4112 ///
4113 /// The returned callbr instruction is identical to \p CBI in every way
4114 /// except that the operand bundles for the new instruction are set to the
4115 /// operand bundles in \p Bundles.
4116 static CallBrInst *Create(CallBrInst *CBI,
4117 ArrayRef<OperandBundleDef> Bundles,
4118 Instruction *InsertPt = nullptr);
4119
4120 /// Return the number of callbr indirect dest labels.
4121 ///
4122 unsigned getNumIndirectDests() const { return NumIndirectDests; }
4123
4124 /// getIndirectDestLabel - Return the i-th indirect dest label.
4125 ///
4126 Value *getIndirectDestLabel(unsigned i) const {
4127 assert(i < getNumIndirectDests() && "Out of bounds!")(static_cast <bool> (i < getNumIndirectDests() &&
"Out of bounds!") ? void (0) : __assert_fail ("i < getNumIndirectDests() && \"Out of bounds!\""
, "llvm/include/llvm/IR/Instructions.h", 4127, __extension__ __PRETTY_FUNCTION__
))
;
4128 return getOperand(i + arg_size() + getNumTotalBundleOperands() + 1);
4129 }
4130
4131 Value *getIndirectDestLabelUse(unsigned i) const {
4132 assert(i < getNumIndirectDests() && "Out of bounds!")(static_cast <bool> (i < getNumIndirectDests() &&
"Out of bounds!") ? void (0) : __assert_fail ("i < getNumIndirectDests() && \"Out of bounds!\""
, "llvm/include/llvm/IR/Instructions.h", 4132, __extension__ __PRETTY_FUNCTION__
))
;
4133 return getOperandUse(i + arg_size() + getNumTotalBundleOperands() + 1);
4134 }
4135
4136 // Return the destination basic blocks...
4137 BasicBlock *getDefaultDest() const {
4138 return cast<BasicBlock>(*(&Op<-1>() - getNumIndirectDests() - 1));
4139 }
4140 BasicBlock *getIndirectDest(unsigned i) const {
4141 return cast_or_null<BasicBlock>(*(&Op<-1>() - getNumIndirectDests() + i));
4142 }
4143 SmallVector<BasicBlock *, 16> getIndirectDests() const {
4144 SmallVector<BasicBlock *, 16> IndirectDests;
4145 for (unsigned i = 0, e = getNumIndirectDests(); i < e; ++i)
4146 IndirectDests.push_back(getIndirectDest(i));
4147 return IndirectDests;
4148 }
4149 void setDefaultDest(BasicBlock *B) {
4150 *(&Op<-1>() - getNumIndirectDests() - 1) = reinterpret_cast<Value *>(B);
4151 }
4152 void setIndirectDest(unsigned i, BasicBlock *B) {
4153 updateArgBlockAddresses(i, B);
4154 *(&Op<-1>() - getNumIndirectDests() + i) = reinterpret_cast<Value *>(B);
4155 }
4156
4157 BasicBlock *getSuccessor(unsigned i) const {
4158 assert(i < getNumSuccessors() + 1 &&(static_cast <bool> (i < getNumSuccessors() + 1 &&
"Successor # out of range for callbr!") ? void (0) : __assert_fail
("i < getNumSuccessors() + 1 && \"Successor # out of range for callbr!\""
, "llvm/include/llvm/IR/Instructions.h", 4159, __extension__ __PRETTY_FUNCTION__
))
4159 "Successor # out of range for callbr!")(static_cast <bool> (i < getNumSuccessors() + 1 &&
"Successor # out of range for callbr!") ? void (0) : __assert_fail
("i < getNumSuccessors() + 1 && \"Successor # out of range for callbr!\""
, "llvm/include/llvm/IR/Instructions.h", 4159, __extension__ __PRETTY_FUNCTION__
))
;
4160 return i == 0 ? getDefaultDest() : getIndirectDest(i - 1);
4161 }
4162
4163 void setSuccessor(unsigned i, BasicBlock *NewSucc) {
4164 assert(i < getNumIndirectDests() + 1 &&(static_cast <bool> (i < getNumIndirectDests() + 1 &&
"Successor # out of range for callbr!") ? void (0) : __assert_fail
("i < getNumIndirectDests() + 1 && \"Successor # out of range for callbr!\""
, "llvm/include/llvm/IR/Instructions.h", 4165, __extension__ __PRETTY_FUNCTION__
))
4165 "Successor # out of range for callbr!")(static_cast <bool> (i < getNumIndirectDests() + 1 &&
"Successor # out of range for callbr!") ? void (0) : __assert_fail
("i < getNumIndirectDests() + 1 && \"Successor # out of range for callbr!\""
, "llvm/include/llvm/IR/Instructions.h", 4165, __extension__ __PRETTY_FUNCTION__
))
;
4166 return i == 0 ? setDefaultDest(NewSucc) : setIndirectDest(i - 1, NewSucc);
4167 }
4168
4169 unsigned getNumSuccessors() const { return getNumIndirectDests() + 1; }
4170
4171 // Methods for support type inquiry through isa, cast, and dyn_cast:
4172 static bool classof(const Instruction *I) {
4173 return (I->getOpcode() == Instruction::CallBr);
4174 }
4175 static bool classof(const Value *V) {
4176 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4177 }
4178
4179private:
4180 // Shadow Instruction::setInstructionSubclassData with a private forwarding
4181 // method so that subclasses cannot accidentally use it.
4182 template <typename Bitfield>
4183 void setSubclassData(typename Bitfield::Type Value) {
4184 Instruction::setSubclassData<Bitfield>(Value);
4185 }
4186};
4187
4188CallBrInst::CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest,
4189 ArrayRef<BasicBlock *> IndirectDests,
4190 ArrayRef<Value *> Args,
4191 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
4192 const Twine &NameStr, Instruction *InsertBefore)
4193 : CallBase(Ty->getReturnType(), Instruction::CallBr,
4194 OperandTraits<CallBase>::op_end(this) - NumOperands, NumOperands,
4195 InsertBefore) {
4196 init(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, NameStr);
4197}
4198
4199CallBrInst::CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest,
4200 ArrayRef<BasicBlock *> IndirectDests,
4201 ArrayRef<Value *> Args,
4202 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
4203 const Twine &NameStr, BasicBlock *InsertAtEnd)
4204 : CallBase(Ty->getReturnType(), Instruction::CallBr,
4205 OperandTraits<CallBase>::op_end(this) - NumOperands, NumOperands,
4206 InsertAtEnd) {
4207 init(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, NameStr);
4208}
4209
4210//===----------------------------------------------------------------------===//
4211// ResumeInst Class
4212//===----------------------------------------------------------------------===//
4213
4214//===---------------------------------------------------------------------------
4215/// Resume the propagation of an exception.
4216///
4217class ResumeInst : public Instruction {
4218 ResumeInst(const ResumeInst &RI);
4219
4220 explicit ResumeInst(Value *Exn, Instruction *InsertBefore=nullptr);
4221 ResumeInst(Value *Exn, BasicBlock *InsertAtEnd);
4222
4223protected:
4224 // Note: Instruction needs to be a friend here to call cloneImpl.
4225 friend class Instruction;
4226
4227 ResumeInst *cloneImpl() const;
4228
4229public:
4230 static ResumeInst *Create(Value *Exn, Instruction *InsertBefore = nullptr) {
4231 return new(1) ResumeInst(Exn, InsertBefore);
4232 }
4233
4234 static ResumeInst *Create(Value *Exn, BasicBlock *InsertAtEnd) {
4235 return new(1) ResumeInst(Exn, InsertAtEnd);
4236 }
4237
4238 /// Provide fast operand accessors
4239 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
4240
4241 /// Convenience accessor.
4242 Value *getValue() const { return Op<0>(); }
4243
4244 unsigned getNumSuccessors() const { return 0; }
4245
4246 // Methods for support type inquiry through isa, cast, and dyn_cast:
4247 static bool classof(const Instruction *I) {
4248 return I->getOpcode() == Instruction::Resume;
4249 }
4250 static bool classof(const Value *V) {
4251 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4252 }
4253
4254private:
4255 BasicBlock *getSuccessor(unsigned idx) const {
4256 llvm_unreachable("ResumeInst has no successors!")::llvm::llvm_unreachable_internal("ResumeInst has no successors!"
, "llvm/include/llvm/IR/Instructions.h", 4256)
;
4257 }
4258
4259 void setSuccessor(unsigned idx, BasicBlock *NewSucc) {
4260 llvm_unreachable("ResumeInst has no successors!")::llvm::llvm_unreachable_internal("ResumeInst has no successors!"
, "llvm/include/llvm/IR/Instructions.h", 4260)
;
4261 }
4262};
4263
4264template <>
4265struct OperandTraits<ResumeInst> :
4266 public FixedNumOperandTraits<ResumeInst, 1> {
4267};
4268
4269DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ResumeInst, Value)ResumeInst::op_iterator ResumeInst::op_begin() { return OperandTraits
<ResumeInst>::op_begin(this); } ResumeInst::const_op_iterator
ResumeInst::op_begin() const { return OperandTraits<ResumeInst
>::op_begin(const_cast<ResumeInst*>(this)); } ResumeInst
::op_iterator ResumeInst::op_end() { return OperandTraits<
ResumeInst>::op_end(this); } ResumeInst::const_op_iterator
ResumeInst::op_end() const { return OperandTraits<ResumeInst
>::op_end(const_cast<ResumeInst*>(this)); } Value *ResumeInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<ResumeInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<ResumeInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 4269, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<ResumeInst
>::op_begin(const_cast<ResumeInst*>(this))[i_nocapture
].get()); } void ResumeInst::setOperand(unsigned i_nocapture,
Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ResumeInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ResumeInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 4269, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<ResumeInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned ResumeInst::getNumOperands() const
{ return OperandTraits<ResumeInst>::operands(this); } template
<int Idx_nocapture> Use &ResumeInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &ResumeInst::Op() const { return
this->OpFrom<Idx_nocapture>(this); }
4270
4271//===----------------------------------------------------------------------===//
4272// CatchSwitchInst Class
4273//===----------------------------------------------------------------------===//
4274class CatchSwitchInst : public Instruction {
4275 using UnwindDestField = BoolBitfieldElementT<0>;
4276
4277 /// The number of operands actually allocated. NumOperands is
4278 /// the number actually in use.
4279 unsigned ReservedSpace;
4280
4281 // Operand[0] = Outer scope
4282 // Operand[1] = Unwind block destination
4283 // Operand[n] = BasicBlock to go to on match
4284 CatchSwitchInst(const CatchSwitchInst &CSI);
4285
4286 /// Create a new switch instruction, specifying a
4287 /// default destination. The number of additional handlers can be specified
4288 /// here to make memory allocation more efficient.
4289 /// This constructor can also autoinsert before another instruction.
4290 CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest,
4291 unsigned NumHandlers, const Twine &NameStr,
4292 Instruction *InsertBefore);
4293
4294 /// Create a new switch instruction, specifying a
4295 /// default destination. The number of additional handlers can be specified
4296 /// here to make memory allocation more efficient.
4297 /// This constructor also autoinserts at the end of the specified BasicBlock.
4298 CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest,
4299 unsigned NumHandlers, const Twine &NameStr,
4300 BasicBlock *InsertAtEnd);
4301
4302 // allocate space for exactly zero operands
4303 void *operator new(size_t S) { return User::operator new(S); }
4304
4305 void init(Value *ParentPad, BasicBlock *UnwindDest, unsigned NumReserved);
4306 void growOperands(unsigned Size);
4307
4308protected:
4309 // Note: Instruction needs to be a friend here to call cloneImpl.
4310 friend class Instruction;
4311
4312 CatchSwitchInst *cloneImpl() const;
4313
4314public:
4315 void operator delete(void *Ptr) { return User::operator delete(Ptr); }
4316
4317 static CatchSwitchInst *Create(Value *ParentPad, BasicBlock *UnwindDest,
4318 unsigned NumHandlers,
4319 const Twine &NameStr = "",
4320 Instruction *InsertBefore = nullptr) {
4321 return new CatchSwitchInst(ParentPad, UnwindDest, NumHandlers, NameStr,
4322 InsertBefore);
4323 }
4324
4325 static CatchSwitchInst *Create(Value *ParentPad, BasicBlock *UnwindDest,
4326 unsigned NumHandlers, const Twine &NameStr,
4327 BasicBlock *InsertAtEnd) {
4328 return new CatchSwitchInst(ParentPad, UnwindDest, NumHandlers, NameStr,
4329 InsertAtEnd);
4330 }
4331
4332 /// Provide fast operand accessors
4333 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
4334
4335 // Accessor Methods for CatchSwitch stmt
4336 Value *getParentPad() const { return getOperand(0); }
4337 void setParentPad(Value *ParentPad) { setOperand(0, ParentPad); }
4338
4339 // Accessor Methods for CatchSwitch stmt
4340 bool hasUnwindDest() const { return getSubclassData<UnwindDestField>(); }
4341 bool unwindsToCaller() const { return !hasUnwindDest(); }
4342 BasicBlock *getUnwindDest() const {
4343 if (hasUnwindDest())
4344 return cast<BasicBlock>(getOperand(1));
4345 return nullptr;
4346 }
4347 void setUnwindDest(BasicBlock *UnwindDest) {
4348 assert(UnwindDest)(static_cast <bool> (UnwindDest) ? void (0) : __assert_fail
("UnwindDest", "llvm/include/llvm/IR/Instructions.h", 4348, __extension__
__PRETTY_FUNCTION__))
;
4349 assert(hasUnwindDest())(static_cast <bool> (hasUnwindDest()) ? void (0) : __assert_fail
("hasUnwindDest()", "llvm/include/llvm/IR/Instructions.h", 4349
, __extension__ __PRETTY_FUNCTION__))
;
4350 setOperand(1, UnwindDest);
4351 }
4352
4353 /// return the number of 'handlers' in this catchswitch
4354 /// instruction, except the default handler
4355 unsigned getNumHandlers() const {
4356 if (hasUnwindDest())
4357 return getNumOperands() - 2;
4358 return getNumOperands() - 1;
4359 }
4360
4361private:
4362 static BasicBlock *handler_helper(Value *V) { return cast<BasicBlock>(V); }
4363 static const BasicBlock *handler_helper(const Value *V) {
4364 return cast<BasicBlock>(V);
4365 }
4366
4367public:
4368 using DerefFnTy = BasicBlock *(*)(Value *);
4369 using handler_iterator = mapped_iterator<op_iterator, DerefFnTy>;
4370 using handler_range = iterator_range<handler_iterator>;
4371 using ConstDerefFnTy = const BasicBlock *(*)(const Value *);
4372 using const_handler_iterator =
4373 mapped_iterator<const_op_iterator, ConstDerefFnTy>;
4374 using const_handler_range = iterator_range<const_handler_iterator>;
4375
4376 /// Returns an iterator that points to the first handler in CatchSwitchInst.
4377 handler_iterator handler_begin() {
4378 op_iterator It = op_begin() + 1;
4379 if (hasUnwindDest())
4380 ++It;
4381 return handler_iterator(It, DerefFnTy(handler_helper));
4382 }
4383
4384 /// Returns an iterator that points to the first handler in the
4385 /// CatchSwitchInst.
4386 const_handler_iterator handler_begin() const {
4387 const_op_iterator It = op_begin() + 1;
4388 if (hasUnwindDest())
4389 ++It;
4390 return const_handler_iterator(It, ConstDerefFnTy(handler_helper));
4391 }
4392
4393 /// Returns a read-only iterator that points one past the last
4394 /// handler in the CatchSwitchInst.
4395 handler_iterator handler_end() {
4396 return handler_iterator(op_end(), DerefFnTy(handler_helper));
4397 }
4398
4399 /// Returns an iterator that points one past the last handler in the
4400 /// CatchSwitchInst.
4401 const_handler_iterator handler_end() const {
4402 return const_handler_iterator(op_end(), ConstDerefFnTy(handler_helper));
4403 }
4404
4405 /// iteration adapter for range-for loops.
4406 handler_range handlers() {
4407 return make_range(handler_begin(), handler_end());
4408 }
4409
4410 /// iteration adapter for range-for loops.
4411 const_handler_range handlers() const {
4412 return make_range(handler_begin(), handler_end());
4413 }
4414
4415 /// Add an entry to the switch instruction...
4416 /// Note:
4417 /// This action invalidates handler_end(). Old handler_end() iterator will
4418 /// point to the added handler.
4419 void addHandler(BasicBlock *Dest);
4420
4421 void removeHandler(handler_iterator HI);
4422
4423 unsigned getNumSuccessors() const { return getNumOperands() - 1; }
4424 BasicBlock *getSuccessor(unsigned Idx) const {
4425 assert(Idx < getNumSuccessors() &&(static_cast <bool> (Idx < getNumSuccessors() &&
"Successor # out of range for catchswitch!") ? void (0) : __assert_fail
("Idx < getNumSuccessors() && \"Successor # out of range for catchswitch!\""
, "llvm/include/llvm/IR/Instructions.h", 4426, __extension__ __PRETTY_FUNCTION__
))
4426 "Successor # out of range for catchswitch!")(static_cast <bool> (Idx < getNumSuccessors() &&
"Successor # out of range for catchswitch!") ? void (0) : __assert_fail
("Idx < getNumSuccessors() && \"Successor # out of range for catchswitch!\""
, "llvm/include/llvm/IR/Instructions.h", 4426, __extension__ __PRETTY_FUNCTION__
))
;
4427 return cast<BasicBlock>(getOperand(Idx + 1));
4428 }
4429 void setSuccessor(unsigned Idx, BasicBlock *NewSucc) {
4430 assert(Idx < getNumSuccessors() &&(static_cast <bool> (Idx < getNumSuccessors() &&
"Successor # out of range for catchswitch!") ? void (0) : __assert_fail
("Idx < getNumSuccessors() && \"Successor # out of range for catchswitch!\""
, "llvm/include/llvm/IR/Instructions.h", 4431, __extension__ __PRETTY_FUNCTION__
))
4431 "Successor # out of range for catchswitch!")(static_cast <bool> (Idx < getNumSuccessors() &&
"Successor # out of range for catchswitch!") ? void (0) : __assert_fail
("Idx < getNumSuccessors() && \"Successor # out of range for catchswitch!\""
, "llvm/include/llvm/IR/Instructions.h", 4431, __extension__ __PRETTY_FUNCTION__
))
;
4432 setOperand(Idx + 1, NewSucc);
4433 }
4434
4435 // Methods for support type inquiry through isa, cast, and dyn_cast:
4436 static bool classof(const Instruction *I) {
4437 return I->getOpcode() == Instruction::CatchSwitch;
4438 }
4439 static bool classof(const Value *V) {
4440 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4441 }
4442};
4443
4444template <>
4445struct OperandTraits<CatchSwitchInst> : public HungoffOperandTraits<2> {};
4446
4447DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CatchSwitchInst, Value)CatchSwitchInst::op_iterator CatchSwitchInst::op_begin() { return
OperandTraits<CatchSwitchInst>::op_begin(this); } CatchSwitchInst
::const_op_iterator CatchSwitchInst::op_begin() const { return
OperandTraits<CatchSwitchInst>::op_begin(const_cast<
CatchSwitchInst*>(this)); } CatchSwitchInst::op_iterator CatchSwitchInst
::op_end() { return OperandTraits<CatchSwitchInst>::op_end
(this); } CatchSwitchInst::const_op_iterator CatchSwitchInst::
op_end() const { return OperandTraits<CatchSwitchInst>::
op_end(const_cast<CatchSwitchInst*>(this)); } Value *CatchSwitchInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<CatchSwitchInst>::
operands(this) && "getOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<CatchSwitchInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 4447, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<CatchSwitchInst
>::op_begin(const_cast<CatchSwitchInst*>(this))[i_nocapture
].get()); } void CatchSwitchInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<CatchSwitchInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<CatchSwitchInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 4447, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<CatchSwitchInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned CatchSwitchInst::getNumOperands
() const { return OperandTraits<CatchSwitchInst>::operands
(this); } template <int Idx_nocapture> Use &CatchSwitchInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &CatchSwitchInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
4448
4449//===----------------------------------------------------------------------===//
4450// CleanupPadInst Class
4451//===----------------------------------------------------------------------===//
4452class CleanupPadInst : public FuncletPadInst {
4453private:
4454 explicit CleanupPadInst(Value *ParentPad, ArrayRef<Value *> Args,
4455 unsigned Values, const Twine &NameStr,
4456 Instruction *InsertBefore)
4457 : FuncletPadInst(Instruction::CleanupPad, ParentPad, Args, Values,
4458 NameStr, InsertBefore) {}
4459 explicit CleanupPadInst(Value *ParentPad, ArrayRef<Value *> Args,
4460 unsigned Values, const Twine &NameStr,
4461 BasicBlock *InsertAtEnd)
4462 : FuncletPadInst(Instruction::CleanupPad, ParentPad, Args, Values,
4463 NameStr, InsertAtEnd) {}
4464
4465public:
4466 static CleanupPadInst *Create(Value *ParentPad, ArrayRef<Value *> Args = None,
4467 const Twine &NameStr = "",
4468 Instruction *InsertBefore = nullptr) {
4469 unsigned Values = 1 + Args.size();
4470 return new (Values)
4471 CleanupPadInst(ParentPad, Args, Values, NameStr, InsertBefore);
4472 }
4473
4474 static CleanupPadInst *Create(Value *ParentPad, ArrayRef<Value *> Args,
4475 const Twine &NameStr, BasicBlock *InsertAtEnd) {
4476 unsigned Values = 1 + Args.size();
4477 return new (Values)
4478 CleanupPadInst(ParentPad, Args, Values, NameStr, InsertAtEnd);
4479 }
4480
4481 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4482 static bool classof(const Instruction *I) {
4483 return I->getOpcode() == Instruction::CleanupPad;
4484 }
4485 static bool classof(const Value *V) {
4486 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4487 }
4488};
4489
4490//===----------------------------------------------------------------------===//
4491// CatchPadInst Class
4492//===----------------------------------------------------------------------===//
4493class CatchPadInst : public FuncletPadInst {
4494private:
4495 explicit CatchPadInst(Value *CatchSwitch, ArrayRef<Value *> Args,
4496 unsigned Values, const Twine &NameStr,
4497 Instruction *InsertBefore)
4498 : FuncletPadInst(Instruction::CatchPad, CatchSwitch, Args, Values,
4499 NameStr, InsertBefore) {}
4500 explicit CatchPadInst(Value *CatchSwitch, ArrayRef<Value *> Args,
4501 unsigned Values, const Twine &NameStr,
4502 BasicBlock *InsertAtEnd)
4503 : FuncletPadInst(Instruction::CatchPad, CatchSwitch, Args, Values,
4504 NameStr, InsertAtEnd) {}
4505
4506public:
4507 static CatchPadInst *Create(Value *CatchSwitch, ArrayRef<Value *> Args,
4508 const Twine &NameStr = "",
4509 Instruction *InsertBefore = nullptr) {
4510 unsigned Values = 1 + Args.size();
4511 return new (Values)
4512 CatchPadInst(CatchSwitch, Args, Values, NameStr, InsertBefore);
4513 }
4514
4515 static CatchPadInst *Create(Value *CatchSwitch, ArrayRef<Value *> Args,
4516 const Twine &NameStr, BasicBlock *InsertAtEnd) {
4517 unsigned Values = 1 + Args.size();
4518 return new (Values)
4519 CatchPadInst(CatchSwitch, Args, Values, NameStr, InsertAtEnd);
4520 }
4521
4522 /// Convenience accessors
4523 CatchSwitchInst *getCatchSwitch() const {
4524 return cast<CatchSwitchInst>(Op<-1>());
4525 }
4526 void setCatchSwitch(Value *CatchSwitch) {
4527 assert(CatchSwitch)(static_cast <bool> (CatchSwitch) ? void (0) : __assert_fail
("CatchSwitch", "llvm/include/llvm/IR/Instructions.h", 4527,
__extension__ __PRETTY_FUNCTION__))
;
4528 Op<-1>() = CatchSwitch;
4529 }
4530
4531 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4532 static bool classof(const Instruction *I) {
4533 return I->getOpcode() == Instruction::CatchPad;
4534 }
4535 static bool classof(const Value *V) {
4536 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4537 }
4538};
4539
4540//===----------------------------------------------------------------------===//
4541// CatchReturnInst Class
4542//===----------------------------------------------------------------------===//
4543
4544class CatchReturnInst : public Instruction {
4545 CatchReturnInst(const CatchReturnInst &RI);
4546 CatchReturnInst(Value *CatchPad, BasicBlock *BB, Instruction *InsertBefore);
4547 CatchReturnInst(Value *CatchPad, BasicBlock *BB, BasicBlock *InsertAtEnd);
4548
4549 void init(Value *CatchPad, BasicBlock *BB);
4550
4551protected:
4552 // Note: Instruction needs to be a friend here to call cloneImpl.
4553 friend class Instruction;
4554
4555 CatchReturnInst *cloneImpl() const;
4556
4557public:
4558 static CatchReturnInst *Create(Value *CatchPad, BasicBlock *BB,
4559 Instruction *InsertBefore = nullptr) {
4560 assert(CatchPad)(static_cast <bool> (CatchPad) ? void (0) : __assert_fail
("CatchPad", "llvm/include/llvm/IR/Instructions.h", 4560, __extension__
__PRETTY_FUNCTION__))
;
4561 assert(BB)(static_cast <bool> (BB) ? void (0) : __assert_fail ("BB"
, "llvm/include/llvm/IR/Instructions.h", 4561, __extension__ __PRETTY_FUNCTION__
))
;
4562 return new (2) CatchReturnInst(CatchPad, BB, InsertBefore);
4563 }
4564
4565 static CatchReturnInst *Create(Value *CatchPad, BasicBlock *BB,
4566 BasicBlock *InsertAtEnd) {
4567 assert(CatchPad)(static_cast <bool> (CatchPad) ? void (0) : __assert_fail
("CatchPad", "llvm/include/llvm/IR/Instructions.h", 4567, __extension__
__PRETTY_FUNCTION__))
;
4568 assert(BB)(static_cast <bool> (BB) ? void (0) : __assert_fail ("BB"
, "llvm/include/llvm/IR/Instructions.h", 4568, __extension__ __PRETTY_FUNCTION__
))
;
4569 return new (2) CatchReturnInst(CatchPad, BB, InsertAtEnd);
4570 }
4571
4572 /// Provide fast operand accessors
4573 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
4574
4575 /// Convenience accessors.
4576 CatchPadInst *getCatchPad() const { return cast<CatchPadInst>(Op<0>()); }
4577 void setCatchPad(CatchPadInst *CatchPad) {
4578 assert(CatchPad)(static_cast <bool> (CatchPad) ? void (0) : __assert_fail
("CatchPad", "llvm/include/llvm/IR/Instructions.h", 4578, __extension__
__PRETTY_FUNCTION__))
;
4579 Op<0>() = CatchPad;
4580 }
4581
4582 BasicBlock *getSuccessor() const { return cast<BasicBlock>(Op<1>()); }
4583 void setSuccessor(BasicBlock *NewSucc) {
4584 assert(NewSucc)(static_cast <bool> (NewSucc) ? void (0) : __assert_fail
("NewSucc", "llvm/include/llvm/IR/Instructions.h", 4584, __extension__
__PRETTY_FUNCTION__))
;
4585 Op<1>() = NewSucc;
4586 }
4587 unsigned getNumSuccessors() const { return 1; }
4588
4589 /// Get the parentPad of this catchret's catchpad's catchswitch.
4590 /// The successor block is implicitly a member of this funclet.
4591 Value *getCatchSwitchParentPad() const {
4592 return getCatchPad()->getCatchSwitch()->getParentPad();
4593 }
4594
4595 // Methods for support type inquiry through isa, cast, and dyn_cast:
4596 static bool classof(const Instruction *I) {
4597 return (I->getOpcode() == Instruction::CatchRet);
4598 }
4599 static bool classof(const Value *V) {
4600 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4601 }
4602
4603private:
4604 BasicBlock *getSuccessor(unsigned Idx) const {
4605 assert(Idx < getNumSuccessors() && "Successor # out of range for catchret!")(static_cast <bool> (Idx < getNumSuccessors() &&
"Successor # out of range for catchret!") ? void (0) : __assert_fail
("Idx < getNumSuccessors() && \"Successor # out of range for catchret!\""
, "llvm/include/llvm/IR/Instructions.h", 4605, __extension__ __PRETTY_FUNCTION__
))
;
4606 return getSuccessor();
4607 }
4608
4609 void setSuccessor(unsigned Idx, BasicBlock *B) {
4610 assert(Idx < getNumSuccessors() && "Successor # out of range for catchret!")(static_cast <bool> (Idx < getNumSuccessors() &&
"Successor # out of range for catchret!") ? void (0) : __assert_fail
("Idx < getNumSuccessors() && \"Successor # out of range for catchret!\""
, "llvm/include/llvm/IR/Instructions.h", 4610, __extension__ __PRETTY_FUNCTION__
))
;
4611 setSuccessor(B);
4612 }
4613};
4614
4615template <>
4616struct OperandTraits<CatchReturnInst>
4617 : public FixedNumOperandTraits<CatchReturnInst, 2> {};
4618
4619DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CatchReturnInst, Value)CatchReturnInst::op_iterator CatchReturnInst::op_begin() { return
OperandTraits<CatchReturnInst>::op_begin(this); } CatchReturnInst
::const_op_iterator CatchReturnInst::op_begin() const { return
OperandTraits<CatchReturnInst>::op_begin(const_cast<
CatchReturnInst*>(this)); } CatchReturnInst::op_iterator CatchReturnInst
::op_end() { return OperandTraits<CatchReturnInst>::op_end
(this); } CatchReturnInst::const_op_iterator CatchReturnInst::
op_end() const { return OperandTraits<CatchReturnInst>::
op_end(const_cast<CatchReturnInst*>(this)); } Value *CatchReturnInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<CatchReturnInst>::
operands(this) && "getOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<CatchReturnInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 4619, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<CatchReturnInst
>::op_begin(const_cast<CatchReturnInst*>(this))[i_nocapture
].get()); } void CatchReturnInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<CatchReturnInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<CatchReturnInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 4619, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<CatchReturnInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned CatchReturnInst::getNumOperands
() const { return OperandTraits<CatchReturnInst>::operands
(this); } template <int Idx_nocapture> Use &CatchReturnInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &CatchReturnInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
4620
4621//===----------------------------------------------------------------------===//
4622// CleanupReturnInst Class
4623//===----------------------------------------------------------------------===//
4624
4625class CleanupReturnInst : public Instruction {
4626 using UnwindDestField = BoolBitfieldElementT<0>;
4627
4628private:
4629 CleanupReturnInst(const CleanupReturnInst &RI);
4630 CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB, unsigned Values,
4631 Instruction *InsertBefore = nullptr);
4632 CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB, unsigned Values,
4633 BasicBlock *InsertAtEnd);
4634
4635 void init(Value *CleanupPad, BasicBlock *UnwindBB);
4636
4637protected:
4638 // Note: Instruction needs to be a friend here to call cloneImpl.
4639 friend class Instruction;
4640
4641 CleanupReturnInst *cloneImpl() const;
4642
4643public:
4644 static CleanupReturnInst *Create(Value *CleanupPad,
4645 BasicBlock *UnwindBB = nullptr,
4646 Instruction *InsertBefore = nullptr) {
4647 assert(CleanupPad)(static_cast <bool> (CleanupPad) ? void (0) : __assert_fail
("CleanupPad", "llvm/include/llvm/IR/Instructions.h", 4647, __extension__
__PRETTY_FUNCTION__))
;
4648 unsigned Values = 1;
4649 if (UnwindBB)
4650 ++Values;
4651 return new (Values)
4652 CleanupReturnInst(CleanupPad, UnwindBB, Values, InsertBefore);
4653 }
4654
4655 static CleanupReturnInst *Create(Value *CleanupPad, BasicBlock *UnwindBB,
4656 BasicBlock *InsertAtEnd) {
4657 assert(CleanupPad)(static_cast <bool> (CleanupPad) ? void (0) : __assert_fail
("CleanupPad", "llvm/include/llvm/IR/Instructions.h", 4657, __extension__
__PRETTY_FUNCTION__))
;
4658 unsigned Values = 1;
4659 if (UnwindBB)
4660 ++Values;
4661 return new (Values)
4662 CleanupReturnInst(CleanupPad, UnwindBB, Values, InsertAtEnd);
4663 }
4664
4665 /// Provide fast operand accessors
4666 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
4667
4668 bool hasUnwindDest() const { return getSubclassData<UnwindDestField>(); }
4669 bool unwindsToCaller() const { return !hasUnwindDest(); }
4670
4671 /// Convenience accessor.
4672 CleanupPadInst *getCleanupPad() const {
4673 return cast<CleanupPadInst>(Op<0>());
4674 }
4675 void setCleanupPad(CleanupPadInst *CleanupPad) {
4676 assert(CleanupPad)(static_cast <bool> (CleanupPad) ? void (0) : __assert_fail
("CleanupPad", "llvm/include/llvm/IR/Instructions.h", 4676, __extension__
__PRETTY_FUNCTION__))
;
4677 Op<0>() = CleanupPad;
4678 }
4679
4680 unsigned getNumSuccessors() const { return hasUnwindDest() ? 1 : 0; }
4681
4682 BasicBlock *getUnwindDest() const {
4683 return hasUnwindDest() ? cast<BasicBlock>(Op<1>()) : nullptr;
4684 }
4685 void setUnwindDest(BasicBlock *NewDest) {
4686 assert(NewDest)(static_cast <bool> (NewDest) ? void (0) : __assert_fail
("NewDest", "llvm/include/llvm/IR/Instructions.h", 4686, __extension__
__PRETTY_FUNCTION__))
;
4687 assert(hasUnwindDest())(static_cast <bool> (hasUnwindDest()) ? void (0) : __assert_fail
("hasUnwindDest()", "llvm/include/llvm/IR/Instructions.h", 4687
, __extension__ __PRETTY_FUNCTION__))
;
4688 Op<1>() = NewDest;
4689 }
4690
4691 // Methods for support type inquiry through isa, cast, and dyn_cast:
4692 static bool classof(const Instruction *I) {
4693 return (I->getOpcode() == Instruction::CleanupRet);
4694 }
4695 static bool classof(const Value *V) {
4696 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4697 }
4698
4699private:
4700 BasicBlock *getSuccessor(unsigned Idx) const {
4701 assert(Idx == 0)(static_cast <bool> (Idx == 0) ? void (0) : __assert_fail
("Idx == 0", "llvm/include/llvm/IR/Instructions.h", 4701, __extension__
__PRETTY_FUNCTION__))
;
4702 return getUnwindDest();
4703 }
4704
4705 void setSuccessor(unsigned Idx, BasicBlock *B) {
4706 assert(Idx == 0)(static_cast <bool> (Idx == 0) ? void (0) : __assert_fail
("Idx == 0", "llvm/include/llvm/IR/Instructions.h", 4706, __extension__
__PRETTY_FUNCTION__))
;
4707 setUnwindDest(B);
4708 }
4709
4710 // Shadow Instruction::setInstructionSubclassData with a private forwarding
4711 // method so that subclasses cannot accidentally use it.
4712 template <typename Bitfield>
4713 void setSubclassData(typename Bitfield::Type Value) {
4714 Instruction::setSubclassData<Bitfield>(Value);
4715 }
4716};
4717
4718template <>
4719struct OperandTraits<CleanupReturnInst>
4720 : public VariadicOperandTraits<CleanupReturnInst, /*MINARITY=*/1> {};
4721
4722DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CleanupReturnInst, Value)CleanupReturnInst::op_iterator CleanupReturnInst::op_begin() {
return OperandTraits<CleanupReturnInst>::op_begin(this
); } CleanupReturnInst::const_op_iterator CleanupReturnInst::
op_begin() const { return OperandTraits<CleanupReturnInst>
::op_begin(const_cast<CleanupReturnInst*>(this)); } CleanupReturnInst
::op_iterator CleanupReturnInst::op_end() { return OperandTraits
<CleanupReturnInst>::op_end(this); } CleanupReturnInst::
const_op_iterator CleanupReturnInst::op_end() const { return OperandTraits
<CleanupReturnInst>::op_end(const_cast<CleanupReturnInst
*>(this)); } Value *CleanupReturnInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<CleanupReturnInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<CleanupReturnInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 4722, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<CleanupReturnInst
>::op_begin(const_cast<CleanupReturnInst*>(this))[i_nocapture
].get()); } void CleanupReturnInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<CleanupReturnInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<CleanupReturnInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 4722, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<CleanupReturnInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned CleanupReturnInst::getNumOperands
() const { return OperandTraits<CleanupReturnInst>::operands
(this); } template <int Idx_nocapture> Use &CleanupReturnInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &CleanupReturnInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
4723
4724//===----------------------------------------------------------------------===//
4725// UnreachableInst Class
4726//===----------------------------------------------------------------------===//
4727
4728//===---------------------------------------------------------------------------
4729/// This function has undefined behavior. In particular, the
4730/// presence of this instruction indicates some higher level knowledge that the
4731/// end of the block cannot be reached.
4732///
4733class UnreachableInst : public Instruction {
4734protected:
4735 // Note: Instruction needs to be a friend here to call cloneImpl.
4736 friend class Instruction;
4737
4738 UnreachableInst *cloneImpl() const;
4739
4740public:
4741 explicit UnreachableInst(LLVMContext &C, Instruction *InsertBefore = nullptr);
4742 explicit UnreachableInst(LLVMContext &C, BasicBlock *InsertAtEnd);
4743
4744 // allocate space for exactly zero operands
4745 void *operator new(size_t S) { return User::operator new(S, 0); }
4746 void operator delete(void *Ptr) { User::operator delete(Ptr); }
4747
4748 unsigned getNumSuccessors() const { return 0; }
4749
4750 // Methods for support type inquiry through isa, cast, and dyn_cast:
4751 static bool classof(const Instruction *I) {
4752 return I->getOpcode() == Instruction::Unreachable;
4753 }
4754 static bool classof(const Value *V) {
4755 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4756 }
4757
4758private:
4759 BasicBlock *getSuccessor(unsigned idx) const {
4760 llvm_unreachable("UnreachableInst has no successors!")::llvm::llvm_unreachable_internal("UnreachableInst has no successors!"
, "llvm/include/llvm/IR/Instructions.h", 4760)
;
4761 }
4762
4763 void setSuccessor(unsigned idx, BasicBlock *B) {
4764 llvm_unreachable("UnreachableInst has no successors!")::llvm::llvm_unreachable_internal("UnreachableInst has no successors!"
, "llvm/include/llvm/IR/Instructions.h", 4764)
;
4765 }
4766};
4767
4768//===----------------------------------------------------------------------===//
4769// TruncInst Class
4770//===----------------------------------------------------------------------===//
4771
4772/// This class represents a truncation of integer types.
4773class TruncInst : public CastInst {
4774protected:
4775 // Note: Instruction needs to be a friend here to call cloneImpl.
4776 friend class Instruction;
4777
4778 /// Clone an identical TruncInst
4779 TruncInst *cloneImpl() const;
4780
4781public:
4782 /// Constructor with insert-before-instruction semantics
4783 TruncInst(
4784 Value *S, ///< The value to be truncated
4785 Type *Ty, ///< The (smaller) type to truncate to
4786 const Twine &NameStr = "", ///< A name for the new instruction
4787 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4788 );
4789
4790 /// Constructor with insert-at-end-of-block semantics
4791 TruncInst(
4792 Value *S, ///< The value to be truncated
4793 Type *Ty, ///< The (smaller) type to truncate to
4794 const Twine &NameStr, ///< A name for the new instruction
4795 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4796 );
4797
4798 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4799 static bool classof(const Instruction *I) {
4800 return I->getOpcode() == Trunc;
4801 }
4802 static bool classof(const Value *V) {
4803 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4804 }
4805};
4806
4807//===----------------------------------------------------------------------===//
4808// ZExtInst Class
4809//===----------------------------------------------------------------------===//
4810
4811/// This class represents zero extension of integer types.
4812class ZExtInst : public CastInst {
4813protected:
4814 // Note: Instruction needs to be a friend here to call cloneImpl.
4815 friend class Instruction;
4816
4817 /// Clone an identical ZExtInst
4818 ZExtInst *cloneImpl() const;
4819
4820public:
4821 /// Constructor with insert-before-instruction semantics
4822 ZExtInst(
4823 Value *S, ///< The value to be zero extended
4824 Type *Ty, ///< The type to zero extend to
4825 const Twine &NameStr = "", ///< A name for the new instruction
4826 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4827 );
4828
4829 /// Constructor with insert-at-end semantics.
4830 ZExtInst(
4831 Value *S, ///< The value to be zero extended
4832 Type *Ty, ///< The type to zero extend to
4833 const Twine &NameStr, ///< A name for the new instruction
4834 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4835 );
4836
4837 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4838 static bool classof(const Instruction *I) {
4839 return I->getOpcode() == ZExt;
4840 }
4841 static bool classof(const Value *V) {
4842 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4843 }
4844};
4845
4846//===----------------------------------------------------------------------===//
4847// SExtInst Class
4848//===----------------------------------------------------------------------===//
4849
4850/// This class represents a sign extension of integer types.
4851class SExtInst : public CastInst {
4852protected:
4853 // Note: Instruction needs to be a friend here to call cloneImpl.
4854 friend class Instruction;
4855
4856 /// Clone an identical SExtInst
4857 SExtInst *cloneImpl() const;
4858
4859public:
4860 /// Constructor with insert-before-instruction semantics
4861 SExtInst(
4862 Value *S, ///< The value to be sign extended
4863 Type *Ty, ///< The type to sign extend to
4864 const Twine &NameStr = "", ///< A name for the new instruction
4865 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4866 );
4867
4868 /// Constructor with insert-at-end-of-block semantics
4869 SExtInst(
4870 Value *S, ///< The value to be sign extended
4871 Type *Ty, ///< The type to sign extend to
4872 const Twine &NameStr, ///< A name for the new instruction
4873 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4874 );
4875
4876 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4877 static bool classof(const Instruction *I) {
4878 return I->getOpcode() == SExt;
4879 }
4880 static bool classof(const Value *V) {
4881 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4882 }
4883};
4884
4885//===----------------------------------------------------------------------===//
4886// FPTruncInst Class
4887//===----------------------------------------------------------------------===//
4888
4889/// This class represents a truncation of floating point types.
4890class FPTruncInst : public CastInst {
4891protected:
4892 // Note: Instruction needs to be a friend here to call cloneImpl.
4893 friend class Instruction;
4894
4895 /// Clone an identical FPTruncInst
4896 FPTruncInst *cloneImpl() const;
4897
4898public:
4899 /// Constructor with insert-before-instruction semantics
4900 FPTruncInst(
4901 Value *S, ///< The value to be truncated
4902 Type *Ty, ///< The type to truncate to
4903 const Twine &NameStr = "", ///< A name for the new instruction
4904 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4905 );
4906
4907 /// Constructor with insert-before-instruction semantics
4908 FPTruncInst(
4909 Value *S, ///< The value to be truncated
4910 Type *Ty, ///< The type to truncate to
4911 const Twine &NameStr, ///< A name for the new instruction
4912 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4913 );
4914
4915 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4916 static bool classof(const Instruction *I) {
4917 return I->getOpcode() == FPTrunc;
4918 }
4919 static bool classof(const Value *V) {
4920 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4921 }
4922};
4923
4924//===----------------------------------------------------------------------===//
4925// FPExtInst Class
4926//===----------------------------------------------------------------------===//
4927
4928/// This class represents an extension of floating point types.
4929class FPExtInst : public CastInst {
4930protected:
4931 // Note: Instruction needs to be a friend here to call cloneImpl.
4932 friend class Instruction;
4933
4934 /// Clone an identical FPExtInst
4935 FPExtInst *cloneImpl() const;
4936
4937public:
4938 /// Constructor with insert-before-instruction semantics
4939 FPExtInst(
4940 Value *S, ///< The value to be extended
4941 Type *Ty, ///< The type to extend to
4942 const Twine &NameStr = "", ///< A name for the new instruction
4943 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4944 );
4945
4946 /// Constructor with insert-at-end-of-block semantics
4947 FPExtInst(
4948 Value *S, ///< The value to be extended
4949 Type *Ty, ///< The type to extend to
4950 const Twine &NameStr, ///< A name for the new instruction
4951 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4952 );
4953
4954 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4955 static bool classof(const Instruction *I) {
4956 return I->getOpcode() == FPExt;
4957 }
4958 static bool classof(const Value *V) {
4959 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4960 }
4961};
4962
4963//===----------------------------------------------------------------------===//
4964// UIToFPInst Class
4965//===----------------------------------------------------------------------===//
4966
4967/// This class represents a cast unsigned integer to floating point.
4968class UIToFPInst : public CastInst {
4969protected:
4970 // Note: Instruction needs to be a friend here to call cloneImpl.
4971 friend class Instruction;
4972
4973 /// Clone an identical UIToFPInst
4974 UIToFPInst *cloneImpl() const;
4975
4976public:
4977 /// Constructor with insert-before-instruction semantics
4978 UIToFPInst(
4979 Value *S, ///< The value to be converted
4980 Type *Ty, ///< The type to convert to
4981 const Twine &NameStr = "", ///< A name for the new instruction
4982 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4983 );
4984
4985 /// Constructor with insert-at-end-of-block semantics
4986 UIToFPInst(
4987 Value *S, ///< The value to be converted
4988 Type *Ty, ///< The type to convert to
4989 const Twine &NameStr, ///< A name for the new instruction
4990 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4991 );
4992
4993 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4994 static bool classof(const Instruction *I) {
4995 return I->getOpcode() == UIToFP;
4996 }
4997 static bool classof(const Value *V) {
4998 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4999 }
5000};
5001
5002//===----------------------------------------------------------------------===//
5003// SIToFPInst Class
5004//===----------------------------------------------------------------------===//
5005
5006/// This class represents a cast from signed integer to floating point.
5007class SIToFPInst : public CastInst {
5008protected:
5009 // Note: Instruction needs to be a friend here to call cloneImpl.
5010 friend class Instruction;
5011
5012 /// Clone an identical SIToFPInst
5013 SIToFPInst *cloneImpl() const;
5014
5015public:
5016 /// Constructor with insert-before-instruction semantics
5017 SIToFPInst(
5018 Value *S, ///< The value to be converted
5019 Type *Ty, ///< The type to convert to
5020 const Twine &NameStr = "", ///< A name for the new instruction
5021 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5022 );
5023
5024 /// Constructor with insert-at-end-of-block semantics
5025 SIToFPInst(
5026 Value *S, ///< The value to be converted
5027 Type *Ty, ///< The type to convert to
5028 const Twine &NameStr, ///< A name for the new instruction
5029 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5030 );
5031
5032 /// Methods for support type inquiry through isa, cast, and dyn_cast:
5033 static bool classof(const Instruction *I) {
5034 return I->getOpcode() == SIToFP;
5035 }
5036 static bool classof(const Value *V) {
5037 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5038 }
5039};
5040
5041//===----------------------------------------------------------------------===//
5042// FPToUIInst Class
5043//===----------------------------------------------------------------------===//
5044
5045/// This class represents a cast from floating point to unsigned integer
5046class FPToUIInst : public CastInst {
5047protected:
5048 // Note: Instruction needs to be a friend here to call cloneImpl.
5049 friend class Instruction;
5050
5051 /// Clone an identical FPToUIInst
5052 FPToUIInst *cloneImpl() const;
5053
5054public:
5055 /// Constructor with insert-before-instruction semantics
5056 FPToUIInst(
5057 Value *S, ///< The value to be converted
5058 Type *Ty, ///< The type to convert to
5059 const Twine &NameStr = "", ///< A name for the new instruction
5060 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5061 );
5062
5063 /// Constructor with insert-at-end-of-block semantics
5064 FPToUIInst(
5065 Value *S, ///< The value to be converted
5066 Type *Ty, ///< The type to convert to
5067 const Twine &NameStr, ///< A name for the new instruction
5068 BasicBlock *InsertAtEnd ///< Where to insert the new instruction
5069 );
5070
5071 /// Methods for support type inquiry through isa, cast, and dyn_cast:
5072 static bool classof(const Instruction *I) {
5073 return I->getOpcode() == FPToUI;
5074 }
5075 static bool classof(const Value *V) {
5076 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5077 }
5078};
5079
5080//===----------------------------------------------------------------------===//
5081// FPToSIInst Class
5082//===----------------------------------------------------------------------===//
5083
5084/// This class represents a cast from floating point to signed integer.
5085class FPToSIInst : public CastInst {
5086protected:
5087 // Note: Instruction needs to be a friend here to call cloneImpl.
5088 friend class Instruction;
5089
5090 /// Clone an identical FPToSIInst
5091 FPToSIInst *cloneImpl() const;
5092
5093public:
5094 /// Constructor with insert-before-instruction semantics
5095 FPToSIInst(
5096 Value *S, ///< The value to be converted
5097 Type *Ty, ///< The type to convert to
5098 const Twine &NameStr = "", ///< A name for the new instruction
5099 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5100 );
5101
5102 /// Constructor with insert-at-end-of-block semantics
5103 FPToSIInst(
5104 Value *S, ///< The value to be converted
5105 Type *Ty, ///< The type to convert to
5106 const Twine &NameStr, ///< A name for the new instruction
5107 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5108 );
5109
5110 /// Methods for support type inquiry through isa, cast, and dyn_cast:
5111 static bool classof(const Instruction *I) {
5112 return I->getOpcode() == FPToSI;
5113 }
5114 static bool classof(const Value *V) {
5115 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5116 }
5117};
5118
5119//===----------------------------------------------------------------------===//
5120// IntToPtrInst Class
5121//===----------------------------------------------------------------------===//
5122
5123/// This class represents a cast from an integer to a pointer.
5124class IntToPtrInst : public CastInst {
5125public:
5126 // Note: Instruction needs to be a friend here to call cloneImpl.
5127 friend class Instruction;
5128
5129 /// Constructor with insert-before-instruction semantics
5130 IntToPtrInst(
5131 Value *S, ///< The value to be converted
5132 Type *Ty, ///< The type to convert to
5133 const Twine &NameStr = "", ///< A name for the new instruction
5134 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5135 );
5136
5137 /// Constructor with insert-at-end-of-block semantics
5138 IntToPtrInst(
5139 Value *S, ///< The value to be converted
5140 Type *Ty, ///< The type to convert to
5141 const Twine &NameStr, ///< A name for the new instruction
5142 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5143 );
5144
5145 /// Clone an identical IntToPtrInst.
5146 IntToPtrInst *cloneImpl() const;
5147
5148 /// Returns the address space of this instruction's pointer type.
5149 unsigned getAddressSpace() const {
5150 return getType()->getPointerAddressSpace();
5151 }
5152
5153 // Methods for support type inquiry through isa, cast, and dyn_cast:
5154 static bool classof(const Instruction *I) {
5155 return I->getOpcode() == IntToPtr;
5156 }
5157 static bool classof(const Value *V) {
5158 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5159 }
5160};
5161
5162//===----------------------------------------------------------------------===//
5163// PtrToIntInst Class
5164//===----------------------------------------------------------------------===//
5165
5166/// This class represents a cast from a pointer to an integer.
5167class PtrToIntInst : public CastInst {
5168protected:
5169 // Note: Instruction needs to be a friend here to call cloneImpl.
5170 friend class Instruction;
5171
5172 /// Clone an identical PtrToIntInst.
5173 PtrToIntInst *cloneImpl() const;
5174
5175public:
5176 /// Constructor with insert-before-instruction semantics
5177 PtrToIntInst(
5178 Value *S, ///< The value to be converted
5179 Type *Ty, ///< The type to convert to
5180 const Twine &NameStr = "", ///< A name for the new instruction
5181 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5182 );
5183
5184 /// Constructor with insert-at-end-of-block semantics
5185 PtrToIntInst(
5186 Value *S, ///< The value to be converted
5187 Type *Ty, ///< The type to convert to
5188 const Twine &NameStr, ///< A name for the new instruction
5189 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5190 );
5191
5192 /// Gets the pointer operand.
5193 Value *getPointerOperand() { return getOperand(0); }
5194 /// Gets the pointer operand.
5195 const Value *getPointerOperand() const { return getOperand(0); }
5196 /// Gets the operand index of the pointer operand.
5197 static unsigned getPointerOperandIndex() { return 0U; }
5198
5199 /// Returns the address space of the pointer operand.
5200 unsigned getPointerAddressSpace() const {
5201 return getPointerOperand()->getType()->getPointerAddressSpace();
5202 }
5203
5204 // Methods for support type inquiry through isa, cast, and dyn_cast:
5205 static bool classof(const Instruction *I) {
5206 return I->getOpcode() == PtrToInt;
5207 }
5208 static bool classof(const Value *V) {
5209 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5210 }
5211};
5212
5213//===----------------------------------------------------------------------===//
5214// BitCastInst Class
5215//===----------------------------------------------------------------------===//
5216
5217/// This class represents a no-op cast from one type to another.
5218class BitCastInst : public CastInst {
5219protected:
5220 // Note: Instruction needs to be a friend here to call cloneImpl.
5221 friend class Instruction;
5222
5223 /// Clone an identical BitCastInst.
5224 BitCastInst *cloneImpl() const;
5225
5226public:
5227 /// Constructor with insert-before-instruction semantics
5228 BitCastInst(
5229 Value *S, ///< The value to be casted
5230 Type *Ty, ///< The type to casted to
5231 const Twine &NameStr = "", ///< A name for the new instruction
5232 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5233 );
5234
5235 /// Constructor with insert-at-end-of-block semantics
5236 BitCastInst(
5237 Value *S, ///< The value to be casted
5238 Type *Ty, ///< The type to casted to
5239 const Twine &NameStr, ///< A name for the new instruction
5240 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5241 );
5242
5243 // Methods for support type inquiry through isa, cast, and dyn_cast:
5244 static bool classof(const Instruction *I) {
5245 return I->getOpcode() == BitCast;
5246 }
5247 static bool classof(const Value *V) {
5248 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5249 }
5250};
5251
5252//===----------------------------------------------------------------------===//
5253// AddrSpaceCastInst Class
5254//===----------------------------------------------------------------------===//
5255
5256/// This class represents a conversion between pointers from one address space
5257/// to another.
5258class AddrSpaceCastInst : public CastInst {
5259protected:
5260 // Note: Instruction needs to be a friend here to call cloneImpl.
5261 friend class Instruction;
5262
5263 /// Clone an identical AddrSpaceCastInst.
5264 AddrSpaceCastInst *cloneImpl() const;
5265
5266public:
5267 /// Constructor with insert-before-instruction semantics
5268 AddrSpaceCastInst(
5269 Value *S, ///< The value to be casted
5270 Type *Ty, ///< The type to casted to
5271 const Twine &NameStr = "", ///< A name for the new instruction
5272 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5273 );
5274
5275 /// Constructor with insert-at-end-of-block semantics
5276 AddrSpaceCastInst(
5277 Value *S, ///< The value to be casted
5278 Type *Ty, ///< The type to casted to
5279 const Twine &NameStr, ///< A name for the new instruction
5280 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5281 );
5282
5283 // Methods for support type inquiry through isa, cast, and dyn_cast:
5284 static bool classof(const Instruction *I) {
5285 return I->getOpcode() == AddrSpaceCast;
5286 }
5287 static bool classof(const Value *V) {
5288 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5289 }
5290
5291 /// Gets the pointer operand.
5292 Value *getPointerOperand() {
5293 return getOperand(0);
5294 }
5295
5296 /// Gets the pointer operand.
5297 const Value *getPointerOperand() const {
5298 return getOperand(0);
5299 }
5300
5301 /// Gets the operand index of the pointer operand.
5302 static unsigned getPointerOperandIndex() {
5303 return 0U;
5304 }
5305
5306 /// Returns the address space of the pointer operand.
5307 unsigned getSrcAddressSpace() const {
5308 return getPointerOperand()->getType()->getPointerAddressSpace();
5309 }
5310
5311 /// Returns the address space of the result.
5312 unsigned getDestAddressSpace() const {
5313 return getType()->getPointerAddressSpace();
5314 }
5315};
5316
5317/// A helper function that returns the pointer operand of a load or store
5318/// instruction. Returns nullptr if not load or store.
5319inline const Value *getLoadStorePointerOperand(const Value *V) {
5320 if (auto *Load
13.1
'Load' is null
13.1
'Load' is null
13.1
'Load' is null
13.1
'Load' is null
= dyn_cast<LoadInst>(V))
13
Assuming 'V' is not a 'LoadInst'
14
Taking false branch
5321 return Load->getPointerOperand();
5322 if (auto *Store
15.1
'Store' is non-null
15.1
'Store' is non-null
15.1
'Store' is non-null
15.1
'Store' is non-null
= dyn_cast<StoreInst>(V))
15
Assuming 'V' is a 'StoreInst'
16
Taking true branch
5323 return Store->getPointerOperand();
17
Calling 'StoreInst::getPointerOperand'
24
Returning from 'StoreInst::getPointerOperand'
25
Returning pointer, which participates in a condition later
5324 return nullptr;
5325}
5326inline Value *getLoadStorePointerOperand(Value *V) {
5327 return const_cast<Value *>(
27
Returning pointer, which participates in a condition later
5328 getLoadStorePointerOperand(static_cast<const Value *>(V)));
12
Calling 'getLoadStorePointerOperand'
26
Returning from 'getLoadStorePointerOperand'
5329}
5330
5331/// A helper function that returns the pointer operand of a load, store
5332/// or GEP instruction. Returns nullptr if not load, store, or GEP.
5333inline const Value *getPointerOperand(const Value *V) {
5334 if (auto *Ptr = getLoadStorePointerOperand(V))
5335 return Ptr;
5336 if (auto *Gep = dyn_cast<GetElementPtrInst>(V))
5337 return Gep->getPointerOperand();
5338 return nullptr;
5339}
5340inline Value *getPointerOperand(Value *V) {
5341 return const_cast<Value *>(getPointerOperand(static_cast<const Value *>(V)));
5342}
5343
5344/// A helper function that returns the alignment of load or store instruction.
5345inline Align getLoadStoreAlignment(Value *I) {
5346 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&(static_cast <bool> ((isa<LoadInst>(I) || isa<
StoreInst>(I)) && "Expected Load or Store instruction"
) ? void (0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "llvm/include/llvm/IR/Instructions.h", 5347, __extension__ __PRETTY_FUNCTION__
))
5347 "Expected Load or Store instruction")(static_cast <bool> ((isa<LoadInst>(I) || isa<
StoreInst>(I)) && "Expected Load or Store instruction"
) ? void (0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "llvm/include/llvm/IR/Instructions.h", 5347, __extension__ __PRETTY_FUNCTION__
))
;
5348 if (auto *LI = dyn_cast<LoadInst>(I))
5349 return LI->getAlign();
5350 return cast<StoreInst>(I)->getAlign();
5351}
5352
5353/// A helper function that returns the address space of the pointer operand of
5354/// load or store instruction.
5355inline unsigned getLoadStoreAddressSpace(Value *I) {
5356 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&(static_cast <bool> ((isa<LoadInst>(I) || isa<
StoreInst>(I)) && "Expected Load or Store instruction"
) ? void (0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "llvm/include/llvm/IR/Instructions.h", 5357, __extension__ __PRETTY_FUNCTION__
))
5357 "Expected Load or Store instruction")(static_cast <bool> ((isa<LoadInst>(I) || isa<
StoreInst>(I)) && "Expected Load or Store instruction"
) ? void (0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "llvm/include/llvm/IR/Instructions.h", 5357, __extension__ __PRETTY_FUNCTION__
))
;
5358 if (auto *LI = dyn_cast<LoadInst>(I))
5359 return LI->getPointerAddressSpace();
5360 return cast<StoreInst>(I)->getPointerAddressSpace();
5361}
5362
5363/// A helper function that returns the type of a load or store instruction.
5364inline Type *getLoadStoreType(Value *I) {
5365 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&(static_cast <bool> ((isa<LoadInst>(I) || isa<
StoreInst>(I)) && "Expected Load or Store instruction"
) ? void (0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "llvm/include/llvm/IR/Instructions.h", 5366, __extension__ __PRETTY_FUNCTION__
))
5366 "Expected Load or Store instruction")(static_cast <bool> ((isa<LoadInst>(I) || isa<
StoreInst>(I)) && "Expected Load or Store instruction"
) ? void (0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "llvm/include/llvm/IR/Instructions.h", 5366, __extension__ __PRETTY_FUNCTION__
))
;
5367 if (auto *LI = dyn_cast<LoadInst>(I))
5368 return LI->getType();
5369 return cast<StoreInst>(I)->getValueOperand()->getType();
5370}
5371
5372//===----------------------------------------------------------------------===//
5373// FreezeInst Class
5374//===----------------------------------------------------------------------===//
5375
5376/// This class represents a freeze function that returns random concrete
5377/// value if an operand is either a poison value or an undef value
5378class FreezeInst : public UnaryInstruction {
5379protected:
5380 // Note: Instruction needs to be a friend here to call cloneImpl.
5381 friend class Instruction;
5382
5383 /// Clone an identical FreezeInst
5384 FreezeInst *cloneImpl() const;
5385
5386public:
5387 explicit FreezeInst(Value *S,
5388 const Twine &NameStr = "",
5389 Instruction *InsertBefore = nullptr);
5390 FreezeInst(Value *S, const Twine &NameStr, BasicBlock *InsertAtEnd);
5391
5392 // Methods for support type inquiry through isa, cast, and dyn_cast:
5393 static inline bool classof(const Instruction *I) {
5394 return I->getOpcode() == Freeze;
5395 }
5396 static inline bool classof(const Value *V) {
5397 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5398 }
5399};
5400
5401} // end namespace llvm
5402
5403#endif // LLVM_IR_INSTRUCTIONS_H

/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/llvm/include/llvm/ADT/APInt.h

1//===-- llvm/ADT/APInt.h - For Arbitrary Precision Integer -----*- C++ -*--===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements a class to represent arbitrary precision
11/// integral constant values and operations on them.
12///
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_ADT_APINT_H
16#define LLVM_ADT_APINT_H
17
18#include "llvm/Support/Compiler.h"
19#include "llvm/Support/MathExtras.h"
20#include <cassert>
21#include <climits>
22#include <cstring>
23#include <utility>
24
25namespace llvm {
26class FoldingSetNodeID;
27class StringRef;
28class hash_code;
29class raw_ostream;
30
31template <typename T> class SmallVectorImpl;
32template <typename T> class ArrayRef;
33template <typename T> class Optional;
34template <typename T, typename Enable> struct DenseMapInfo;
35
36class APInt;
37
38inline APInt operator-(APInt);
39
40//===----------------------------------------------------------------------===//
41// APInt Class
42//===----------------------------------------------------------------------===//
43
44/// Class for arbitrary precision integers.
45///
46/// APInt is a functional replacement for common case unsigned integer type like
47/// "unsigned", "unsigned long" or "uint64_t", but also allows non-byte-width
48/// integer sizes and large integer value types such as 3-bits, 15-bits, or more
49/// than 64-bits of precision. APInt provides a variety of arithmetic operators
50/// and methods to manipulate integer values of any bit-width. It supports both
51/// the typical integer arithmetic and comparison operations as well as bitwise
52/// manipulation.
53///
54/// The class has several invariants worth noting:
55/// * All bit, byte, and word positions are zero-based.
56/// * Once the bit width is set, it doesn't change except by the Truncate,
57/// SignExtend, or ZeroExtend operations.
58/// * All binary operators must be on APInt instances of the same bit width.
59/// Attempting to use these operators on instances with different bit
60/// widths will yield an assertion.
61/// * The value is stored canonically as an unsigned value. For operations
62/// where it makes a difference, there are both signed and unsigned variants
63/// of the operation. For example, sdiv and udiv. However, because the bit
64/// widths must be the same, operations such as Mul and Add produce the same
65/// results regardless of whether the values are interpreted as signed or
66/// not.
67/// * In general, the class tries to follow the style of computation that LLVM
68/// uses in its IR. This simplifies its use for LLVM.
69/// * APInt supports zero-bit-width values, but operations that require bits
70/// are not defined on it (e.g. you cannot ask for the sign of a zero-bit
71/// integer). This means that operations like zero extension and logical
72/// shifts are defined, but sign extension and ashr is not. Zero bit values
73/// compare and hash equal to themselves, and countLeadingZeros returns 0.
74///
75class LLVM_NODISCARD[[clang::warn_unused_result]] APInt {
76public:
77 typedef uint64_t WordType;
78
79 /// This enum is used to hold the constants we needed for APInt.
80 enum : unsigned {
81 /// Byte size of a word.
82 APINT_WORD_SIZE = sizeof(WordType),
83 /// Bits in a word.
84 APINT_BITS_PER_WORD = APINT_WORD_SIZE * CHAR_BIT8
85 };
86
87 enum class Rounding {
88 DOWN,
89 TOWARD_ZERO,
90 UP,
91 };
92
93 static constexpr WordType WORDTYPE_MAX = ~WordType(0);
94
95 /// \name Constructors
96 /// @{
97
98 /// Create a new APInt of numBits width, initialized as val.
99 ///
100 /// If isSigned is true then val is treated as if it were a signed value
101 /// (i.e. as an int64_t) and the appropriate sign extension to the bit width
102 /// will be done. Otherwise, no sign extension occurs (high order bits beyond
103 /// the range of val are zero filled).
104 ///
105 /// \param numBits the bit width of the constructed APInt
106 /// \param val the initial value of the APInt
107 /// \param isSigned how to treat signedness of val
108 APInt(unsigned numBits, uint64_t val, bool isSigned = false)
109 : BitWidth(numBits) {
110 if (isSingleWord()) {
111 U.VAL = val;
112 clearUnusedBits();
113 } else {
114 initSlowCase(val, isSigned);
115 }
116 }
117
118 /// Construct an APInt of numBits width, initialized as bigVal[].
119 ///
120 /// Note that bigVal.size() can be smaller or larger than the corresponding
121 /// bit width but any extraneous bits will be dropped.
122 ///
123 /// \param numBits the bit width of the constructed APInt
124 /// \param bigVal a sequence of words to form the initial value of the APInt
125 APInt(unsigned numBits, ArrayRef<uint64_t> bigVal);
126
127 /// Equivalent to APInt(numBits, ArrayRef<uint64_t>(bigVal, numWords)), but
128 /// deprecated because this constructor is prone to ambiguity with the
129 /// APInt(unsigned, uint64_t, bool) constructor.
130 ///
131 /// If this overload is ever deleted, care should be taken to prevent calls
132 /// from being incorrectly captured by the APInt(unsigned, uint64_t, bool)
133 /// constructor.
134 APInt(unsigned numBits, unsigned numWords, const uint64_t bigVal[]);
135
136 /// Construct an APInt from a string representation.
137 ///
138 /// This constructor interprets the string \p str in the given radix. The
139 /// interpretation stops when the first character that is not suitable for the
140 /// radix is encountered, or the end of the string. Acceptable radix values
141 /// are 2, 8, 10, 16, and 36. It is an error for the value implied by the
142 /// string to require more bits than numBits.
143 ///
144 /// \param numBits the bit width of the constructed APInt
145 /// \param str the string to be interpreted
146 /// \param radix the radix to use for the conversion
147 APInt(unsigned numBits, StringRef str, uint8_t radix);
148
149 /// Default constructor that creates an APInt with a 1-bit zero value.
150 explicit APInt() : BitWidth(1) { U.VAL = 0; }
151
152 /// Copy Constructor.
153 APInt(const APInt &that) : BitWidth(that.BitWidth) {
154 if (isSingleWord())
155 U.VAL = that.U.VAL;
156 else
157 initSlowCase(that);
158 }
159
160 /// Move Constructor.
161 APInt(APInt &&that) : BitWidth(that.BitWidth) {
162 memcpy(&U, &that.U, sizeof(U));
163 that.BitWidth = 0;
164 }
165
166 /// Destructor.
167 ~APInt() {
168 if (needsCleanup())
169 delete[] U.pVal;
170 }
171
172 /// @}
173 /// \name Value Generators
174 /// @{
175
176 /// Get the '0' value for the specified bit-width.
177 static APInt getZero(unsigned numBits) { return APInt(numBits, 0); }
178
179 /// NOTE: This is soft-deprecated. Please use `getZero()` instead.
180 static APInt getNullValue(unsigned numBits) { return getZero(numBits); }
181
182 /// Return an APInt zero bits wide.
183 static APInt getZeroWidth() { return getZero(0); }
184
185 /// Gets maximum unsigned value of APInt for specific bit width.
186 static APInt getMaxValue(unsigned numBits) { return getAllOnes(numBits); }
187
188 /// Gets maximum signed value of APInt for a specific bit width.
189 static APInt getSignedMaxValue(unsigned numBits) {
190 APInt API = getAllOnes(numBits);
191 API.clearBit(numBits - 1);
192 return API;
193 }
194
195 /// Gets minimum unsigned value of APInt for a specific bit width.
196 static APInt getMinValue(unsigned numBits) { return APInt(numBits, 0); }
197
198 /// Gets minimum signed value of APInt for a specific bit width.
199 static APInt getSignedMinValue(unsigned numBits) {
200 APInt API(numBits, 0);
201 API.setBit(numBits - 1);
202 return API;
203 }
204
205 /// Get the SignMask for a specific bit width.
206 ///
207 /// This is just a wrapper function of getSignedMinValue(), and it helps code
208 /// readability when we want to get a SignMask.
209 static APInt getSignMask(unsigned BitWidth) {
210 return getSignedMinValue(BitWidth);
211 }
212
213 /// Return an APInt of a specified width with all bits set.
214 static APInt getAllOnes(unsigned numBits) {
215 return APInt(numBits, WORDTYPE_MAX, true);
216 }
217
218 /// NOTE: This is soft-deprecated. Please use `getAllOnes()` instead.
219 static APInt getAllOnesValue(unsigned numBits) { return getAllOnes(numBits); }
220
221 /// Return an APInt with exactly one bit set in the result.
222 static APInt getOneBitSet(unsigned numBits, unsigned BitNo) {
223 APInt Res(numBits, 0);
224 Res.setBit(BitNo);
225 return Res;
226 }
227
228 /// Get a value with a block of bits set.
229 ///
230 /// Constructs an APInt value that has a contiguous range of bits set. The
231 /// bits from loBit (inclusive) to hiBit (exclusive) will be set. All other
232 /// bits will be zero. For example, with parameters(32, 0, 16) you would get
233 /// 0x0000FFFF. Please call getBitsSetWithWrap if \p loBit may be greater than
234 /// \p hiBit.
235 ///
236 /// \param numBits the intended bit width of the result
237 /// \param loBit the index of the lowest bit set.
238 /// \param hiBit the index of the highest bit set.
239 ///
240 /// \returns An APInt value with the requested bits set.
241 static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit) {
242 APInt Res(numBits, 0);
243 Res.setBits(loBit, hiBit);
244 return Res;
245 }
246
247 /// Wrap version of getBitsSet.
248 /// If \p hiBit is bigger than \p loBit, this is same with getBitsSet.
249 /// If \p hiBit is not bigger than \p loBit, the set bits "wrap". For example,
250 /// with parameters (32, 28, 4), you would get 0xF000000F.
251 /// If \p hiBit is equal to \p loBit, you would get a result with all bits
252 /// set.
253 static APInt getBitsSetWithWrap(unsigned numBits, unsigned loBit,
254 unsigned hiBit) {
255 APInt Res(numBits, 0);
256 Res.setBitsWithWrap(loBit, hiBit);
257 return Res;
258 }
259
260 /// Constructs an APInt value that has a contiguous range of bits set. The
261 /// bits from loBit (inclusive) to numBits (exclusive) will be set. All other
262 /// bits will be zero. For example, with parameters(32, 12) you would get
263 /// 0xFFFFF000.
264 ///
265 /// \param numBits the intended bit width of the result
266 /// \param loBit the index of the lowest bit to set.
267 ///
268 /// \returns An APInt value with the requested bits set.
269 static APInt getBitsSetFrom(unsigned numBits, unsigned loBit) {
270 APInt Res(numBits, 0);
271 Res.setBitsFrom(loBit);
272 return Res;
273 }
274
275 /// Constructs an APInt value that has the top hiBitsSet bits set.
276 ///
277 /// \param numBits the bitwidth of the result
278 /// \param hiBitsSet the number of high-order bits set in the result.
279 static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet) {
280 APInt Res(numBits, 0);
281 Res.setHighBits(hiBitsSet);
282 return Res;
283 }
284
285 /// Constructs an APInt value that has the bottom loBitsSet bits set.
286 ///
287 /// \param numBits the bitwidth of the result
288 /// \param loBitsSet the number of low-order bits set in the result.
289 static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet) {
290 APInt Res(numBits, 0);
291 Res.setLowBits(loBitsSet);
292 return Res;
293 }
294
295 /// Return a value containing V broadcasted over NewLen bits.
296 static APInt getSplat(unsigned NewLen, const APInt &V);
297
298 /// @}
299 /// \name Value Tests
300 /// @{
301
302 /// Determine if this APInt just has one word to store value.
303 ///
304 /// \returns true if the number of bits <= 64, false otherwise.
305 bool isSingleWord() const { return BitWidth <= APINT_BITS_PER_WORD; }
64
Assuming field 'BitWidth' is > APINT_BITS_PER_WORD
65
Returning zero, which participates in a condition later
306
307 /// Determine sign of this APInt.
308 ///
309 /// This tests the high bit of this APInt to determine if it is set.
310 ///
311 /// \returns true if this APInt is negative, false otherwise
312 bool isNegative() const { return (*this)[BitWidth - 1]; }
47
Calling 'APInt::operator[]'
52
Returning from 'APInt::operator[]'
53
Returning zero, which participates in a condition later
313
314 /// Determine if this APInt Value is non-negative (>= 0)
315 ///
316 /// This tests the high bit of the APInt to determine if it is unset.
317 bool isNonNegative() const { return !isNegative(); }
318
319 /// Determine if sign bit of this APInt is set.
320 ///
321 /// This tests the high bit of this APInt to determine if it is set.
322 ///
323 /// \returns true if this APInt has its sign bit set, false otherwise.
324 bool isSignBitSet() const { return (*this)[BitWidth - 1]; }
325
326 /// Determine if sign bit of this APInt is clear.
327 ///
328 /// This tests the high bit of this APInt to determine if it is clear.
329 ///
330 /// \returns true if this APInt has its sign bit clear, false otherwise.
331 bool isSignBitClear() const { return !isSignBitSet(); }
332
333 /// Determine if this APInt Value is positive.
334 ///
335 /// This tests if the value of this APInt is positive (> 0). Note
336 /// that 0 is not a positive value.
337 ///
338 /// \returns true if this APInt is positive.
339 bool isStrictlyPositive() const { return isNonNegative() && !isZero(); }
340
341 /// Determine if this APInt Value is non-positive (<= 0).
342 ///
343 /// \returns true if this APInt is non-positive.
344 bool isNonPositive() const { return !isStrictlyPositive(); }
345
346 /// Determine if all bits are set. This is true for zero-width values.
347 bool isAllOnes() const {
348 if (BitWidth == 0)
349 return true;
350 if (isSingleWord())
351 return U.VAL == WORDTYPE_MAX >> (APINT_BITS_PER_WORD - BitWidth);
352 return countTrailingOnesSlowCase() == BitWidth;
353 }
354
355 /// NOTE: This is soft-deprecated. Please use `isAllOnes()` instead.
356 bool isAllOnesValue() const { return isAllOnes(); }
357
358 /// Determine if this value is zero, i.e. all bits are clear.
359 bool isZero() const {
360 if (isSingleWord())
361 return U.VAL == 0;
362 return countLeadingZerosSlowCase() == BitWidth;
363 }
364
365 /// NOTE: This is soft-deprecated. Please use `isZero()` instead.
366 bool isNullValue() const { return isZero(); }
367
368 /// Determine if this is a value of 1.
369 ///
370 /// This checks to see if the value of this APInt is one.
371 bool isOne() const {
372 if (isSingleWord())
63
Calling 'APInt::isSingleWord'
66
Returning from 'APInt::isSingleWord'
67
Taking false branch
373 return U.VAL == 1;
374 return countLeadingZerosSlowCase() == BitWidth - 1;
68
Assuming the condition is true
69
Returning the value 1, which participates in a condition later
375 }
376
377 /// NOTE: This is soft-deprecated. Please use `isOne()` instead.
378 bool isOneValue() const { return isOne(); }
379
380 /// Determine if this is the largest unsigned value.
381 ///
382 /// This checks to see if the value of this APInt is the maximum unsigned
383 /// value for the APInt's bit width.
384 bool isMaxValue() const { return isAllOnes(); }
385
386 /// Determine if this is the largest signed value.
387 ///
388 /// This checks to see if the value of this APInt is the maximum signed
389 /// value for the APInt's bit width.
390 bool isMaxSignedValue() const {
391 if (isSingleWord()) {
392 assert(BitWidth && "zero width values not allowed")(static_cast <bool> (BitWidth && "zero width values not allowed"
) ? void (0) : __assert_fail ("BitWidth && \"zero width values not allowed\""
, "llvm/include/llvm/ADT/APInt.h", 392, __extension__ __PRETTY_FUNCTION__
))
;
393 return U.VAL == ((WordType(1) << (BitWidth - 1)) - 1);
394 }
395 return !isNegative() && countTrailingOnesSlowCase() == BitWidth - 1;
396 }
397
398 /// Determine if this is the smallest unsigned value.
399 ///
400 /// This checks to see if the value of this APInt is the minimum unsigned
401 /// value for the APInt's bit width.
402 bool isMinValue() const { return isZero(); }
403
404 /// Determine if this is the smallest signed value.
405 ///
406 /// This checks to see if the value of this APInt is the minimum signed
407 /// value for the APInt's bit width.
408 bool isMinSignedValue() const {
409 if (isSingleWord()) {
410 assert(BitWidth && "zero width values not allowed")(static_cast <bool> (BitWidth && "zero width values not allowed"
) ? void (0) : __assert_fail ("BitWidth && \"zero width values not allowed\""
, "llvm/include/llvm/ADT/APInt.h", 410, __extension__ __PRETTY_FUNCTION__
))
;
411 return U.VAL == (WordType(1) << (BitWidth - 1));
412 }
413 return isNegative() && countTrailingZerosSlowCase() == BitWidth - 1;
414 }
415
416 /// Check if this APInt has an N-bits unsigned integer value.
417 bool isIntN(unsigned N) const { return getActiveBits() <= N; }
418
419 /// Check if this APInt has an N-bits signed integer value.
420 bool isSignedIntN(unsigned N) const { return getSignificantBits() <= N; }
421
422 /// Check if this APInt's value is a power of two greater than zero.
423 ///
424 /// \returns true if the argument APInt value is a power of two > 0.
425 bool isPowerOf2() const {
426 if (isSingleWord()) {
427 assert(BitWidth && "zero width values not allowed")(static_cast <bool> (BitWidth && "zero width values not allowed"
) ? void (0) : __assert_fail ("BitWidth && \"zero width values not allowed\""
, "llvm/include/llvm/ADT/APInt.h", 427, __extension__ __PRETTY_FUNCTION__
))
;
428 return isPowerOf2_64(U.VAL);
429 }
430 return countPopulationSlowCase() == 1;
431 }
432
433 /// Check if this APInt's negated value is a power of two greater than zero.
434 bool isNegatedPowerOf2() const {
435 assert(BitWidth && "zero width values not allowed")(static_cast <bool> (BitWidth && "zero width values not allowed"
) ? void (0) : __assert_fail ("BitWidth && \"zero width values not allowed\""
, "llvm/include/llvm/ADT/APInt.h", 435, __extension__ __PRETTY_FUNCTION__
))
;
436 if (isNonNegative())
437 return false;
438 // NegatedPowerOf2 - shifted mask in the top bits.
439 unsigned LO = countLeadingOnes();
440 unsigned TZ = countTrailingZeros();
441 return (LO + TZ) == BitWidth;
442 }
443
444 /// Check if the APInt's value is returned by getSignMask.
445 ///
446 /// \returns true if this is the value returned by getSignMask.
447 bool isSignMask() const { return isMinSignedValue(); }
448
449 /// Convert APInt to a boolean value.
450 ///
451 /// This converts the APInt to a boolean value as a test against zero.
452 bool getBoolValue() const { return !isZero(); }
453
454 /// If this value is smaller than the specified limit, return it, otherwise
455 /// return the limit value. This causes the value to saturate to the limit.
456 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) const {
457 return ugt(Limit) ? Limit : getZExtValue();
458 }
459
460 /// Check if the APInt consists of a repeated bit pattern.
461 ///
462 /// e.g. 0x01010101 satisfies isSplat(8).
463 /// \param SplatSizeInBits The size of the pattern in bits. Must divide bit
464 /// width without remainder.
465 bool isSplat(unsigned SplatSizeInBits) const;
466
467 /// \returns true if this APInt value is a sequence of \param numBits ones
468 /// starting at the least significant bit with the remainder zero.
469 bool isMask(unsigned numBits) const {
470 assert(numBits != 0 && "numBits must be non-zero")(static_cast <bool> (numBits != 0 && "numBits must be non-zero"
) ? void (0) : __assert_fail ("numBits != 0 && \"numBits must be non-zero\""
, "llvm/include/llvm/ADT/APInt.h", 470, __extension__ __PRETTY_FUNCTION__
))
;
471 assert(numBits <= BitWidth && "numBits out of range")(static_cast <bool> (numBits <= BitWidth && "numBits out of range"
) ? void (0) : __assert_fail ("numBits <= BitWidth && \"numBits out of range\""
, "llvm/include/llvm/ADT/APInt.h", 471, __extension__ __PRETTY_FUNCTION__
))
;
472 if (isSingleWord())
473 return U.VAL == (WORDTYPE_MAX >> (APINT_BITS_PER_WORD - numBits));
474 unsigned Ones = countTrailingOnesSlowCase();
475 return (numBits == Ones) &&
476 ((Ones + countLeadingZerosSlowCase()) == BitWidth);
477 }
478
479 /// \returns true if this APInt is a non-empty sequence of ones starting at
480 /// the least significant bit with the remainder zero.
481 /// Ex. isMask(0x0000FFFFU) == true.
482 bool isMask() const {
483 if (isSingleWord())
484 return isMask_64(U.VAL);
485 unsigned Ones = countTrailingOnesSlowCase();
486 return (Ones > 0) && ((Ones + countLeadingZerosSlowCase()) == BitWidth);
487 }
488
489 /// Return true if this APInt value contains a sequence of ones with
490 /// the remainder zero.
491 bool isShiftedMask() const {
492 if (isSingleWord())
493 return isShiftedMask_64(U.VAL);
494 unsigned Ones = countPopulationSlowCase();
495 unsigned LeadZ = countLeadingZerosSlowCase();
496 return (Ones + LeadZ + countTrailingZeros()) == BitWidth;
497 }
498
499 /// Compute an APInt containing numBits highbits from this APInt.
500 ///
501 /// Get an APInt with the same BitWidth as this APInt, just zero mask the low
502 /// bits and right shift to the least significant bit.
503 ///
504 /// \returns the high "numBits" bits of this APInt.
505 APInt getHiBits(unsigned numBits) const;
506
507 /// Compute an APInt containing numBits lowbits from this APInt.
508 ///
509 /// Get an APInt with the same BitWidth as this APInt, just zero mask the high
510 /// bits.
511 ///
512 /// \returns the low "numBits" bits of this APInt.
513 APInt getLoBits(unsigned numBits) const;
514
515 /// Determine if two APInts have the same value, after zero-extending
516 /// one of them (if needed!) to ensure that the bit-widths match.
517 static bool isSameValue(const APInt &I1, const APInt &I2) {
518 if (I1.getBitWidth() == I2.getBitWidth())
519 return I1 == I2;
520
521 if (I1.getBitWidth() > I2.getBitWidth())
522 return I1 == I2.zext(I1.getBitWidth());
523
524 return I1.zext(I2.getBitWidth()) == I2;
525 }
526
527 /// Overload to compute a hash_code for an APInt value.
528 friend hash_code hash_value(const APInt &Arg);
529
530 /// This function returns a pointer to the internal storage of the APInt.
531 /// This is useful for writing out the APInt in binary form without any
532 /// conversions.
533 const uint64_t *getRawData() const {
534 if (isSingleWord())
535 return &U.VAL;
536 return &U.pVal[0];
537 }
538
539 /// @}
540 /// \name Unary Operators
541 /// @{
542
543 /// Postfix increment operator. Increment *this by 1.
544 ///
545 /// \returns a new APInt value representing the original value of *this.
546 APInt operator++(int) {
547 APInt API(*this);
548 ++(*this);
549 return API;
550 }
551
552 /// Prefix increment operator.
553 ///
554 /// \returns *this incremented by one
555 APInt &operator++();
556
557 /// Postfix decrement operator. Decrement *this by 1.
558 ///
559 /// \returns a new APInt value representing the original value of *this.
560 APInt operator--(int) {
561 APInt API(*this);
562 --(*this);
563 return API;
564 }
565
566 /// Prefix decrement operator.
567 ///
568 /// \returns *this decremented by one.
569 APInt &operator--();
570
571 /// Logical negation operation on this APInt returns true if zero, like normal
572 /// integers.
573 bool operator!() const { return isZero(); }
574
575 /// @}
576 /// \name Assignment Operators
577 /// @{
578
579 /// Copy assignment operator.
580 ///
581 /// \returns *this after assignment of RHS.
582 APInt &operator=(const APInt &RHS) {
583 // The common case (both source or dest being inline) doesn't require
584 // allocation or deallocation.
585 if (isSingleWord() && RHS.isSingleWord()) {
586 U.VAL = RHS.U.VAL;
587 BitWidth = RHS.BitWidth;
588 return *this;
589 }
590
591 assignSlowCase(RHS);
592 return *this;
593 }
594
595 /// Move assignment operator.
596 APInt &operator=(APInt &&that) {
597#ifdef EXPENSIVE_CHECKS
598 // Some std::shuffle implementations still do self-assignment.
599 if (this == &that)
600 return *this;
601#endif
602 assert(this != &that && "Self-move not supported")(static_cast <bool> (this != &that && "Self-move not supported"
) ? void (0) : __assert_fail ("this != &that && \"Self-move not supported\""
, "llvm/include/llvm/ADT/APInt.h", 602, __extension__ __PRETTY_FUNCTION__
))
;
603 if (!isSingleWord())
604 delete[] U.pVal;
605
606 // Use memcpy so that type based alias analysis sees both VAL and pVal
607 // as modified.
608 memcpy(&U, &that.U, sizeof(U));
609
610 BitWidth = that.BitWidth;
611 that.BitWidth = 0;
612 return *this;
613 }
614
615 /// Assignment operator.
616 ///
617 /// The RHS value is assigned to *this. If the significant bits in RHS exceed
618 /// the bit width, the excess bits are truncated. If the bit width is larger
619 /// than 64, the value is zero filled in the unspecified high order bits.
620 ///
621 /// \returns *this after assignment of RHS value.
622 APInt &operator=(uint64_t RHS) {
623 if (isSingleWord()) {
624 U.VAL = RHS;
625 return clearUnusedBits();
626 }
627 U.pVal[0] = RHS;
628 memset(U.pVal + 1, 0, (getNumWords() - 1) * APINT_WORD_SIZE);
629 return *this;
630 }
631
632 /// Bitwise AND assignment operator.
633 ///
634 /// Performs a bitwise AND operation on this APInt and RHS. The result is
635 /// assigned to *this.
636 ///
637 /// \returns *this after ANDing with RHS.
638 APInt &operator&=(const APInt &RHS) {
639 assert(BitWidth == RHS.BitWidth && "Bit widths must be the same")(static_cast <bool> (BitWidth == RHS.BitWidth &&
"Bit widths must be the same") ? void (0) : __assert_fail ("BitWidth == RHS.BitWidth && \"Bit widths must be the same\""
, "llvm/include/llvm/ADT/APInt.h", 639, __extension__ __PRETTY_FUNCTION__
))
;
640 if (isSingleWord())
641 U.VAL &= RHS.U.VAL;
642 else
643 andAssignSlowCase(RHS);
644 return *this;
645 }
646
647 /// Bitwise AND assignment operator.
648 ///
649 /// Performs a bitwise AND operation on this APInt and RHS. RHS is
650 /// logically zero-extended or truncated to match the bit-width of
651 /// the LHS.
652 APInt &operator&=(uint64_t RHS) {
653 if (isSingleWord()) {
654 U.VAL &= RHS;
655 return *this;
656 }
657 U.pVal[0] &= RHS;
658 memset(U.pVal + 1, 0, (getNumWords() - 1) * APINT_WORD_SIZE);
659 return *this;
660 }
661
662 /// Bitwise OR assignment operator.
663 ///
664 /// Performs a bitwise OR operation on this APInt and RHS. The result is
665 /// assigned *this;
666 ///
667 /// \returns *this after ORing with RHS.
668 APInt &operator|=(const APInt &RHS) {
669 assert(BitWidth == RHS.BitWidth && "Bit widths must be the same")(static_cast <bool> (BitWidth == RHS.BitWidth &&
"Bit widths must be the same") ? void (0) : __assert_fail ("BitWidth == RHS.BitWidth && \"Bit widths must be the same\""
, "llvm/include/llvm/ADT/APInt.h", 669, __extension__ __PRETTY_FUNCTION__
))
;
670 if (isSingleWord())
671 U.VAL |= RHS.U.VAL;
672 else
673 orAssignSlowCase(RHS);
674 return *this;
675 }
676
677 /// Bitwise OR assignment operator.
678 ///
679 /// Performs a bitwise OR operation on this APInt and RHS. RHS is
680 /// logically zero-extended or truncated to match the bit-width of
681 /// the LHS.
682 APInt &operator|=(uint64_t RHS) {
683 if (isSingleWord()) {
684 U.VAL |= RHS;
685 return clearUnusedBits();
686 }
687 U.pVal[0] |= RHS;
688 return *this;
689 }
690
691 /// Bitwise XOR assignment operator.
692 ///
693 /// Performs a bitwise XOR operation on this APInt and RHS. The result is
694 /// assigned to *this.
695 ///
696 /// \returns *this after XORing with RHS.
697 APInt &operator^=(const APInt &RHS) {
698 assert(BitWidth == RHS.BitWidth && "Bit widths must be the same")(static_cast <bool> (BitWidth == RHS.BitWidth &&
"Bit widths must be the same") ? void (0) : __assert_fail ("BitWidth == RHS.BitWidth && \"Bit widths must be the same\""
, "llvm/include/llvm/ADT/APInt.h", 698, __extension__ __PRETTY_FUNCTION__
))
;
699 if (isSingleWord())
700 U.VAL ^= RHS.U.VAL;
701 else
702 xorAssignSlowCase(RHS);
703 return *this;
704 }
705
706 /// Bitwise XOR assignment operator.
707 ///
708 /// Performs a bitwise XOR operation on this APInt and RHS. RHS is
709 /// logically zero-extended or truncated to match the bit-width of
710 /// the LHS.
711 APInt &operator^=(uint64_t RHS) {
712 if (isSingleWord()) {
713 U.VAL ^= RHS;
714 return clearUnusedBits();
715 }
716 U.pVal[0] ^= RHS;
717 return *this;
718 }
719
720 /// Multiplication assignment operator.
721 ///
722 /// Multiplies this APInt by RHS and assigns the result to *this.
723 ///
724 /// \returns *this
725 APInt &operator*=(const APInt &RHS);
726 APInt &operator*=(uint64_t RHS);
727
728 /// Addition assignment operator.
729 ///
730 /// Adds RHS to *this and assigns the result to *this.
731 ///
732 /// \returns *this
733 APInt &operator+=(const APInt &RHS);
734 APInt &operator+=(uint64_t RHS);
735
736 /// Subtraction assignment operator.
737 ///
738 /// Subtracts RHS from *this and assigns the result to *this.
739 ///
740 /// \returns *this
741 APInt &operator-=(const APInt &RHS);
742 APInt &operator-=(uint64_t RHS);
743
744 /// Left-shift assignment function.
745 ///
746 /// Shifts *this left by shiftAmt and assigns the result to *this.
747 ///
748 /// \returns *this after shifting left by ShiftAmt
749 APInt &operator<<=(unsigned ShiftAmt) {
750 assert(ShiftAmt <= BitWidth && "Invalid shift amount")(static_cast <bool> (ShiftAmt <= BitWidth &&
"Invalid shift amount") ? void (0) : __assert_fail ("ShiftAmt <= BitWidth && \"Invalid shift amount\""
, "llvm/include/llvm/ADT/APInt.h", 750, __extension__ __PRETTY_FUNCTION__
))
;
751 if (isSingleWord()) {
752 if (ShiftAmt == BitWidth)
753 U.VAL = 0;
754 else
755 U.VAL <<= ShiftAmt;
756 return clearUnusedBits();
757 }
758 shlSlowCase(ShiftAmt);
759 return *this;
760 }
761
762 /// Left-shift assignment function.
763 ///
764 /// Shifts *this left by shiftAmt and assigns the result to *this.
765 ///
766 /// \returns *this after shifting left by ShiftAmt
767 APInt &operator<<=(const APInt &ShiftAmt);
768
769 /// @}
770 /// \name Binary Operators
771 /// @{
772
773 /// Multiplication operator.
774 ///
775 /// Multiplies this APInt by RHS and returns the result.
776 APInt operator*(const APInt &RHS) const;
777
778 /// Left logical shift operator.
779 ///
780 /// Shifts this APInt left by \p Bits and returns the result.
781 APInt operator<<(unsigned Bits) const { return shl(Bits); }
782
783 /// Left logical shift operator.
784 ///
785 /// Shifts this APInt left by \p Bits and returns the result.
786 APInt operator<<(const APInt &Bits) const { return shl(Bits); }
787
788 /// Arithmetic right-shift function.
789 ///
790 /// Arithmetic right-shift this APInt by shiftAmt.
791 APInt ashr(unsigned ShiftAmt) const {
792 APInt R(*this);
793 R.ashrInPlace(ShiftAmt);
794 return R;
795 }
796
797 /// Arithmetic right-shift this APInt by ShiftAmt in place.
798 void ashrInPlace(unsigned ShiftAmt) {
799 assert(ShiftAmt <= BitWidth && "Invalid shift amount")(static_cast <bool> (ShiftAmt <= BitWidth &&
"Invalid shift amount") ? void (0) : __assert_fail ("ShiftAmt <= BitWidth && \"Invalid shift amount\""
, "llvm/include/llvm/ADT/APInt.h", 799, __extension__ __PRETTY_FUNCTION__
))
;
800 if (isSingleWord()) {
801 int64_t SExtVAL = SignExtend64(U.VAL, BitWidth);
802 if (ShiftAmt == BitWidth)
803 U.VAL = SExtVAL >> (APINT_BITS_PER_WORD - 1); // Fill with sign bit.
804 else
805 U.VAL = SExtVAL >> ShiftAmt;
806 clearUnusedBits();
807 return;
808 }
809 ashrSlowCase(ShiftAmt);
810 }
811
812 /// Logical right-shift function.
813 ///
814 /// Logical right-shift this APInt by shiftAmt.
815 APInt lshr(unsigned shiftAmt) const {
816 APInt R(*this);
817 R.lshrInPlace(shiftAmt);
818 return R;
819 }
820
821 /// Logical right-shift this APInt by ShiftAmt in place.
822 void lshrInPlace(unsigned ShiftAmt) {
823 assert(ShiftAmt <= BitWidth && "Invalid shift amount")(static_cast <bool> (ShiftAmt <= BitWidth &&
"Invalid shift amount") ? void (0) : __assert_fail ("ShiftAmt <= BitWidth && \"Invalid shift amount\""
, "llvm/include/llvm/ADT/APInt.h", 823, __extension__ __PRETTY_FUNCTION__
))
;
824 if (isSingleWord()) {
825 if (ShiftAmt == BitWidth)
826 U.VAL = 0;
827 else
828 U.VAL >>= ShiftAmt;
829 return;
830 }
831 lshrSlowCase(ShiftAmt);
832 }
833
834 /// Left-shift function.
835 ///
836 /// Left-shift this APInt by shiftAmt.
837 APInt shl(unsigned shiftAmt) const {
838 APInt R(*this);
839 R <<= shiftAmt;
840 return R;
841 }
842
843 /// Rotate left by rotateAmt.
844 APInt rotl(unsigned rotateAmt) const;
845
846 /// Rotate right by rotateAmt.
847 APInt rotr(unsigned rotateAmt) const;
848
849 /// Arithmetic right-shift function.
850 ///
851 /// Arithmetic right-shift this APInt by shiftAmt.
852 APInt ashr(const APInt &ShiftAmt) const {
853 APInt R(*this);
854 R.ashrInPlace(ShiftAmt);
855 return R;
856 }
857
858 /// Arithmetic right-shift this APInt by shiftAmt in place.
859 void ashrInPlace(const APInt &shiftAmt);
860
861 /// Logical right-shift function.
862 ///
863 /// Logical right-shift this APInt by shiftAmt.
864 APInt lshr(const APInt &ShiftAmt) const {
865 APInt R(*this);
866 R.lshrInPlace(ShiftAmt);
867 return R;
868 }
869
870 /// Logical right-shift this APInt by ShiftAmt in place.
871 void lshrInPlace(const APInt &ShiftAmt);
872
873 /// Left-shift function.
874 ///
875 /// Left-shift this APInt by shiftAmt.
876 APInt shl(const APInt &ShiftAmt) const {
877 APInt R(*this);
878 R <<= ShiftAmt;
879 return R;
880 }
881
882 /// Rotate left by rotateAmt.
883 APInt rotl(const APInt &rotateAmt) const;
884
885 /// Rotate right by rotateAmt.
886 APInt rotr(const APInt &rotateAmt) const;
887
888 /// Concatenate the bits from "NewLSB" onto the bottom of *this. This is
889 /// equivalent to:
890 /// (this->zext(NewWidth) << NewLSB.getBitWidth()) | NewLSB.zext(NewWidth)
891 APInt concat(const APInt &NewLSB) const {
892 /// If the result will be small, then both the merged values are small.
893 unsigned NewWidth = getBitWidth() + NewLSB.getBitWidth();
894 if (NewWidth <= APINT_BITS_PER_WORD)
895 return APInt(NewWidth, (U.VAL << NewLSB.getBitWidth()) | NewLSB.U.VAL);
896 return concatSlowCase(NewLSB);
897 }
898
899 /// Unsigned division operation.
900 ///
901 /// Perform an unsigned divide operation on this APInt by RHS. Both this and
902 /// RHS are treated as unsigned quantities for purposes of this division.
903 ///
904 /// \returns a new APInt value containing the division result, rounded towards
905 /// zero.
906 APInt udiv(const APInt &RHS) const;
907 APInt udiv(uint64_t RHS) const;
908
909 /// Signed division function for APInt.
910 ///
911 /// Signed divide this APInt by APInt RHS.
912 ///
913 /// The result is rounded towards zero.
914 APInt sdiv(const APInt &RHS) const;
915 APInt sdiv(int64_t RHS) const;
916
917 /// Unsigned remainder operation.
918 ///
919 /// Perform an unsigned remainder operation on this APInt with RHS being the
920 /// divisor. Both this and RHS are treated as unsigned quantities for purposes
921 /// of this operation. Note that this is a true remainder operation and not a
922 /// modulo operation because the sign follows the sign of the dividend which
923 /// is *this.
924 ///
925 /// \returns a new APInt value containing the remainder result
926 APInt urem(const APInt &RHS) const;
927 uint64_t urem(uint64_t RHS) const;
928
929 /// Function for signed remainder operation.
930 ///
931 /// Signed remainder operation on APInt.
932 APInt srem(const APInt &RHS) const;
933 int64_t srem(int64_t RHS) const;
934
935 /// Dual division/remainder interface.
936 ///
937 /// Sometimes it is convenient to divide two APInt values and obtain both the
938 /// quotient and remainder. This function does both operations in the same
939 /// computation making it a little more efficient. The pair of input arguments
940 /// may overlap with the pair of output arguments. It is safe to call
941 /// udivrem(X, Y, X, Y), for example.
942 static void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient,
943 APInt &Remainder);
944 static void udivrem(const APInt &LHS, uint64_t RHS, APInt &Quotient,
945 uint64_t &Remainder);
946
947 static void sdivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient,
948 APInt &Remainder);
949 static void sdivrem(const APInt &LHS, int64_t RHS, APInt &Quotient,
950 int64_t &Remainder);
951
952 // Operations that return overflow indicators.
953 APInt sadd_ov(const APInt &RHS, bool &Overflow) const;
954 APInt uadd_ov(const APInt &RHS, bool &Overflow) const;
955 APInt ssub_ov(const APInt &RHS, bool &Overflow) const;
956 APInt usub_ov(const APInt &RHS, bool &Overflow) const;
957 APInt sdiv_ov(const APInt &RHS, bool &Overflow) const;
958 APInt smul_ov(const APInt &RHS, bool &Overflow) const;
959 APInt umul_ov(const APInt &RHS, bool &Overflow) const;
960 APInt sshl_ov(const APInt &Amt, bool &Overflow) const;
961 APInt ushl_ov(const APInt &Amt, bool &Overflow) const;
962
963 // Operations that saturate
964 APInt sadd_sat(const APInt &RHS) const;
965 APInt uadd_sat(const APInt &RHS) const;
966 APInt ssub_sat(const APInt &RHS) const;
967 APInt usub_sat(const APInt &RHS) const;
968 APInt smul_sat(const APInt &RHS) const;
969 APInt umul_sat(const APInt &RHS) const;
970 APInt sshl_sat(const APInt &RHS) const;
971 APInt ushl_sat(const APInt &RHS) const;
972
973 /// Array-indexing support.
974 ///
975 /// \returns the bit value at bitPosition
976 bool operator[](unsigned bitPosition) const {
977 assert(bitPosition < getBitWidth() && "Bit position out of bounds!")(static_cast <bool> (bitPosition < getBitWidth() &&
"Bit position out of bounds!") ? void (0) : __assert_fail ("bitPosition < getBitWidth() && \"Bit position out of bounds!\""
, "llvm/include/llvm/ADT/APInt.h", 977, __extension__ __PRETTY_FUNCTION__
))
;
48
Assuming the condition is true
49
'?' condition is true
978 return (maskBit(bitPosition) & getWord(bitPosition)) != 0;
50
Assuming the condition is false
51
Returning zero, which participates in a condition later
979 }
980
981 /// @}
982 /// \name Comparison Operators
983 /// @{
984
985 /// Equality operator.
986 ///
987 /// Compares this APInt with RHS for the validity of the equality
988 /// relationship.
989 bool operator==(const APInt &RHS) const {
990 assert(BitWidth == RHS.BitWidth && "Comparison requires equal bit widths")(static_cast <bool> (BitWidth == RHS.BitWidth &&
"Comparison requires equal bit widths") ? void (0) : __assert_fail
("BitWidth == RHS.BitWidth && \"Comparison requires equal bit widths\""
, "llvm/include/llvm/ADT/APInt.h", 990, __extension__ __PRETTY_FUNCTION__
))
;
991 if (isSingleWord())
992 return U.VAL == RHS.U.VAL;
993 return equalSlowCase(RHS);
994 }
995
996 /// Equality operator.
997 ///
998 /// Compares this APInt with a uint64_t for the validity of the equality
999 /// relationship.
1000 ///
1001 /// \returns true if *this == Val
1002 bool operator==(uint64_t Val) const {
1003 return (isSingleWord() || getActiveBits() <= 64) && getZExtValue() == Val;
1004 }
1005
1006 /// Equality comparison.
1007 ///
1008 /// Compares this APInt with RHS for the validity of the equality
1009 /// relationship.
1010 ///
1011 /// \returns true if *this == Val
1012 bool eq(const APInt &RHS) const { return (*this) == RHS; }
1013
1014 /// Inequality operator.
1015 ///
1016 /// Compares this APInt with RHS for the validity of the inequality
1017 /// relationship.
1018 ///
1019 /// \returns true if *this != Val
1020 bool operator!=(const APInt &RHS) const { return !((*this) == RHS); }
1021
1022 /// Inequality operator.
1023 ///
1024 /// Compares this APInt with a uint64_t for the validity of the inequality
1025 /// relationship.
1026 ///
1027 /// \returns true if *this != Val
1028 bool operator!=(uint64_t Val) const { return !((*this) == Val); }
1029
1030 /// Inequality comparison
1031 ///
1032 /// Compares this APInt with RHS for the validity of the inequality
1033 /// relationship.
1034 ///
1035 /// \returns true if *this != Val
1036 bool ne(const APInt &RHS) const { return !((*this) == RHS); }
1037
1038 /// Unsigned less than comparison
1039 ///
1040 /// Regards both *this and RHS as unsigned quantities and compares them for
1041 /// the validity of the less-than relationship.
1042 ///
1043 /// \returns true if *this < RHS when both are considered unsigned.
1044 bool ult(const APInt &RHS) const { return compare(RHS) < 0; }
1045
1046 /// Unsigned less than comparison
1047 ///
1048 /// Regards both *this as an unsigned quantity and compares it with RHS for
1049 /// the validity of the less-than relationship.
1050 ///
1051 /// \returns true if *this < RHS when considered unsigned.
1052 bool ult(uint64_t RHS) const {
1053 // Only need to check active bits if not a single word.
1054 return (isSingleWord() || getActiveBits() <= 64) && getZExtValue() < RHS;
1055 }
1056
1057 /// Signed less than comparison
1058 ///
1059 /// Regards both *this and RHS as signed quantities and compares them for
1060 /// validity of the less-than relationship.
1061 ///
1062 /// \returns true if *this < RHS when both are considered signed.
1063 bool slt(const APInt &RHS) const { return compareSigned(RHS) < 0; }
1064
1065 /// Signed less than comparison
1066 ///
1067 /// Regards both *this as a signed quantity and compares it with RHS for
1068 /// the validity of the less-than relationship.
1069 ///
1070 /// \returns true if *this < RHS when considered signed.
1071 bool slt(int64_t RHS) const {
1072 return (!isSingleWord() && getSignificantBits() > 64)
1073 ? isNegative()
1074 : getSExtValue() < RHS;
1075 }
1076
1077 /// Unsigned less or equal comparison
1078 ///
1079 /// Regards both *this and RHS as unsigned quantities and compares them for
1080 /// validity of the less-or-equal relationship.
1081 ///
1082 /// \returns true if *this <= RHS when both are considered unsigned.
1083 bool ule(const APInt &RHS) const { return compare(RHS) <= 0; }
1084
1085 /// Unsigned less or equal comparison
1086 ///
1087 /// Regards both *this as an unsigned quantity and compares it with RHS for
1088 /// the validity of the less-or-equal relationship.
1089 ///
1090 /// \returns true if *this <= RHS when considered unsigned.
1091 bool ule(uint64_t RHS) const { return !ugt(RHS); }
1092
1093 /// Signed less or equal comparison
1094 ///
1095 /// Regards both *this and RHS as signed quantities and compares them for
1096 /// validity of the less-or-equal relationship.
1097 ///
1098 /// \returns true if *this <= RHS when both are considered signed.
1099 bool sle(const APInt &RHS) const { return compareSigned(RHS) <= 0; }
1100
1101 /// Signed less or equal comparison
1102 ///
1103 /// Regards both *this as a signed quantity and compares it with RHS for the
1104 /// validity of the less-or-equal relationship.
1105 ///
1106 /// \returns true if *this <= RHS when considered signed.
1107 bool sle(uint64_t RHS) const { return !sgt(RHS); }
1108
1109 /// Unsigned greater than comparison
1110 ///
1111 /// Regards both *this and RHS as unsigned quantities and compares them for
1112 /// the validity of the greater-than relationship.
1113 ///
1114 /// \returns true if *this > RHS when both are considered unsigned.
1115 bool ugt(const APInt &RHS) const { return !ule(RHS); }
1116
1117 /// Unsigned greater than comparison
1118 ///
1119 /// Regards both *this as an unsigned quantity and compares it with RHS for
1120 /// the validity of the greater-than relationship.
1121 ///
1122 /// \returns true if *this > RHS when considered unsigned.
1123 bool ugt(uint64_t RHS) const {
1124 // Only need to check active bits if not a single word.
1125 return (!isSingleWord() && getActiveBits() > 64) || getZExtValue() > RHS;
1126 }
1127
1128 /// Signed greater than comparison
1129 ///
1130 /// Regards both *this and RHS as signed quantities and compares them for the
1131 /// validity of the greater-than relationship.
1132 ///
1133 /// \returns true if *this > RHS when both are considered signed.
1134 bool sgt(const APInt &RHS) const { return !sle(RHS); }
1135
1136 /// Signed greater than comparison
1137 ///
1138 /// Regards both *this as a signed quantity and compares it with RHS for
1139 /// the validity of the greater-than relationship.
1140 ///
1141 /// \returns true if *this > RHS when considered signed.
1142 bool sgt(int64_t RHS) const {
1143 return (!isSingleWord() && getSignificantBits() > 64)
1144 ? !isNegative()
1145 : getSExtValue() > RHS;
1146 }
1147
1148 /// Unsigned greater or equal comparison
1149 ///
1150 /// Regards both *this and RHS as unsigned quantities and compares them for
1151 /// validity of the greater-or-equal relationship.
1152 ///
1153 /// \returns true if *this >= RHS when both are considered unsigned.
1154 bool uge(const APInt &RHS) const { return !ult(RHS); }
1155
1156 /// Unsigned greater or equal comparison
1157 ///
1158 /// Regards both *this as an unsigned quantity and compares it with RHS for
1159 /// the validity of the greater-or-equal relationship.
1160 ///
1161 /// \returns true if *this >= RHS when considered unsigned.
1162 bool uge(uint64_t RHS) const { return !ult(RHS); }
1163
1164 /// Signed greater or equal comparison
1165 ///
1166 /// Regards both *this and RHS as signed quantities and compares them for
1167 /// validity of the greater-or-equal relationship.
1168 ///
1169 /// \returns true if *this >= RHS when both are considered signed.
1170 bool sge(const APInt &RHS) const { return !slt(RHS); }
1171
1172 /// Signed greater or equal comparison
1173 ///
1174 /// Regards both *this as a signed quantity and compares it with RHS for
1175 /// the validity of the greater-or-equal relationship.
1176 ///
1177 /// \returns true if *this >= RHS when considered signed.
1178 bool sge(int64_t RHS) const { return !slt(RHS); }
1179
1180 /// This operation tests if there are any pairs of corresponding bits
1181 /// between this APInt and RHS that are both set.
1182 bool intersects(const APInt &RHS) const {
1183 assert(BitWidth == RHS.BitWidth && "Bit widths must be the same")(static_cast <bool> (BitWidth == RHS.BitWidth &&
"Bit widths must be the same") ? void (0) : __assert_fail ("BitWidth == RHS.BitWidth && \"Bit widths must be the same\""
, "llvm/include/llvm/ADT/APInt.h", 1183, __extension__ __PRETTY_FUNCTION__
))
;
1184 if (isSingleWord())
1185 return (U.VAL & RHS.U.VAL) != 0;
1186 return intersectsSlowCase(RHS);
1187 }
1188
1189 /// This operation checks that all bits set in this APInt are also set in RHS.
1190 bool isSubsetOf(const APInt &RHS) const {
1191 assert(BitWidth == RHS.BitWidth && "Bit widths must be the same")(static_cast <bool> (BitWidth == RHS.BitWidth &&
"Bit widths must be the same") ? void (0) : __assert_fail ("BitWidth == RHS.BitWidth && \"Bit widths must be the same\""
, "llvm/include/llvm/ADT/APInt.h", 1191, __extension__ __PRETTY_FUNCTION__
))
;
1192 if (isSingleWord())
1193 return (U.VAL & ~RHS.U.VAL) == 0;
1194 return isSubsetOfSlowCase(RHS);
1195 }
1196
1197 /// @}
1198 /// \name Resizing Operators
1199 /// @{
1200
1201 /// Truncate to new width.
1202 ///
1203 /// Truncate the APInt to a specified width. It is an error to specify a width
1204 /// that is greater than or equal to the current width.
1205 APInt trunc(unsigned width) const;
1206
1207 /// Truncate to new width with unsigned saturation.
1208 ///
1209 /// If the APInt, treated as unsigned integer, can be losslessly truncated to
1210 /// the new bitwidth, then return truncated APInt. Else, return max value.
1211 APInt truncUSat(unsigned width) const;
1212
1213 /// Truncate to new width with signed saturation.
1214 ///
1215 /// If this APInt, treated as signed integer, can be losslessly truncated to
1216 /// the new bitwidth, then return truncated APInt. Else, return either
1217 /// signed min value if the APInt was negative, or signed max value.
1218 APInt truncSSat(unsigned width) const;
1219
1220 /// Sign extend to a new width.
1221 ///
1222 /// This operation sign extends the APInt to a new width. If the high order
1223 /// bit is set, the fill on the left will be done with 1 bits, otherwise zero.
1224 /// It is an error to specify a width that is less than or equal to the
1225 /// current width.
1226 APInt sext(unsigned width) const;
1227
1228 /// Zero extend to a new width.
1229 ///
1230 /// This operation zero extends the APInt to a new width. The high order bits
1231 /// are filled with 0 bits. It is an error to specify a width that is less
1232 /// than or equal to the current width.
1233 APInt zext(unsigned width) const;
1234
1235 /// Sign extend or truncate to width
1236 ///
1237 /// Make this APInt have the bit width given by \p width. The value is sign
1238 /// extended, truncated, or left alone to make it that width.
1239 APInt sextOrTrunc(unsigned width) const;
1240
1241 /// Zero extend or truncate to width
1242 ///
1243 /// Make this APInt have the bit width given by \p width. The value is zero
1244 /// extended, truncated, or left alone to make it that width.
1245 APInt zextOrTrunc(unsigned width) const;
1246
1247 /// Truncate to width
1248 ///
1249 /// Make this APInt have the bit width given by \p width. The value is
1250 /// truncated or left alone to make it that width.
1251 APInt truncOrSelf(unsigned width) const;
1252
1253 /// Sign extend or truncate to width
1254 ///
1255 /// Make this APInt have the bit width given by \p width. The value is sign
1256 /// extended, or left alone to make it that width.
1257 APInt sextOrSelf(unsigned width) const;
1258
1259 /// Zero extend or truncate to width
1260 ///
1261 /// Make this APInt have the bit width given by \p width. The value is zero
1262 /// extended, or left alone to make it that width.
1263 APInt zextOrSelf(unsigned width) const;
1264
1265 /// @}
1266 /// \name Bit Manipulation Operators
1267 /// @{
1268
1269 /// Set every bit to 1.
1270 void setAllBits() {
1271 if (isSingleWord())
1272 U.VAL = WORDTYPE_MAX;
1273 else
1274 // Set all the bits in all the words.
1275 memset(U.pVal, -1, getNumWords() * APINT_WORD_SIZE);
1276 // Clear the unused ones
1277 clearUnusedBits();
1278 }
1279
1280 /// Set the given bit to 1 whose position is given as "bitPosition".
1281 void setBit(unsigned BitPosition) {
1282 assert(BitPosition < BitWidth && "BitPosition out of range")(static_cast <bool> (BitPosition < BitWidth &&
"BitPosition out of range") ? void (0) : __assert_fail ("BitPosition < BitWidth && \"BitPosition out of range\""
, "llvm/include/llvm/ADT/APInt.h", 1282, __extension__ __PRETTY_FUNCTION__
))
;
1283 WordType Mask = maskBit(BitPosition);
1284 if (isSingleWord())
1285 U.VAL |= Mask;
1286 else
1287 U.pVal[whichWord(BitPosition)] |= Mask;
1288 }
1289
1290 /// Set the sign bit to 1.
1291 void setSignBit() { setBit(BitWidth - 1); }
1292
1293 /// Set a given bit to a given value.
1294 void setBitVal(unsigned BitPosition, bool BitValue) {
1295 if (BitValue)
1296 setBit(BitPosition);
1297 else
1298 clearBit(BitPosition);
1299 }
1300
1301 /// Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
1302 /// This function handles "wrap" case when \p loBit >= \p hiBit, and calls
1303 /// setBits when \p loBit < \p hiBit.
1304 /// For \p loBit == \p hiBit wrap case, set every bit to 1.
1305 void setBitsWithWrap(unsigned loBit, unsigned hiBit) {
1306 assert(hiBit <= BitWidth && "hiBit out of range")(static_cast <bool> (hiBit <= BitWidth && "hiBit out of range"
) ? void (0) : __assert_fail ("hiBit <= BitWidth && \"hiBit out of range\""
, "llvm/include/llvm/ADT/APInt.h", 1306, __extension__ __PRETTY_FUNCTION__
))
;
1307 assert(loBit <= BitWidth && "loBit out of range")(static_cast <bool> (loBit <= BitWidth && "loBit out of range"
) ? void (0) : __assert_fail ("loBit <= BitWidth && \"loBit out of range\""
, "llvm/include/llvm/ADT/APInt.h", 1307, __extension__ __PRETTY_FUNCTION__
))
;
1308 if (loBit < hiBit) {
1309 setBits(loBit, hiBit);
1310 return;
1311 }
1312 setLowBits(hiBit);
1313 setHighBits(BitWidth - loBit);
1314 }
1315
1316 /// Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
1317 /// This function handles case when \p loBit <= \p hiBit.
1318 void setBits(unsigned loBit, unsigned hiBit) {
1319 assert(hiBit <= BitWidth && "hiBit out of range")(static_cast <bool> (hiBit <= BitWidth && "hiBit out of range"
) ? void (0) : __assert_fail ("hiBit <= BitWidth && \"hiBit out of range\""
, "llvm/include/llvm/ADT/APInt.h", 1319, __extension__ __PRETTY_FUNCTION__
))
;
1320 assert(loBit <= BitWidth && "loBit out of range")(static_cast <bool> (loBit <= BitWidth && "loBit out of range"
) ? void (0) : __assert_fail ("loBit <= BitWidth && \"loBit out of range\""
, "llvm/include/llvm/ADT/APInt.h", 1320, __extension__ __PRETTY_FUNCTION__
))
;
1321 assert(loBit <= hiBit && "loBit greater than hiBit")(static_cast <bool> (loBit <= hiBit && "loBit greater than hiBit"
) ? void (0) : __assert_fail ("loBit <= hiBit && \"loBit greater than hiBit\""
, "llvm/include/llvm/ADT/APInt.h", 1321, __extension__ __PRETTY_FUNCTION__
))
;
1322 if (loBit == hiBit)
1323 return;
1324 if (loBit < APINT_BITS_PER_WORD && hiBit <= APINT_BITS_PER_WORD) {
1325 uint64_t mask = WORDTYPE_MAX >> (APINT_BITS_PER_WORD - (hiBit - loBit));
1326 mask <<= loBit;
1327 if (isSingleWord())
1328 U.VAL |= mask;
1329 else
1330 U.pVal[0] |= mask;
1331 } else {
1332 setBitsSlowCase(loBit, hiBit);
1333 }
1334 }
1335
1336 /// Set the top bits starting from loBit.
1337 void setBitsFrom(unsigned loBit) { return setBits(loBit, BitWidth); }
1338
1339 /// Set the bottom loBits bits.
1340 void setLowBits(unsigned loBits) { return setBits(0, loBits); }
1341
1342 /// Set the top hiBits bits.
1343 void setHighBits(unsigned hiBits) {
1344 return setBits(BitWidth - hiBits, BitWidth);
1345 }
1346
1347 /// Set every bit to 0.
1348 void clearAllBits() {
1349 if (isSingleWord())
1350 U.VAL = 0;
1351 else
1352 memset(U.pVal, 0, getNumWords() * APINT_WORD_SIZE);
1353 }
1354
1355 /// Set a given bit to 0.
1356 ///
1357 /// Set the given bit to 0 whose position is given as "bitPosition".
1358 void clearBit(unsigned BitPosition) {
1359 assert(BitPosition < BitWidth && "BitPosition out of range")(static_cast <bool> (BitPosition < BitWidth &&
"BitPosition out of range") ? void (0) : __assert_fail ("BitPosition < BitWidth && \"BitPosition out of range\""
, "llvm/include/llvm/ADT/APInt.h", 1359, __extension__ __PRETTY_FUNCTION__
))
;
1360 WordType Mask = ~maskBit(BitPosition);
1361 if (isSingleWord())
1362 U.VAL &= Mask;
1363 else
1364 U.pVal[whichWord(BitPosition)] &= Mask;
1365 }
1366
1367 /// Set bottom loBits bits to 0.
1368 void clearLowBits(unsigned loBits) {
1369 assert(loBits <= BitWidth && "More bits than bitwidth")(static_cast <bool> (loBits <= BitWidth && "More bits than bitwidth"
) ? void (0) : __assert_fail ("loBits <= BitWidth && \"More bits than bitwidth\""
, "llvm/include/llvm/ADT/APInt.h", 1369, __extension__ __PRETTY_FUNCTION__
))
;
1370 APInt Keep = getHighBitsSet(BitWidth, BitWidth - loBits);
1371 *this &= Keep;
1372 }
1373
1374 /// Set the sign bit to 0.
1375 void clearSignBit() { clearBit(BitWidth - 1); }
1376
1377 /// Toggle every bit to its opposite value.
1378 void flipAllBits() {
1379 if (isSingleWord()) {
1380 U.VAL ^= WORDTYPE_MAX;
1381 clearUnusedBits();
1382 } else {
1383 flipAllBitsSlowCase();
1384 }
1385 }
1386
1387 /// Toggles a given bit to its opposite value.
1388 ///
1389 /// Toggle a given bit to its opposite value whose position is given
1390 /// as "bitPosition".
1391 void flipBit(unsigned bitPosition);
1392
1393 /// Negate this APInt in place.
1394 void negate() {
1395 flipAllBits();
1396 ++(*this);
1397 }
1398
1399 /// Insert the bits from a smaller APInt starting at bitPosition.
1400 void insertBits(const APInt &SubBits, unsigned bitPosition);
1401 void insertBits(uint64_t SubBits, unsigned bitPosition, unsigned numBits);
1402
1403 /// Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
1404 APInt extractBits(unsigned numBits, unsigned bitPosition) const;
1405 uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const;
1406
1407 /// @}
1408 /// \name Value Characterization Functions
1409 /// @{
1410
1411 /// Return the number of bits in the APInt.
1412 unsigned getBitWidth() const { return BitWidth; }
1413
1414 /// Get the number of words.
1415 ///
1416 /// Here one word's bitwidth equals to that of uint64_t.
1417 ///
1418 /// \returns the number of words to hold the integer value of this APInt.
1419 unsigned getNumWords() const { return getNumWords(BitWidth); }
1420
1421 /// Get the number of words.
1422 ///
1423 /// *NOTE* Here one word's bitwidth equals to that of uint64_t.
1424 ///
1425 /// \returns the number of words to hold the integer value with a given bit
1426 /// width.
1427 static unsigned getNumWords(unsigned BitWidth) {
1428 return ((uint64_t)BitWidth + APINT_BITS_PER_WORD - 1) / APINT_BITS_PER_WORD;
1429 }
1430
1431 /// Compute the number of active bits in the value
1432 ///
1433 /// This function returns the number of active bits which is defined as the
1434 /// bit width minus the number of leading zeros. This is used in several
1435 /// computations to see how "wide" the value is.
1436 unsigned getActiveBits() const { return BitWidth - countLeadingZeros(); }
1437
1438 /// Compute the number of active words in the value of this APInt.
1439 ///
1440 /// This is used in conjunction with getActiveData to extract the raw value of
1441 /// the APInt.
1442 unsigned getActiveWords() const {
1443 unsigned numActiveBits = getActiveBits();
1444 return numActiveBits ? whichWord(numActiveBits - 1) + 1 : 1;
1445 }
1446
1447 /// Get the minimum bit size for this signed APInt
1448 ///
1449 /// Computes the minimum bit width for this APInt while considering it to be a
1450 /// signed (and probably negative) value. If the value is not negative, this
1451 /// function returns the same value as getActiveBits()+1. Otherwise, it
1452 /// returns the smallest bit width that will retain the negative value. For
1453 /// example, -1 can be written as 0b1 or 0xFFFFFFFFFF. 0b1 is shorter and so
1454 /// for -1, this function will always return 1.
1455 unsigned getSignificantBits() const {
1456 return BitWidth - getNumSignBits() + 1;
1457 }
1458
1459 /// NOTE: This is soft-deprecated. Please use `getSignificantBits()` instead.
1460 unsigned getMinSignedBits() const { return getSignificantBits(); }
1461
1462 /// Get zero extended value
1463 ///
1464 /// This method attempts to return the value of this APInt as a zero extended
1465 /// uint64_t. The bitwidth must be <= 64 or the value must fit within a
1466 /// uint64_t. Otherwise an assertion will result.
1467 uint64_t getZExtValue() const {
1468 if (isSingleWord())
1469 return U.VAL;
1470 assert(getActiveBits() <= 64 && "Too many bits for uint64_t")(static_cast <bool> (getActiveBits() <= 64 &&
"Too many bits for uint64_t") ? void (0) : __assert_fail ("getActiveBits() <= 64 && \"Too many bits for uint64_t\""
, "llvm/include/llvm/ADT/APInt.h", 1470, __extension__ __PRETTY_FUNCTION__
))
;
1471 return U.pVal[0];
1472 }
1473
1474 /// Get sign extended value
1475 ///
1476 /// This method attempts to return the value of this APInt as a sign extended
1477 /// int64_t. The bit width must be <= 64 or the value must fit within an
1478 /// int64_t. Otherwise an assertion will result.
1479 int64_t getSExtValue() const {
1480 if (isSingleWord())
1481 return SignExtend64(U.VAL, BitWidth);
1482 assert(getSignificantBits() <= 64 && "Too many bits for int64_t")(static_cast <bool> (getSignificantBits() <= 64 &&
"Too many bits for int64_t") ? void (0) : __assert_fail ("getSignificantBits() <= 64 && \"Too many bits for int64_t\""
, "llvm/include/llvm/ADT/APInt.h", 1482, __extension__ __PRETTY_FUNCTION__
))
;
1483 return int64_t(U.pVal[0]);
1484 }
1485
1486 /// Get bits required for string value.
1487 ///
1488 /// This method determines how many bits are required to hold the APInt
1489 /// equivalent of the string given by \p str.
1490 static unsigned getBitsNeeded(StringRef str, uint8_t radix);
1491
1492 /// The APInt version of the countLeadingZeros functions in
1493 /// MathExtras.h.
1494 ///
1495 /// It counts the number of zeros from the most significant bit to the first
1496 /// one bit.
1497 ///
1498 /// \returns BitWidth if the value is zero, otherwise returns the number of
1499 /// zeros from the most significant bit to the first one bits.
1500 unsigned countLeadingZeros() const {
1501 if (isSingleWord()) {
1502 unsigned unusedBits = APINT_BITS_PER_WORD - BitWidth;
1503 return llvm::countLeadingZeros(U.VAL) - unusedBits;
1504 }
1505 return countLeadingZerosSlowCase();
1506 }
1507
1508 /// Count the number of leading one bits.
1509 ///
1510 /// This function is an APInt version of the countLeadingOnes
1511 /// functions in MathExtras.h. It counts the number of ones from the most
1512 /// significant bit to the first zero bit.
1513 ///
1514 /// \returns 0 if the high order bit is not set, otherwise returns the number
1515 /// of 1 bits from the most significant to the least
1516 unsigned countLeadingOnes() const {
1517 if (isSingleWord()) {
1518 if (LLVM_UNLIKELY(BitWidth == 0)__builtin_expect((bool)(BitWidth == 0), false))
1519 return 0;
1520 return llvm::countLeadingOnes(U.VAL << (APINT_BITS_PER_WORD - BitWidth));
1521 }
1522 return countLeadingOnesSlowCase();
1523 }
1524
1525 /// Computes the number of leading bits of this APInt that are equal to its
1526 /// sign bit.
1527 unsigned getNumSignBits() const {
1528 return isNegative() ? countLeadingOnes() : countLeadingZeros();
1529 }
1530
1531 /// Count the number of trailing zero bits.
1532 ///
1533 /// This function is an APInt version of the countTrailingZeros
1534 /// functions in MathExtras.h. It counts the number of zeros from the least
1535 /// significant bit to the first set bit.
1536 ///
1537 /// \returns BitWidth if the value is zero, otherwise returns the number of
1538 /// zeros from the least significant bit to the first one bit.
1539 unsigned countTrailingZeros() const {
1540 if (isSingleWord()) {
1541 unsigned TrailingZeros = llvm::countTrailingZeros(U.VAL);
1542 return (TrailingZeros > BitWidth ? BitWidth : TrailingZeros);
1543 }
1544 return countTrailingZerosSlowCase();
1545 }
1546
1547 /// Count the number of trailing one bits.
1548 ///
1549 /// This function is an APInt version of the countTrailingOnes
1550 /// functions in MathExtras.h. It counts the number of ones from the least
1551 /// significant bit to the first zero bit.
1552 ///
1553 /// \returns BitWidth if the value is all ones, otherwise returns the number
1554 /// of ones from the least significant bit to the first zero bit.
1555 unsigned countTrailingOnes() const {
1556 if (isSingleWord())
1557 return llvm::countTrailingOnes(U.VAL);
1558 return countTrailingOnesSlowCase();
1559 }
1560
1561 /// Count the number of bits set.
1562 ///
1563 /// This function is an APInt version of the countPopulation functions
1564 /// in MathExtras.h. It counts the number of 1 bits in the APInt value.
1565 ///
1566 /// \returns 0 if the value is zero, otherwise returns the number of set bits.
1567 unsigned countPopulation() const {
1568 if (isSingleWord())
1569 return llvm::countPopulation(U.VAL);
1570 return countPopulationSlowCase();
1571 }
1572
1573 /// @}
1574 /// \name Conversion Functions
1575 /// @{
1576 void print(raw_ostream &OS, bool isSigned) const;
1577
1578 /// Converts an APInt to a string and append it to Str. Str is commonly a
1579 /// SmallString.
1580 void toString(SmallVectorImpl<char> &Str, unsigned Radix, bool Signed,
1581 bool formatAsCLiteral = false) const;
1582
1583 /// Considers the APInt to be unsigned and converts it into a string in the
1584 /// radix given. The radix can be 2, 8, 10 16, or 36.
1585 void toStringUnsigned(SmallVectorImpl<char> &Str, unsigned Radix = 10) const {
1586 toString(Str, Radix, false, false);
1587 }
1588
1589 /// Considers the APInt to be signed and converts it into a string in the
1590 /// radix given. The radix can be 2, 8, 10, 16, or 36.
1591 void toStringSigned(SmallVectorImpl<char> &Str, unsigned Radix = 10) const {
1592 toString(Str, Radix, true, false);
1593 }
1594
1595 /// \returns a byte-swapped representation of this APInt Value.
1596 APInt byteSwap() const;
1597
1598 /// \returns the value with the bit representation reversed of this APInt
1599 /// Value.
1600 APInt reverseBits() const;
1601
1602 /// Converts this APInt to a double value.
1603 double roundToDouble(bool isSigned) const;
1604
1605 /// Converts this unsigned APInt to a double value.
1606 double roundToDouble() const { return roundToDouble(false); }
1607
1608 /// Converts this signed APInt to a double value.
1609 double signedRoundToDouble() const { return roundToDouble(true); }
1610
1611 /// Converts APInt bits to a double
1612 ///
1613 /// The conversion does not do a translation from integer to double, it just
1614 /// re-interprets the bits as a double. Note that it is valid to do this on
1615 /// any bit width. Exactly 64 bits will be translated.
1616 double bitsToDouble() const { return BitsToDouble(getWord(0)); }
1617
1618 /// Converts APInt bits to a float
1619 ///
1620 /// The conversion does not do a translation from integer to float, it just
1621 /// re-interprets the bits as a float. Note that it is valid to do this on
1622 /// any bit width. Exactly 32 bits will be translated.
1623 float bitsToFloat() const {
1624 return BitsToFloat(static_cast<uint32_t>(getWord(0)));
1625 }
1626
1627 /// Converts a double to APInt bits.
1628 ///
1629 /// The conversion does not do a translation from double to integer, it just
1630 /// re-interprets the bits of the double.
1631 static APInt doubleToBits(double V) {
1632 return APInt(sizeof(double) * CHAR_BIT8, DoubleToBits(V));
1633 }
1634
1635 /// Converts a float to APInt bits.
1636 ///
1637 /// The conversion does not do a translation from float to integer, it just
1638 /// re-interprets the bits of the float.
1639 static APInt floatToBits(float V) {
1640 return APInt(sizeof(float) * CHAR_BIT8, FloatToBits(V));
1641 }
1642
1643 /// @}
1644 /// \name Mathematics Operations
1645 /// @{
1646
1647 /// \returns the floor log base 2 of this APInt.
1648 unsigned logBase2() const { return getActiveBits() - 1; }
1649
1650 /// \returns the ceil log base 2 of this APInt.
1651 unsigned ceilLogBase2() const {
1652 APInt temp(*this);
1653 --temp;
1654 return temp.getActiveBits();
1655 }
1656
1657 /// \returns the nearest log base 2 of this APInt. Ties round up.
1658 ///
1659 /// NOTE: When we have a BitWidth of 1, we define:
1660 ///
1661 /// log2(0) = UINT32_MAX
1662 /// log2(1) = 0
1663 ///
1664 /// to get around any mathematical concerns resulting from
1665 /// referencing 2 in a space where 2 does no exist.
1666 unsigned nearestLogBase2() const;
1667
1668 /// \returns the log base 2 of this APInt if its an exact power of two, -1
1669 /// otherwise
1670 int32_t exactLogBase2() const {
1671 if (!isPowerOf2())
1672 return -1;
1673 return logBase2();
1674 }
1675
1676 /// Compute the square root.
1677 APInt sqrt() const;
1678
1679 /// Get the absolute value. If *this is < 0 then return -(*this), otherwise
1680 /// *this. Note that the "most negative" signed number (e.g. -128 for 8 bit
1681 /// wide APInt) is unchanged due to how negation works.
1682 APInt abs() const {
1683 if (isNegative())
1684 return -(*this);
1685 return *this;
1686 }
1687
1688 /// \returns the multiplicative inverse for a given modulo.
1689 APInt multiplicativeInverse(const APInt &modulo) const;
1690
1691 /// @}
1692 /// \name Building-block Operations for APInt and APFloat
1693 /// @{
1694
1695 // These building block operations operate on a representation of arbitrary
1696 // precision, two's-complement, bignum integer values. They should be
1697 // sufficient to implement APInt and APFloat bignum requirements. Inputs are
1698 // generally a pointer to the base of an array of integer parts, representing
1699 // an unsigned bignum, and a count of how many parts there are.
1700
1701 /// Sets the least significant part of a bignum to the input value, and zeroes
1702 /// out higher parts.
1703 static void tcSet(WordType *, WordType, unsigned);
1704
1705 /// Assign one bignum to another.
1706 static void tcAssign(WordType *, const WordType *, unsigned);
1707
1708 /// Returns true if a bignum is zero, false otherwise.
1709 static bool tcIsZero(const WordType *, unsigned);
1710
1711 /// Extract the given bit of a bignum; returns 0 or 1. Zero-based.
1712 static int tcExtractBit(const WordType *, unsigned bit);
1713
1714 /// Copy the bit vector of width srcBITS from SRC, starting at bit srcLSB, to
1715 /// DST, of dstCOUNT parts, such that the bit srcLSB becomes the least
1716 /// significant bit of DST. All high bits above srcBITS in DST are
1717 /// zero-filled.
1718 static void tcExtract(WordType *, unsigned dstCount, const WordType *,
1719 unsigned srcBits, unsigned srcLSB);
1720
1721 /// Set the given bit of a bignum. Zero-based.
1722 static void tcSetBit(WordType *, unsigned bit);
1723
1724 /// Clear the given bit of a bignum. Zero-based.
1725 static void tcClearBit(WordType *, unsigned bit);
1726
1727 /// Returns the bit number of the least or most significant set bit of a
1728 /// number. If the input number has no bits set -1U is returned.
1729 static unsigned tcLSB(const WordType *, unsigned n);
1730 static unsigned tcMSB(const WordType *parts, unsigned n);
1731
1732 /// Negate a bignum in-place.
1733 static void tcNegate(WordType *, unsigned);
1734
1735 /// DST += RHS + CARRY where CARRY is zero or one. Returns the carry flag.
1736 static WordType tcAdd(WordType *, const WordType *, WordType carry, unsigned);
1737 /// DST += RHS. Returns the carry flag.
1738 static WordType tcAddPart(WordType *, WordType, unsigned);
1739
1740 /// DST -= RHS + CARRY where CARRY is zero or one. Returns the carry flag.
1741 static WordType tcSubtract(WordType *, const WordType *, WordType carry,
1742 unsigned);
1743 /// DST -= RHS. Returns the carry flag.
1744 static WordType tcSubtractPart(WordType *, WordType, unsigned);
1745
1746 /// DST += SRC * MULTIPLIER + PART if add is true
1747 /// DST = SRC * MULTIPLIER + PART if add is false
1748 ///
1749 /// Requires 0 <= DSTPARTS <= SRCPARTS + 1. If DST overlaps SRC they must
1750 /// start at the same point, i.e. DST == SRC.
1751 ///
1752 /// If DSTPARTS == SRC_PARTS + 1 no overflow occurs and zero is returned.
1753 /// Otherwise DST is filled with the least significant DSTPARTS parts of the
1754 /// result, and if all of the omitted higher parts were zero return zero,
1755 /// otherwise overflow occurred and return one.
1756 static int tcMultiplyPart(WordType *dst, const WordType *src,
1757 WordType multiplier, WordType carry,
1758 unsigned srcParts, unsigned dstParts, bool add);
1759
1760 /// DST = LHS * RHS, where DST has the same width as the operands and is
1761 /// filled with the least significant parts of the result. Returns one if
1762 /// overflow occurred, otherwise zero. DST must be disjoint from both
1763 /// operands.
1764 static int tcMultiply(WordType *, const WordType *, const WordType *,
1765 unsigned);
1766
1767 /// DST = LHS * RHS, where DST has width the sum of the widths of the
1768 /// operands. No overflow occurs. DST must be disjoint from both operands.
1769 static void tcFullMultiply(WordType *, const WordType *, const WordType *,
1770 unsigned, unsigned);
1771
1772 /// If RHS is zero LHS and REMAINDER are left unchanged, return one.
1773 /// Otherwise set LHS to LHS / RHS with the fractional part discarded, set
1774 /// REMAINDER to the remainder, return zero. i.e.
1775 ///
1776 /// OLD_LHS = RHS * LHS + REMAINDER
1777 ///
1778 /// SCRATCH is a bignum of the same size as the operands and result for use by
1779 /// the routine; its contents need not be initialized and are destroyed. LHS,
1780 /// REMAINDER and SCRATCH must be distinct.
1781 static int tcDivide(WordType *lhs, const WordType *rhs, WordType *remainder,
1782 WordType *scratch, unsigned parts);
1783
1784 /// Shift a bignum left Count bits. Shifted in bits are zero. There are no
1785 /// restrictions on Count.
1786 static void tcShiftLeft(WordType *, unsigned Words, unsigned Count);
1787
1788 /// Shift a bignum right Count bits. Shifted in bits are zero. There are no
1789 /// restrictions on Count.
1790 static void tcShiftRight(WordType *, unsigned Words, unsigned Count);
1791
1792 /// Comparison (unsigned) of two bignums.
1793 static int tcCompare(const WordType *, const WordType *, unsigned);
1794
1795 /// Increment a bignum in-place. Return the carry flag.
1796 static WordType tcIncrement(WordType *dst, unsigned parts) {
1797 return tcAddPart(dst, 1, parts);
1798 }
1799
1800 /// Decrement a bignum in-place. Return the borrow flag.
1801 static WordType tcDecrement(WordType *dst, unsigned parts) {
1802 return tcSubtractPart(dst, 1, parts);
1803 }
1804
1805 /// Used to insert APInt objects, or objects that contain APInt objects, into
1806 /// FoldingSets.
1807 void Profile(FoldingSetNodeID &id) const;
1808
1809 /// debug method
1810 void dump() const;
1811
1812 /// Returns whether this instance allocated memory.
1813 bool needsCleanup() const { return !isSingleWord(); }
1814
1815private:
1816 /// This union is used to store the integer value. When the
1817 /// integer bit-width <= 64, it uses VAL, otherwise it uses pVal.
1818 union {
1819 uint64_t VAL; ///< Used to store the <= 64 bits integer value.
1820 uint64_t *pVal; ///< Used to store the >64 bits integer value.
1821 } U;
1822
1823 unsigned BitWidth; ///< The number of bits in this APInt.
1824
1825 friend struct DenseMapInfo<APInt, void>;
1826 friend class APSInt;
1827
1828 /// This constructor is used only internally for speed of construction of
1829 /// temporaries. It is unsafe since it takes ownership of the pointer, so it
1830 /// is not public.
1831 APInt(uint64_t *val, unsigned bits) : BitWidth(bits) { U.pVal = val; }
1832
1833 /// Determine which word a bit is in.
1834 ///
1835 /// \returns the word position for the specified bit position.
1836 static unsigned whichWord(unsigned bitPosition) {
1837 return bitPosition / APINT_BITS_PER_WORD;
1838 }
1839
1840 /// Determine which bit in a word the specified bit position is in.
1841 static unsigned whichBit(unsigned bitPosition) {
1842 return bitPosition % APINT_BITS_PER_WORD;
1843 }
1844
1845 /// Get a single bit mask.
1846 ///
1847 /// \returns a uint64_t with only bit at "whichBit(bitPosition)" set
1848 /// This method generates and returns a uint64_t (word) mask for a single
1849 /// bit at a specific bit position. This is used to mask the bit in the
1850 /// corresponding word.
1851 static uint64_t maskBit(unsigned bitPosition) {
1852 return 1ULL << whichBit(bitPosition);
1853 }
1854
1855 /// Clear unused high order bits
1856 ///
1857 /// This method is used internally to clear the top "N" bits in the high order
1858 /// word that are not used by the APInt. This is needed after the most
1859 /// significant word is assigned a value to ensure that those bits are
1860 /// zero'd out.
1861 APInt &clearUnusedBits() {
1862 // Compute how many bits are used in the final word.
1863 unsigned WordBits = ((BitWidth - 1) % APINT_BITS_PER_WORD) + 1;
1864
1865 // Mask out the high bits.
1866 uint64_t mask = WORDTYPE_MAX >> (APINT_BITS_PER_WORD - WordBits);
1867 if (LLVM_UNLIKELY(BitWidth == 0)__builtin_expect((bool)(BitWidth == 0), false))
1868 mask = 0;
1869
1870 if (isSingleWord())
1871 U.VAL &= mask;
1872 else
1873 U.pVal[getNumWords() - 1] &= mask;
1874 return *this;
1875 }
1876
1877 /// Get the word corresponding to a bit position
1878 /// \returns the corresponding word for the specified bit position.
1879 uint64_t getWord(unsigned bitPosition) const {
1880 return isSingleWord() ? U.VAL : U.pVal[whichWord(bitPosition)];
1881 }
1882
1883 /// Utility method to change the bit width of this APInt to new bit width,
1884 /// allocating and/or deallocating as necessary. There is no guarantee on the
1885 /// value of any bits upon return. Caller should populate the bits after.
1886 void reallocate(unsigned NewBitWidth);
1887
1888 /// Convert a char array into an APInt
1889 ///
1890 /// \param radix 2, 8, 10, 16, or 36
1891 /// Converts a string into a number. The string must be non-empty
1892 /// and well-formed as a number of the given base. The bit-width
1893 /// must be sufficient to hold the result.
1894 ///
1895 /// This is used by the constructors that take string arguments.
1896 ///
1897 /// StringRef::getAsInteger is superficially similar but (1) does
1898 /// not assume that the string is well-formed and (2) grows the
1899 /// result to hold the input.
1900 void fromString(unsigned numBits, StringRef str, uint8_t radix);
1901
1902 /// An internal division function for dividing APInts.
1903 ///
1904 /// This is used by the toString method to divide by the radix. It simply
1905 /// provides a more convenient form of divide for internal use since KnuthDiv
1906 /// has specific constraints on its inputs. If those constraints are not met
1907 /// then it provides a simpler form of divide.
1908 static void divide(const WordType *LHS, unsigned lhsWords,
1909 const WordType *RHS, unsigned rhsWords, WordType *Quotient,
1910 WordType *Remainder);
1911
1912 /// out-of-line slow case for inline constructor
1913 void initSlowCase(uint64_t val, bool isSigned);
1914
1915 /// shared code between two array constructors
1916 void initFromArray(ArrayRef<uint64_t> array);
1917
1918 /// out-of-line slow case for inline copy constructor
1919 void initSlowCase(const APInt &that);
1920
1921 /// out-of-line slow case for shl
1922 void shlSlowCase(unsigned ShiftAmt);
1923
1924 /// out-of-line slow case for lshr.
1925 void lshrSlowCase(unsigned ShiftAmt);
1926
1927 /// out-of-line slow case for ashr.
1928 void ashrSlowCase(unsigned ShiftAmt);
1929
1930 /// out-of-line slow case for operator=
1931 void assignSlowCase(const APInt &RHS);
1932
1933 /// out-of-line slow case for operator==
1934 bool equalSlowCase(const APInt &RHS) const LLVM_READONLY__attribute__((__pure__));
1935
1936 /// out-of-line slow case for countLeadingZeros
1937 unsigned countLeadingZerosSlowCase() const LLVM_READONLY__attribute__((__pure__));
1938
1939 /// out-of-line slow case for countLeadingOnes.
1940 unsigned countLeadingOnesSlowCase() const LLVM_READONLY__attribute__((__pure__));
1941
1942 /// out-of-line slow case for countTrailingZeros.
1943 unsigned countTrailingZerosSlowCase() const LLVM_READONLY__attribute__((__pure__));
1944
1945 /// out-of-line slow case for countTrailingOnes
1946 unsigned countTrailingOnesSlowCase() const LLVM_READONLY__attribute__((__pure__));
1947
1948 /// out-of-line slow case for countPopulation
1949 unsigned countPopulationSlowCase() const LLVM_READONLY__attribute__((__pure__));
1950
1951 /// out-of-line slow case for intersects.
1952 bool intersectsSlowCase(const APInt &RHS) const LLVM_READONLY__attribute__((__pure__));
1953
1954 /// out-of-line slow case for isSubsetOf.
1955 bool isSubsetOfSlowCase(const APInt &RHS) const LLVM_READONLY__attribute__((__pure__));
1956
1957 /// out-of-line slow case for setBits.
1958 void setBitsSlowCase(unsigned loBit, unsigned hiBit);
1959
1960 /// out-of-line slow case for flipAllBits.
1961 void flipAllBitsSlowCase();
1962
1963 /// out-of-line slow case for concat.
1964 APInt concatSlowCase(const APInt &NewLSB) const;
1965
1966 /// out-of-line slow case for operator&=.
1967 void andAssignSlowCase(const APInt &RHS);
1968
1969 /// out-of-line slow case for operator|=.
1970 void orAssignSlowCase(const APInt &RHS);
1971
1972 /// out-of-line slow case for operator^=.
1973 void xorAssignSlowCase(const APInt &RHS);
1974
1975 /// Unsigned comparison. Returns -1, 0, or 1 if this APInt is less than, equal
1976 /// to, or greater than RHS.
1977 int compare(const APInt &RHS) const LLVM_READONLY__attribute__((__pure__));
1978
1979 /// Signed comparison. Returns -1, 0, or 1 if this APInt is less than, equal
1980 /// to, or greater than RHS.
1981 int compareSigned(const APInt &RHS) const LLVM_READONLY__attribute__((__pure__));
1982
1983 /// @}
1984};
1985
1986inline bool operator==(uint64_t V1, const APInt &V2) { return V2 == V1; }
1987
1988inline bool operator!=(uint64_t V1, const APInt &V2) { return V2 != V1; }
1989
1990/// Unary bitwise complement operator.
1991///
1992/// \returns an APInt that is the bitwise complement of \p v.
1993inline APInt operator~(APInt v) {
1994 v.flipAllBits();
1995 return v;
1996}
1997
1998inline APInt operator&(APInt a, const APInt &b) {
1999 a &= b;
2000 return a;
2001}
2002
2003inline APInt operator&(const APInt &a, APInt &&b) {
2004 b &= a;
2005 return std::move(b);
2006}
2007
2008inline APInt operator&(APInt a, uint64_t RHS) {
2009 a &= RHS;
2010 return a;
2011}
2012
2013inline APInt operator&(uint64_t LHS, APInt b) {
2014 b &= LHS;
2015 return b;
2016}
2017
2018inline APInt operator|(APInt a, const APInt &b) {
2019 a |= b;
2020 return a;
2021}
2022
2023inline APInt operator|(const APInt &a, APInt &&b) {
2024 b |= a;
2025 return std::move(b);
2026}
2027
2028inline APInt operator|(APInt a, uint64_t RHS) {
2029 a |= RHS;
2030 return a;
2031}
2032
2033inline APInt operator|(uint64_t LHS, APInt b) {
2034 b |= LHS;
2035 return b;
2036}
2037
2038inline APInt operator^(APInt a, const APInt &b) {
2039 a ^= b;
2040 return a;
2041}
2042
2043inline APInt operator^(const APInt &a, APInt &&b) {
2044 b ^= a;
2045 return std::move(b);
2046}
2047
2048inline APInt operator^(APInt a, uint64_t RHS) {
2049 a ^= RHS;
2050 return a;
2051}
2052
2053inline APInt operator^(uint64_t LHS, APInt b) {
2054 b ^= LHS;
2055 return b;
2056}
2057
2058inline raw_ostream &operator<<(raw_ostream &OS, const APInt &I) {
2059 I.print(OS, true);
2060 return OS;
2061}
2062
2063inline APInt operator-(APInt v) {
2064 v.negate();
2065 return v;
2066}
2067
2068inline APInt operator+(APInt a, const APInt &b) {
2069 a += b;
2070 return a;
2071}
2072
2073inline APInt operator+(const APInt &a, APInt &&b) {
2074 b += a;
2075 return std::move(b);
2076}
2077
2078inline APInt operator+(APInt a, uint64_t RHS) {
2079 a += RHS;
2080 return a;
2081}
2082
2083inline APInt operator+(uint64_t LHS, APInt b) {
2084 b += LHS;
2085 return b;
2086}
2087
2088inline APInt operator-(APInt a, const APInt &b) {
2089 a -= b;
2090 return a;
2091}
2092
2093inline APInt operator-(const APInt &a, APInt &&b) {
2094 b.negate();
2095 b += a;
2096 return std::move(b);
2097}
2098
2099inline APInt operator-(APInt a, uint64_t RHS) {
2100 a -= RHS;
2101 return a;
2102}
2103
2104inline APInt operator-(uint64_t LHS, APInt b) {
2105 b.negate();
2106 b += LHS;
2107 return b;
2108}
2109
2110inline APInt operator*(APInt a, uint64_t RHS) {
2111 a *= RHS;
2112 return a;
2113}
2114
2115inline APInt operator*(uint64_t LHS, APInt b) {
2116 b *= LHS;
2117 return b;
2118}
2119
2120namespace APIntOps {
2121
2122/// Determine the smaller of two APInts considered to be signed.
2123inline const APInt &smin(const APInt &A, const APInt &B) {
2124 return A.slt(B) ? A : B;
2125}
2126
2127/// Determine the larger of two APInts considered to be signed.
2128inline const APInt &smax(const APInt &A, const APInt &B) {
2129 return A.sgt(B) ? A : B;
2130}
2131
2132/// Determine the smaller of two APInts considered to be unsigned.
2133inline const APInt &umin(const APInt &A, const APInt &B) {
2134 return A.ult(B) ? A : B;
2135}
2136
2137/// Determine the larger of two APInts considered to be unsigned.
2138inline const APInt &umax(const APInt &A, const APInt &B) {
2139 return A.ugt(B) ? A : B;
2140}
2141
2142/// Compute GCD of two unsigned APInt values.
2143///
2144/// This function returns the greatest common divisor of the two APInt values
2145/// using Stein's algorithm.
2146///
2147/// \returns the greatest common divisor of A and B.
2148APInt GreatestCommonDivisor(APInt A, APInt B);
2149
2150/// Converts the given APInt to a double value.
2151///
2152/// Treats the APInt as an unsigned value for conversion purposes.
2153inline double RoundAPIntToDouble(const APInt &APIVal) {
2154 return APIVal.roundToDouble();
2155}
2156
2157/// Converts the given APInt to a double value.
2158///
2159/// Treats the APInt as a signed value for conversion purposes.
2160inline double RoundSignedAPIntToDouble(const APInt &APIVal) {
2161 return APIVal.signedRoundToDouble();
2162}
2163
2164/// Converts the given APInt to a float value.
2165inline float RoundAPIntToFloat(const APInt &APIVal) {
2166 return float(RoundAPIntToDouble(APIVal));
2167}
2168
2169/// Converts the given APInt to a float value.
2170///
2171/// Treats the APInt as a signed value for conversion purposes.
2172inline float RoundSignedAPIntToFloat(const APInt &APIVal) {
2173 return float(APIVal.signedRoundToDouble());
2174}
2175
2176/// Converts the given double value into a APInt.
2177///
2178/// This function convert a double value to an APInt value.
2179APInt RoundDoubleToAPInt(double Double, unsigned width);
2180
2181/// Converts a float value into a APInt.
2182///
2183/// Converts a float value into an APInt value.
2184inline APInt RoundFloatToAPInt(float Float, unsigned width) {
2185 return RoundDoubleToAPInt(double(Float), width);
2186}
2187
2188/// Return A unsign-divided by B, rounded by the given rounding mode.
2189APInt RoundingUDiv(const APInt &A, const APInt &B, APInt::Rounding RM);
2190
2191/// Return A sign-divided by B, rounded by the given rounding mode.
2192APInt RoundingSDiv(const APInt &A, const APInt &B, APInt::Rounding RM);
2193
2194/// Let q(n) = An^2 + Bn + C, and BW = bit width of the value range
2195/// (e.g. 32 for i32).
2196/// This function finds the smallest number n, such that
2197/// (a) n >= 0 and q(n) = 0, or
2198/// (b) n >= 1 and q(n-1) and q(n), when evaluated in the set of all
2199/// integers, belong to two different intervals [Rk, Rk+R),
2200/// where R = 2^BW, and k is an integer.
2201/// The idea here is to find when q(n) "overflows" 2^BW, while at the
2202/// same time "allowing" subtraction. In unsigned modulo arithmetic a
2203/// subtraction (treated as addition of negated numbers) would always
2204/// count as an overflow, but here we want to allow values to decrease
2205/// and increase as long as they are within the same interval.
2206/// Specifically, adding of two negative numbers should not cause an
2207/// overflow (as long as the magnitude does not exceed the bit width).
2208/// On the other hand, given a positive number, adding a negative
2209/// number to it can give a negative result, which would cause the
2210/// value to go from [-2^BW, 0) to [0, 2^BW). In that sense, zero is
2211/// treated as a special case of an overflow.
2212///
2213/// This function returns None if after finding k that minimizes the
2214/// positive solution to q(n) = kR, both solutions are contained between
2215/// two consecutive integers.
2216///
2217/// There are cases where q(n) > T, and q(n+1) < T (assuming evaluation
2218/// in arithmetic modulo 2^BW, and treating the values as signed) by the
2219/// virtue of *signed* overflow. This function will *not* find such an n,
2220/// however it may find a value of n satisfying the inequalities due to
2221/// an *unsigned* overflow (if the values are treated as unsigned).
2222/// To find a solution for a signed overflow, treat it as a problem of
2223/// finding an unsigned overflow with a range with of BW-1.
2224///
2225/// The returned value may have a different bit width from the input
2226/// coefficients.
2227Optional<APInt> SolveQuadraticEquationWrap(APInt A, APInt B, APInt C,
2228 unsigned RangeWidth);
2229
2230/// Compare two values, and if they are different, return the position of the
2231/// most significant bit that is different in the values.
2232Optional<unsigned> GetMostSignificantDifferentBit(const APInt &A,
2233 const APInt &B);
2234
2235/// Splat/Merge neighboring bits to widen/narrow the bitmask represented
2236/// by \param A to \param NewBitWidth bits.
2237///
2238/// e.g. ScaleBitMask(0b0101, 8) -> 0b00110011
2239/// e.g. ScaleBitMask(0b00011011, 4) -> 0b0111
2240/// A.getBitwidth() or NewBitWidth must be a whole multiples of the other.
2241///
2242/// TODO: Do we need a mode where all bits must be set when merging down?
2243APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth);
2244} // namespace APIntOps
2245
2246// See friend declaration above. This additional declaration is required in
2247// order to compile LLVM with IBM xlC compiler.
2248hash_code hash_value(const APInt &Arg);
2249
2250/// StoreIntToMemory - Fills the StoreBytes bytes of memory starting from Dst
2251/// with the integer held in IntVal.
2252void StoreIntToMemory(const APInt &IntVal, uint8_t *Dst, unsigned StoreBytes);
2253
2254/// LoadIntFromMemory - Loads the integer stored in the LoadBytes bytes starting
2255/// from Src into IntVal, which is assumed to be wide enough and to hold zero.
2256void LoadIntFromMemory(APInt &IntVal, const uint8_t *Src, unsigned LoadBytes);
2257
2258/// Provide DenseMapInfo for APInt.
2259template <> struct DenseMapInfo<APInt, void> {
2260 static inline APInt getEmptyKey() {
2261 APInt V(nullptr, 0);
2262 V.U.VAL = 0;
2263 return V;
2264 }
2265
2266 static inline APInt getTombstoneKey() {
2267 APInt V(nullptr, 0);
2268 V.U.VAL = 1;
2269 return V;
2270 }
2271
2272 static unsigned getHashValue(const APInt &Key);
2273
2274 static bool isEqual(const APInt &LHS, const APInt &RHS) {
2275 return LHS.getBitWidth() == RHS.getBitWidth() && LHS == RHS;
2276 }
2277};
2278
2279} // namespace llvm
2280
2281#endif

/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/llvm/include/llvm/IR/Constants.h

1//===-- llvm/Constants.h - Constant class subclass definitions --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// @file
10/// This file contains the declarations for the subclasses of Constant,
11/// which represent the different flavors of constant values that live in LLVM.
12/// Note that Constants are immutable (once created they never change) and are
13/// fully shared by structural equivalence. This means that two structurally
14/// equivalent constants will always have the same address. Constants are
15/// created on demand as needed and never deleted: thus clients don't have to
16/// worry about the lifetime of the objects.
17//
18//===----------------------------------------------------------------------===//
19
20#ifndef LLVM_IR_CONSTANTS_H
21#define LLVM_IR_CONSTANTS_H
22
23#include "llvm/ADT/APFloat.h"
24#include "llvm/ADT/APInt.h"
25#include "llvm/ADT/ArrayRef.h"
26#include "llvm/ADT/None.h"
27#include "llvm/ADT/Optional.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/ADT/StringRef.h"
30#include "llvm/IR/Constant.h"
31#include "llvm/IR/DerivedTypes.h"
32#include "llvm/IR/OperandTraits.h"
33#include "llvm/IR/User.h"
34#include "llvm/IR/Value.h"
35#include "llvm/Support/Casting.h"
36#include "llvm/Support/Compiler.h"
37#include "llvm/Support/ErrorHandling.h"
38#include <cassert>
39#include <cstddef>
40#include <cstdint>
41
42namespace llvm {
43
44template <class ConstantClass> struct ConstantAggrKeyType;
45
46/// Base class for constants with no operands.
47///
48/// These constants have no operands; they represent their data directly.
49/// Since they can be in use by unrelated modules (and are never based on
50/// GlobalValues), it never makes sense to RAUW them.
51class ConstantData : public Constant {
52 friend class Constant;
53
54 Value *handleOperandChangeImpl(Value *From, Value *To) {
55 llvm_unreachable("Constant data does not have operands!")::llvm::llvm_unreachable_internal("Constant data does not have operands!"
, "llvm/include/llvm/IR/Constants.h", 55)
;
56 }
57
58protected:
59 explicit ConstantData(Type *Ty, ValueTy VT) : Constant(Ty, VT, nullptr, 0) {}
60
61 void *operator new(size_t S) { return User::operator new(S, 0); }
62
63public:
64 void operator delete(void *Ptr) { User::operator delete(Ptr); }
65
66 ConstantData(const ConstantData &) = delete;
67
68 /// Methods to support type inquiry through isa, cast, and dyn_cast.
69 static bool classof(const Value *V) {
70 return V->getValueID() >= ConstantDataFirstVal &&
71 V->getValueID() <= ConstantDataLastVal;
72 }
73};
74
75//===----------------------------------------------------------------------===//
76/// This is the shared class of boolean and integer constants. This class
77/// represents both boolean and integral constants.
78/// Class for constant integers.
79class ConstantInt final : public ConstantData {
80 friend class Constant;
81
82 APInt Val;
83
84 ConstantInt(IntegerType *Ty, const APInt &V);
85
86 void destroyConstantImpl();
87
88public:
89 ConstantInt(const ConstantInt &) = delete;
90
91 static ConstantInt *getTrue(LLVMContext &Context);
92 static ConstantInt *getFalse(LLVMContext &Context);
93 static ConstantInt *getBool(LLVMContext &Context, bool V);
94 static Constant *getTrue(Type *Ty);
95 static Constant *getFalse(Type *Ty);
96 static Constant *getBool(Type *Ty, bool V);
97
98 /// If Ty is a vector type, return a Constant with a splat of the given
99 /// value. Otherwise return a ConstantInt for the given value.
100 static Constant *get(Type *Ty, uint64_t V, bool IsSigned = false);
101
102 /// Return a ConstantInt with the specified integer value for the specified
103 /// type. If the type is wider than 64 bits, the value will be zero-extended
104 /// to fit the type, unless IsSigned is true, in which case the value will
105 /// be interpreted as a 64-bit signed integer and sign-extended to fit
106 /// the type.
107 /// Get a ConstantInt for a specific value.
108 static ConstantInt *get(IntegerType *Ty, uint64_t V, bool IsSigned = false);
109
110 /// Return a ConstantInt with the specified value for the specified type. The
111 /// value V will be canonicalized to a an unsigned APInt. Accessing it with
112 /// either getSExtValue() or getZExtValue() will yield a correctly sized and
113 /// signed value for the type Ty.
114 /// Get a ConstantInt for a specific signed value.
115 static ConstantInt *getSigned(IntegerType *Ty, int64_t V);
116 static Constant *getSigned(Type *Ty, int64_t V);
117
118 /// Return a ConstantInt with the specified value and an implied Type. The
119 /// type is the integer type that corresponds to the bit width of the value.
120 static ConstantInt *get(LLVMContext &Context, const APInt &V);
121
122 /// Return a ConstantInt constructed from the string strStart with the given
123 /// radix.
124 static ConstantInt *get(IntegerType *Ty, StringRef Str, uint8_t Radix);
125
126 /// If Ty is a vector type, return a Constant with a splat of the given
127 /// value. Otherwise return a ConstantInt for the given value.
128 static Constant *get(Type *Ty, const APInt &V);
129
130 /// Return the constant as an APInt value reference. This allows clients to
131 /// obtain a full-precision copy of the value.
132 /// Return the constant's value.
133 inline const APInt &getValue() const { return Val; }
134
135 /// getBitWidth - Return the bitwidth of this constant.
136 unsigned getBitWidth() const { return Val.getBitWidth(); }
137
138 /// Return the constant as a 64-bit unsigned integer value after it
139 /// has been zero extended as appropriate for the type of this constant. Note
140 /// that this method can assert if the value does not fit in 64 bits.
141 /// Return the zero extended value.
142 inline uint64_t getZExtValue() const { return Val.getZExtValue(); }
143
144 /// Return the constant as a 64-bit integer value after it has been sign
145 /// extended as appropriate for the type of this constant. Note that
146 /// this method can assert if the value does not fit in 64 bits.
147 /// Return the sign extended value.
148 inline int64_t getSExtValue() const { return Val.getSExtValue(); }
149
150 /// Return the constant as an llvm::MaybeAlign.
151 /// Note that this method can assert if the value does not fit in 64 bits or
152 /// is not a power of two.
153 inline MaybeAlign getMaybeAlignValue() const {
154 return MaybeAlign(getZExtValue());
155 }
156
157 /// Return the constant as an llvm::Align, interpreting `0` as `Align(1)`.
158 /// Note that this method can assert if the value does not fit in 64 bits or
159 /// is not a power of two.
160 inline Align getAlignValue() const {
161 return getMaybeAlignValue().valueOrOne();
162 }
163
164 /// A helper method that can be used to determine if the constant contained
165 /// within is equal to a constant. This only works for very small values,
166 /// because this is all that can be represented with all types.
167 /// Determine if this constant's value is same as an unsigned char.
168 bool equalsInt(uint64_t V) const { return Val == V; }
169
170 /// getType - Specialize the getType() method to always return an IntegerType,
171 /// which reduces the amount of casting needed in parts of the compiler.
172 ///
173 inline IntegerType *getType() const {
174 return cast<IntegerType>(Value::getType());
175 }
176
177 /// This static method returns true if the type Ty is big enough to
178 /// represent the value V. This can be used to avoid having the get method
179 /// assert when V is larger than Ty can represent. Note that there are two
180 /// versions of this method, one for unsigned and one for signed integers.
181 /// Although ConstantInt canonicalizes everything to an unsigned integer,
182 /// the signed version avoids callers having to convert a signed quantity
183 /// to the appropriate unsigned type before calling the method.
184 /// @returns true if V is a valid value for type Ty
185 /// Determine if the value is in range for the given type.
186 static bool isValueValidForType(Type *Ty, uint64_t V);
187 static bool isValueValidForType(Type *Ty, int64_t V);
188
189 bool isNegative() const { return Val.isNegative(); }
190
191 /// This is just a convenience method to make client code smaller for a
192 /// common code. It also correctly performs the comparison without the
193 /// potential for an assertion from getZExtValue().
194 bool isZero() const { return Val.isZero(); }
195
196 /// This is just a convenience method to make client code smaller for a
197 /// common case. It also correctly performs the comparison without the
198 /// potential for an assertion from getZExtValue().
199 /// Determine if the value is one.
200 bool isOne() const { return Val.isOne(); }
62
Calling 'APInt::isOne'
70
Returning from 'APInt::isOne'
71
Returning the value 1, which participates in a condition later
201
202 /// This function will return true iff every bit in this constant is set
203 /// to true.
204 /// @returns true iff this constant's bits are all set to true.
205 /// Determine if the value is all ones.
206 bool isMinusOne() const { return Val.isAllOnes(); }
207
208 /// This function will return true iff this constant represents the largest
209 /// value that may be represented by the constant's type.
210 /// @returns true iff this is the largest value that may be represented
211 /// by this type.
212 /// Determine if the value is maximal.
213 bool isMaxValue(bool IsSigned) const {
214 if (IsSigned)
215 return Val.isMaxSignedValue();
216 else
217 return Val.isMaxValue();
218 }
219
220 /// This function will return true iff this constant represents the smallest
221 /// value that may be represented by this constant's type.
222 /// @returns true if this is the smallest value that may be represented by
223 /// this type.
224 /// Determine if the value is minimal.
225 bool isMinValue(bool IsSigned) const {
226 if (IsSigned)
227 return Val.isMinSignedValue();
228 else
229 return Val.isMinValue();
230 }
231
232 /// This function will return true iff this constant represents a value with
233 /// active bits bigger than 64 bits or a value greater than the given uint64_t
234 /// value.
235 /// @returns true iff this constant is greater or equal to the given number.
236 /// Determine if the value is greater or equal to the given number.
237 bool uge(uint64_t Num) const { return Val.uge(Num); }
238
239 /// getLimitedValue - If the value is smaller than the specified limit,
240 /// return it, otherwise return the limit value. This causes the value
241 /// to saturate to the limit.
242 /// @returns the min of the value of the constant and the specified value
243 /// Get the constant's value with a saturation limit
244 uint64_t getLimitedValue(uint64_t Limit = ~0ULL) const {
245 return Val.getLimitedValue(Limit);
246 }
247
248 /// Methods to support type inquiry through isa, cast, and dyn_cast.
249 static bool classof(const Value *V) {
250 return V->getValueID() == ConstantIntVal;
251 }
252};
253
254//===----------------------------------------------------------------------===//
255/// ConstantFP - Floating Point Values [float, double]
256///
257class ConstantFP final : public ConstantData {
258 friend class Constant;
259
260 APFloat Val;
261
262 ConstantFP(Type *Ty, const APFloat &V);
263
264 void destroyConstantImpl();
265
266public:
267 ConstantFP(const ConstantFP &) = delete;
268
269 /// Floating point negation must be implemented with f(x) = -0.0 - x. This
270 /// method returns the negative zero constant for floating point or vector
271 /// floating point types; for all other types, it returns the null value.
272 static Constant *getZeroValueForNegation(Type *Ty);
273
274 /// This returns a ConstantFP, or a vector containing a splat of a ConstantFP,
275 /// for the specified value in the specified type. This should only be used
276 /// for simple constant values like 2.0/1.0 etc, that are known-valid both as
277 /// host double and as the target format.
278 static Constant *get(Type *Ty, double V);
279
280 /// If Ty is a vector type, return a Constant with a splat of the given
281 /// value. Otherwise return a ConstantFP for the given value.
282 static Constant *get(Type *Ty, const APFloat &V);
283
284 static Constant *get(Type *Ty, StringRef Str);
285 static ConstantFP *get(LLVMContext &Context, const APFloat &V);
286 static Constant *getNaN(Type *Ty, bool Negative = false,
287 uint64_t Payload = 0);
288 static Constant *getQNaN(Type *Ty, bool Negative = false,
289 APInt *Payload = nullptr);
290 static Constant *getSNaN(Type *Ty, bool Negative = false,
291 APInt *Payload = nullptr);
292 static Constant *getNegativeZero(Type *Ty);
293 static Constant *getInfinity(Type *Ty, bool Negative = false);
294
295 /// Return true if Ty is big enough to represent V.
296 static bool isValueValidForType(Type *Ty, const APFloat &V);
297 inline const APFloat &getValueAPF() const { return Val; }
298 inline const APFloat &getValue() const { return Val; }
299
300 /// Return true if the value is positive or negative zero.
301 bool isZero() const { return Val.isZero(); }
302
303 /// Return true if the sign bit is set.
304 bool isNegative() const { return Val.isNegative(); }
305
306 /// Return true if the value is infinity
307 bool isInfinity() const { return Val.isInfinity(); }
308
309 /// Return true if the value is a NaN.
310 bool isNaN() const { return Val.isNaN(); }
311
312 /// We don't rely on operator== working on double values, as it returns true
313 /// for things that are clearly not equal, like -0.0 and 0.0.
314 /// As such, this method can be used to do an exact bit-for-bit comparison of
315 /// two floating point values. The version with a double operand is retained
316 /// because it's so convenient to write isExactlyValue(2.0), but please use
317 /// it only for simple constants.
318 bool isExactlyValue(const APFloat &V) const;
319
320 bool isExactlyValue(double V) const {
321 bool ignored;
322 APFloat FV(V);
323 FV.convert(Val.getSemantics(), APFloat::rmNearestTiesToEven, &ignored);
324 return isExactlyValue(FV);
325 }
326
327 /// Methods for support type inquiry through isa, cast, and dyn_cast:
328 static bool classof(const Value *V) {
329 return V->getValueID() == ConstantFPVal;
330 }
331};
332
333//===----------------------------------------------------------------------===//
334/// All zero aggregate value
335///
336class ConstantAggregateZero final : public ConstantData {
337 friend class Constant;
338
339 explicit ConstantAggregateZero(Type *Ty)
340 : ConstantData(Ty, ConstantAggregateZeroVal) {}
341
342 void destroyConstantImpl();
343
344public:
345 ConstantAggregateZero(const ConstantAggregateZero &) = delete;
346
347 static ConstantAggregateZero *get(Type *Ty);
348
349 /// If this CAZ has array or vector type, return a zero with the right element
350 /// type.
351 Constant *getSequentialElement() const;
352
353 /// If this CAZ has struct type, return a zero with the right element type for
354 /// the specified element.
355 Constant *getStructElement(unsigned Elt) const;
356
357 /// Return a zero of the right value for the specified GEP index if we can,
358 /// otherwise return null (e.g. if C is a ConstantExpr).
359 Constant *getElementValue(Constant *C) const;
360
361 /// Return a zero of the right value for the specified GEP index.
362 Constant *getElementValue(unsigned Idx) const;
363
364 /// Return the number of elements in the array, vector, or struct.
365 ElementCount getElementCount() const;
366
367 /// Methods for support type inquiry through isa, cast, and dyn_cast:
368 ///
369 static bool classof(const Value *V) {
370 return V->getValueID() == ConstantAggregateZeroVal;
371 }
372};
373
374/// Base class for aggregate constants (with operands).
375///
376/// These constants are aggregates of other constants, which are stored as
377/// operands.
378///
379/// Subclasses are \a ConstantStruct, \a ConstantArray, and \a
380/// ConstantVector.
381///
382/// \note Some subclasses of \a ConstantData are semantically aggregates --
383/// such as \a ConstantDataArray -- but are not subclasses of this because they
384/// use operands.
385class ConstantAggregate : public Constant {
386protected:
387 ConstantAggregate(Type *T, ValueTy VT, ArrayRef<Constant *> V);
388
389public:
390 /// Transparently provide more efficient getOperand methods.
391 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant)public: inline Constant *getOperand(unsigned) const; inline void
setOperand(unsigned, Constant*); inline op_iterator op_begin
(); inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
392
393 /// Methods for support type inquiry through isa, cast, and dyn_cast:
394 static bool classof(const Value *V) {
395 return V->getValueID() >= ConstantAggregateFirstVal &&
396 V->getValueID() <= ConstantAggregateLastVal;
397 }
398};
399
400template <>
401struct OperandTraits<ConstantAggregate>
402 : public VariadicOperandTraits<ConstantAggregate> {};
403
404DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantAggregate, Constant)ConstantAggregate::op_iterator ConstantAggregate::op_begin() {
return OperandTraits<ConstantAggregate>::op_begin(this
); } ConstantAggregate::const_op_iterator ConstantAggregate::
op_begin() const { return OperandTraits<ConstantAggregate>
::op_begin(const_cast<ConstantAggregate*>(this)); } ConstantAggregate
::op_iterator ConstantAggregate::op_end() { return OperandTraits
<ConstantAggregate>::op_end(this); } ConstantAggregate::
const_op_iterator ConstantAggregate::op_end() const { return OperandTraits
<ConstantAggregate>::op_end(const_cast<ConstantAggregate
*>(this)); } Constant *ConstantAggregate::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<ConstantAggregate>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ConstantAggregate>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Constants.h", 404, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Constant>( OperandTraits<ConstantAggregate
>::op_begin(const_cast<ConstantAggregate*>(this))[i_nocapture
].get()); } void ConstantAggregate::setOperand(unsigned i_nocapture
, Constant *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ConstantAggregate>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ConstantAggregate>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Constants.h", 404, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<ConstantAggregate>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned ConstantAggregate::getNumOperands
() const { return OperandTraits<ConstantAggregate>::operands
(this); } template <int Idx_nocapture> Use &ConstantAggregate
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &ConstantAggregate
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
405
406//===----------------------------------------------------------------------===//
407/// ConstantArray - Constant Array Declarations
408///
409class ConstantArray final : public ConstantAggregate {
410 friend struct ConstantAggrKeyType<ConstantArray>;
411 friend class Constant;
412
413 ConstantArray(ArrayType *T, ArrayRef<Constant *> Val);
414
415 void destroyConstantImpl();
416 Value *handleOperandChangeImpl(Value *From, Value *To);
417
418public:
419 // ConstantArray accessors
420 static Constant *get(ArrayType *T, ArrayRef<Constant *> V);
421
422private:
423 static Constant *getImpl(ArrayType *T, ArrayRef<Constant *> V);
424
425public:
426 /// Specialize the getType() method to always return an ArrayType,
427 /// which reduces the amount of casting needed in parts of the compiler.
428 inline ArrayType *getType() const {
429 return cast<ArrayType>(Value::getType());
430 }
431
432 /// Methods for support type inquiry through isa, cast, and dyn_cast:
433 static bool classof(const Value *V) {
434 return V->getValueID() == ConstantArrayVal;
435 }
436};
437
438//===----------------------------------------------------------------------===//
439// Constant Struct Declarations
440//
441class ConstantStruct final : public ConstantAggregate {
442 friend struct ConstantAggrKeyType<ConstantStruct>;
443 friend class Constant;
444
445 ConstantStruct(StructType *T, ArrayRef<Constant *> Val);
446
447 void destroyConstantImpl();
448 Value *handleOperandChangeImpl(Value *From, Value *To);
449
450public:
451 // ConstantStruct accessors
452 static Constant *get(StructType *T, ArrayRef<Constant *> V);
453
454 template <typename... Csts>
455 static std::enable_if_t<are_base_of<Constant, Csts...>::value, Constant *>
456 get(StructType *T, Csts *...Vs) {
457 return get(T, ArrayRef<Constant *>({Vs...}));
458 }
459
460 /// Return an anonymous struct that has the specified elements.
461 /// If the struct is possibly empty, then you must specify a context.
462 static Constant *getAnon(ArrayRef<Constant *> V, bool Packed = false) {
463 return get(getTypeForElements(V, Packed), V);
464 }
465 static Constant *getAnon(LLVMContext &Ctx, ArrayRef<Constant *> V,
466 bool Packed = false) {
467 return get(getTypeForElements(Ctx, V, Packed), V);
468 }
469
470 /// Return an anonymous struct type to use for a constant with the specified
471 /// set of elements. The list must not be empty.
472 static StructType *getTypeForElements(ArrayRef<Constant *> V,
473 bool Packed = false);
474 /// This version of the method allows an empty list.
475 static StructType *getTypeForElements(LLVMContext &Ctx,
476 ArrayRef<Constant *> V,
477 bool Packed = false);
478
479 /// Specialization - reduce amount of casting.
480 inline StructType *getType() const {
481 return cast<StructType>(Value::getType());
482 }
483
484 /// Methods for support type inquiry through isa, cast, and dyn_cast:
485 static bool classof(const Value *V) {
486 return V->getValueID() == ConstantStructVal;
487 }
488};
489
490//===----------------------------------------------------------------------===//
491/// Constant Vector Declarations
492///
493class ConstantVector final : public ConstantAggregate {
494 friend struct ConstantAggrKeyType<ConstantVector>;
495 friend class Constant;
496
497 ConstantVector(VectorType *T, ArrayRef<Constant *> Val);
498
499 void destroyConstantImpl();
500 Value *handleOperandChangeImpl(Value *From, Value *To);
501
502public:
503 // ConstantVector accessors
504 static Constant *get(ArrayRef<Constant *> V);
505
506private:
507 static Constant *getImpl(ArrayRef<Constant *> V);
508
509public:
510 /// Return a ConstantVector with the specified constant in each element.
511 /// Note that this might not return an instance of ConstantVector
512 static Constant *getSplat(ElementCount EC, Constant *Elt);
513
514 /// Specialize the getType() method to always return a FixedVectorType,
515 /// which reduces the amount of casting needed in parts of the compiler.
516 inline FixedVectorType *getType() const {
517 return cast<FixedVectorType>(Value::getType());
518 }
519
520 /// If all elements of the vector constant have the same value, return that
521 /// value. Otherwise, return nullptr. Ignore undefined elements by setting
522 /// AllowUndefs to true.
523 Constant *getSplatValue(bool AllowUndefs = false) const;
524
525 /// Methods for support type inquiry through isa, cast, and dyn_cast:
526 static bool classof(const Value *V) {
527 return V->getValueID() == ConstantVectorVal;
528 }
529};
530
531//===----------------------------------------------------------------------===//
532/// A constant pointer value that points to null
533///
534class ConstantPointerNull final : public ConstantData {
535 friend class Constant;
536
537 explicit ConstantPointerNull(PointerType *T)
538 : ConstantData(T, Value::ConstantPointerNullVal) {}
539
540 void destroyConstantImpl();
541
542public:
543 ConstantPointerNull(const ConstantPointerNull &) = delete;
544
545 /// Static factory methods - Return objects of the specified value
546 static ConstantPointerNull *get(PointerType *T);
547
548 /// Specialize the getType() method to always return an PointerType,
549 /// which reduces the amount of casting needed in parts of the compiler.
550 inline PointerType *getType() const {
551 return cast<PointerType>(Value::getType());
552 }
553
554 /// Methods for support type inquiry through isa, cast, and dyn_cast:
555 static bool classof(const Value *V) {
556 return V->getValueID() == ConstantPointerNullVal;
557 }
558};
559
560//===----------------------------------------------------------------------===//
561/// ConstantDataSequential - A vector or array constant whose element type is a
562/// simple 1/2/4/8-byte integer or half/bfloat/float/double, and whose elements
563/// are just simple data values (i.e. ConstantInt/ConstantFP). This Constant
564/// node has no operands because it stores all of the elements of the constant
565/// as densely packed data, instead of as Value*'s.
566///
567/// This is the common base class of ConstantDataArray and ConstantDataVector.
568///
569class ConstantDataSequential : public ConstantData {
570 friend class LLVMContextImpl;
571 friend class Constant;
572
573 /// A pointer to the bytes underlying this constant (which is owned by the
574 /// uniquing StringMap).
575 const char *DataElements;
576
577 /// This forms a link list of ConstantDataSequential nodes that have
578 /// the same value but different type. For example, 0,0,0,1 could be a 4
579 /// element array of i8, or a 1-element array of i32. They'll both end up in
580 /// the same StringMap bucket, linked up.
581 std::unique_ptr<ConstantDataSequential> Next;
582
583 void destroyConstantImpl();
584
585protected:
586 explicit ConstantDataSequential(Type *ty, ValueTy VT, const char *Data)
587 : ConstantData(ty, VT), DataElements(Data) {}
588
589 static Constant *getImpl(StringRef Bytes, Type *Ty);
590
591public:
592 ConstantDataSequential(const ConstantDataSequential &) = delete;
593
594 /// Return true if a ConstantDataSequential can be formed with a vector or
595 /// array of the specified element type.
596 /// ConstantDataArray only works with normal float and int types that are
597 /// stored densely in memory, not with things like i42 or x86_f80.
598 static bool isElementTypeCompatible(Type *Ty);
599
600 /// If this is a sequential container of integers (of any size), return the
601 /// specified element in the low bits of a uint64_t.
602 uint64_t getElementAsInteger(unsigned i) const;
603
604 /// If this is a sequential container of integers (of any size), return the
605 /// specified element as an APInt.
606 APInt getElementAsAPInt(unsigned i) const;
607
608 /// If this is a sequential container of floating point type, return the
609 /// specified element as an APFloat.
610 APFloat getElementAsAPFloat(unsigned i) const;
611
612 /// If this is an sequential container of floats, return the specified element
613 /// as a float.
614 float getElementAsFloat(unsigned i) const;
615
616 /// If this is an sequential container of doubles, return the specified
617 /// element as a double.
618 double getElementAsDouble(unsigned i) const;
619
620 /// Return a Constant for a specified index's element.
621 /// Note that this has to compute a new constant to return, so it isn't as
622 /// efficient as getElementAsInteger/Float/Double.
623 Constant *getElementAsConstant(unsigned i) const;
624
625 /// Return the element type of the array/vector.
626 Type *getElementType() const;
627
628 /// Return the number of elements in the array or vector.
629 unsigned getNumElements() const;
630
631 /// Return the size (in bytes) of each element in the array/vector.
632 /// The size of the elements is known to be a multiple of one byte.
633 uint64_t getElementByteSize() const;
634
635 /// This method returns true if this is an array of \p CharSize integers.
636 bool isString(unsigned CharSize = 8) const;
637
638 /// This method returns true if the array "isString", ends with a null byte,
639 /// and does not contains any other null bytes.
640 bool isCString() const;
641
642 /// If this array is isString(), then this method returns the array as a
643 /// StringRef. Otherwise, it asserts out.
644 StringRef getAsString() const {
645 assert(isString() && "Not a string")(static_cast <bool> (isString() && "Not a string"
) ? void (0) : __assert_fail ("isString() && \"Not a string\""
, "llvm/include/llvm/IR/Constants.h", 645, __extension__ __PRETTY_FUNCTION__
))
;
646 return getRawDataValues();
647 }
648
649 /// If this array is isCString(), then this method returns the array (without
650 /// the trailing null byte) as a StringRef. Otherwise, it asserts out.
651 StringRef getAsCString() const {
652 assert(isCString() && "Isn't a C string")(static_cast <bool> (isCString() && "Isn't a C string"
) ? void (0) : __assert_fail ("isCString() && \"Isn't a C string\""
, "llvm/include/llvm/IR/Constants.h", 652, __extension__ __PRETTY_FUNCTION__
))
;
653 StringRef Str = getAsString();
654 return Str.substr(0, Str.size() - 1);
655 }
656
657 /// Return the raw, underlying, bytes of this data. Note that this is an
658 /// extremely tricky thing to work with, as it exposes the host endianness of
659 /// the data elements.
660 StringRef getRawDataValues() const;
661
662 /// Methods for support type inquiry through isa, cast, and dyn_cast:
663 static bool classof(const Value *V) {
664 return V->getValueID() == ConstantDataArrayVal ||
665 V->getValueID() == ConstantDataVectorVal;
666 }
667
668private:
669 const char *getElementPointer(unsigned Elt) const;
670};
671
672//===----------------------------------------------------------------------===//
673/// An array constant whose element type is a simple 1/2/4/8-byte integer or
674/// float/double, and whose elements are just simple data values
675/// (i.e. ConstantInt/ConstantFP). This Constant node has no operands because it
676/// stores all of the elements of the constant as densely packed data, instead
677/// of as Value*'s.
678class ConstantDataArray final : public ConstantDataSequential {
679 friend class ConstantDataSequential;
680
681 explicit ConstantDataArray(Type *ty, const char *Data)
682 : ConstantDataSequential(ty, ConstantDataArrayVal, Data) {}
683
684public:
685 ConstantDataArray(const ConstantDataArray &) = delete;
686
687 /// get() constructor - Return a constant with array type with an element
688 /// count and element type matching the ArrayRef passed in. Note that this
689 /// can return a ConstantAggregateZero object.
690 template <typename ElementTy>
691 static Constant *get(LLVMContext &Context, ArrayRef<ElementTy> Elts) {
692 const char *Data = reinterpret_cast<const char *>(Elts.data());
693 return getRaw(StringRef(Data, Elts.size() * sizeof(ElementTy)), Elts.size(),
694 Type::getScalarTy<ElementTy>(Context));
695 }
696
697 /// get() constructor - ArrayTy needs to be compatible with
698 /// ArrayRef<ElementTy>. Calls get(LLVMContext, ArrayRef<ElementTy>).
699 template <typename ArrayTy>
700 static Constant *get(LLVMContext &Context, ArrayTy &Elts) {
701 return ConstantDataArray::get(Context, makeArrayRef(Elts));
702 }
703
704 /// getRaw() constructor - Return a constant with array type with an element
705 /// count and element type matching the NumElements and ElementTy parameters
706 /// passed in. Note that this can return a ConstantAggregateZero object.
707 /// ElementTy must be one of i8/i16/i32/i64/half/bfloat/float/double. Data is
708 /// the buffer containing the elements. Be careful to make sure Data uses the
709 /// right endianness, the buffer will be used as-is.
710 static Constant *getRaw(StringRef Data, uint64_t NumElements,
711 Type *ElementTy) {
712 Type *Ty = ArrayType::get(ElementTy, NumElements);
713 return getImpl(Data, Ty);
714 }
715
716 /// getFP() constructors - Return a constant of array type with a float
717 /// element type taken from argument `ElementType', and count taken from
718 /// argument `Elts'. The amount of bits of the contained type must match the
719 /// number of bits of the type contained in the passed in ArrayRef.
720 /// (i.e. half or bfloat for 16bits, float for 32bits, double for 64bits) Note
721 /// that this can return a ConstantAggregateZero object.
722 static Constant *getFP(Type *ElementType, ArrayRef<uint16_t> Elts);
723 static Constant *getFP(Type *ElementType, ArrayRef<uint32_t> Elts);
724 static Constant *getFP(Type *ElementType, ArrayRef<uint64_t> Elts);
725
726 /// This method constructs a CDS and initializes it with a text string.
727 /// The default behavior (AddNull==true) causes a null terminator to
728 /// be placed at the end of the array (increasing the length of the string by
729 /// one more than the StringRef would normally indicate. Pass AddNull=false
730 /// to disable this behavior.
731 static Constant *getString(LLVMContext &Context, StringRef Initializer,
732 bool AddNull = true);
733
734 /// Specialize the getType() method to always return an ArrayType,
735 /// which reduces the amount of casting needed in parts of the compiler.
736 inline ArrayType *getType() const {
737 return cast<ArrayType>(Value::getType());
738 }
739
740 /// Methods for support type inquiry through isa, cast, and dyn_cast:
741 static bool classof(const Value *V) {
742 return V->getValueID() == ConstantDataArrayVal;
743 }
744};
745
746//===----------------------------------------------------------------------===//
747/// A vector constant whose element type is a simple 1/2/4/8-byte integer or
748/// float/double, and whose elements are just simple data values
749/// (i.e. ConstantInt/ConstantFP). This Constant node has no operands because it
750/// stores all of the elements of the constant as densely packed data, instead
751/// of as Value*'s.
752class ConstantDataVector final : public ConstantDataSequential {
753 friend class ConstantDataSequential;
754
755 explicit ConstantDataVector(Type *ty, const char *Data)
756 : ConstantDataSequential(ty, ConstantDataVectorVal, Data),
757 IsSplatSet(false) {}
758 // Cache whether or not the constant is a splat.
759 mutable bool IsSplatSet : 1;
760 mutable bool IsSplat : 1;
761 bool isSplatData() const;
762
763public:
764 ConstantDataVector(const ConstantDataVector &) = delete;
765
766 /// get() constructors - Return a constant with vector type with an element
767 /// count and element type matching the ArrayRef passed in. Note that this
768 /// can return a ConstantAggregateZero object.
769 static Constant *get(LLVMContext &Context, ArrayRef<uint8_t> Elts);
770 static Constant *get(LLVMContext &Context, ArrayRef<uint16_t> Elts);
771 static Constant *get(LLVMContext &Context, ArrayRef<uint32_t> Elts);
772 static Constant *get(LLVMContext &Context, ArrayRef<uint64_t> Elts);
773 static Constant *get(LLVMContext &Context, ArrayRef<float> Elts);
774 static Constant *get(LLVMContext &Context, ArrayRef<double> Elts);
775
776 /// getRaw() constructor - Return a constant with vector type with an element
777 /// count and element type matching the NumElements and ElementTy parameters
778 /// passed in. Note that this can return a ConstantAggregateZero object.
779 /// ElementTy must be one of i8/i16/i32/i64/half/bfloat/float/double. Data is
780 /// the buffer containing the elements. Be careful to make sure Data uses the
781 /// right endianness, the buffer will be used as-is.
782 static Constant *getRaw(StringRef Data, uint64_t NumElements,
783 Type *ElementTy) {
784 Type *Ty = VectorType::get(ElementTy, ElementCount::getFixed(NumElements));
785 return getImpl(Data, Ty);
786 }
787
788 /// getFP() constructors - Return a constant of vector type with a float
789 /// element type taken from argument `ElementType', and count taken from
790 /// argument `Elts'. The amount of bits of the contained type must match the
791 /// number of bits of the type contained in the passed in ArrayRef.
792 /// (i.e. half or bfloat for 16bits, float for 32bits, double for 64bits) Note
793 /// that this can return a ConstantAggregateZero object.
794 static Constant *getFP(Type *ElementType, ArrayRef<uint16_t> Elts);
795 static Constant *getFP(Type *ElementType, ArrayRef<uint32_t> Elts);
796 static Constant *getFP(Type *ElementType, ArrayRef<uint64_t> Elts);
797
798 /// Return a ConstantVector with the specified constant in each element.
799 /// The specified constant has to be a of a compatible type (i8/i16/
800 /// i32/i64/half/bfloat/float/double) and must be a ConstantFP or ConstantInt.
801 static Constant *getSplat(unsigned NumElts, Constant *Elt);
802
803 /// Returns true if this is a splat constant, meaning that all elements have
804 /// the same value.
805 bool isSplat() const;
806
807 /// If this is a splat constant, meaning that all of the elements have the
808 /// same value, return that value. Otherwise return NULL.
809 Constant *getSplatValue() const;
810
811 /// Specialize the getType() method to always return a FixedVectorType,
812 /// which reduces the amount of casting needed in parts of the compiler.
813 inline FixedVectorType *getType() const {
814 return cast<FixedVectorType>(Value::getType());
815 }
816
817 /// Methods for support type inquiry through isa, cast, and dyn_cast:
818 static bool classof(const Value *V) {
819 return V->getValueID() == ConstantDataVectorVal;
820 }
821};
822
823//===----------------------------------------------------------------------===//
824/// A constant token which is empty
825///
826class ConstantTokenNone final : public ConstantData {
827 friend class Constant;
828
829 explicit ConstantTokenNone(LLVMContext &Context)
830 : ConstantData(Type::getTokenTy(Context), ConstantTokenNoneVal) {}
831
832 void destroyConstantImpl();
833
834public:
835 ConstantTokenNone(const ConstantTokenNone &) = delete;
836
837 /// Return the ConstantTokenNone.
838 static ConstantTokenNone *get(LLVMContext &Context);
839
840 /// Methods to support type inquiry through isa, cast, and dyn_cast.
841 static bool classof(const Value *V) {
842 return V->getValueID() == ConstantTokenNoneVal;
843 }
844};
845
846/// The address of a basic block.
847///
848class BlockAddress final : public Constant {
849 friend class Constant;
850
851 BlockAddress(Function *F, BasicBlock *BB);
852
853 void *operator new(size_t S) { return User::operator new(S, 2); }
854
855 void destroyConstantImpl();
856 Value *handleOperandChangeImpl(Value *From, Value *To);
857
858public:
859 void operator delete(void *Ptr) { User::operator delete(Ptr); }
860
861 /// Return a BlockAddress for the specified function and basic block.
862 static BlockAddress *get(Function *F, BasicBlock *BB);
863
864 /// Return a BlockAddress for the specified basic block. The basic
865 /// block must be embedded into a function.
866 static BlockAddress *get(BasicBlock *BB);
867
868 /// Lookup an existing \c BlockAddress constant for the given BasicBlock.
869 ///
870 /// \returns 0 if \c !BB->hasAddressTaken(), otherwise the \c BlockAddress.
871 static BlockAddress *lookup(const BasicBlock *BB);
872
873 /// Transparently provide more efficient getOperand methods.
874 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
875
876 Function *getFunction() const { return (Function *)Op<0>().get(); }
877 BasicBlock *getBasicBlock() const { return (BasicBlock *)Op<1>().get(); }
878
879 /// Methods for support type inquiry through isa, cast, and dyn_cast:
880 static bool classof(const Value *V) {
881 return V->getValueID() == BlockAddressVal;
882 }
883};
884
885template <>
886struct OperandTraits<BlockAddress>
887 : public FixedNumOperandTraits<BlockAddress, 2> {};
888
889DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BlockAddress, Value)BlockAddress::op_iterator BlockAddress::op_begin() { return OperandTraits
<BlockAddress>::op_begin(this); } BlockAddress::const_op_iterator
BlockAddress::op_begin() const { return OperandTraits<BlockAddress
>::op_begin(const_cast<BlockAddress*>(this)); } BlockAddress
::op_iterator BlockAddress::op_end() { return OperandTraits<
BlockAddress>::op_end(this); } BlockAddress::const_op_iterator
BlockAddress::op_end() const { return OperandTraits<BlockAddress
>::op_end(const_cast<BlockAddress*>(this)); } Value *
BlockAddress::getOperand(unsigned i_nocapture) const { (static_cast
<bool> (i_nocapture < OperandTraits<BlockAddress
>::operands(this) && "getOperand() out of range!")
? void (0) : __assert_fail ("i_nocapture < OperandTraits<BlockAddress>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Constants.h", 889, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<BlockAddress
>::op_begin(const_cast<BlockAddress*>(this))[i_nocapture
].get()); } void BlockAddress::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<BlockAddress>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<BlockAddress>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Constants.h", 889, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<BlockAddress>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned BlockAddress::getNumOperands() const
{ return OperandTraits<BlockAddress>::operands(this); }
template <int Idx_nocapture> Use &BlockAddress::Op
() { return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &BlockAddress::Op() const
{ return this->OpFrom<Idx_nocapture>(this); }
890
891/// Wrapper for a function that represents a value that
892/// functionally represents the original function. This can be a function,
893/// global alias to a function, or an ifunc.
894class DSOLocalEquivalent final : public Constant {
895 friend class Constant;
896
897 DSOLocalEquivalent(GlobalValue *GV);
898
899 void *operator new(size_t S) { return User::operator new(S, 1); }
900
901 void destroyConstantImpl();
902 Value *handleOperandChangeImpl(Value *From, Value *To);
903
904public:
905 void operator delete(void *Ptr) { User::operator delete(Ptr); }
906
907 /// Return a DSOLocalEquivalent for the specified global value.
908 static DSOLocalEquivalent *get(GlobalValue *GV);
909
910 /// Transparently provide more efficient getOperand methods.
911 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
912
913 GlobalValue *getGlobalValue() const {
914 return cast<GlobalValue>(Op<0>().get());
915 }
916
917 /// Methods for support type inquiry through isa, cast, and dyn_cast:
918 static bool classof(const Value *V) {
919 return V->getValueID() == DSOLocalEquivalentVal;
920 }
921};
922
923template <>
924struct OperandTraits<DSOLocalEquivalent>
925 : public FixedNumOperandTraits<DSOLocalEquivalent, 1> {};
926
927DEFINE_TRANSPARENT_OPERAND_ACCESSORS(DSOLocalEquivalent, Value)DSOLocalEquivalent::op_iterator DSOLocalEquivalent::op_begin(
) { return OperandTraits<DSOLocalEquivalent>::op_begin(
this); } DSOLocalEquivalent::const_op_iterator DSOLocalEquivalent
::op_begin() const { return OperandTraits<DSOLocalEquivalent
>::op_begin(const_cast<DSOLocalEquivalent*>(this)); }
DSOLocalEquivalent::op_iterator DSOLocalEquivalent::op_end()
{ return OperandTraits<DSOLocalEquivalent>::op_end(this
); } DSOLocalEquivalent::const_op_iterator DSOLocalEquivalent
::op_end() const { return OperandTraits<DSOLocalEquivalent
>::op_end(const_cast<DSOLocalEquivalent*>(this)); } Value
*DSOLocalEquivalent::getOperand(unsigned i_nocapture) const {
(static_cast <bool> (i_nocapture < OperandTraits<
DSOLocalEquivalent>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<DSOLocalEquivalent>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Constants.h", 927, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<DSOLocalEquivalent
>::op_begin(const_cast<DSOLocalEquivalent*>(this))[i_nocapture
].get()); } void DSOLocalEquivalent::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<DSOLocalEquivalent>::operands(this)
&& "setOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<DSOLocalEquivalent>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Constants.h", 927, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<DSOLocalEquivalent>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned DSOLocalEquivalent::getNumOperands
() const { return OperandTraits<DSOLocalEquivalent>::operands
(this); } template <int Idx_nocapture> Use &DSOLocalEquivalent
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &DSOLocalEquivalent
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
928
929/// Wrapper for a value that won't be replaced with a CFI jump table
930/// pointer in LowerTypeTestsModule.
931class NoCFIValue final : public Constant {
932 friend class Constant;
933
934 NoCFIValue(GlobalValue *GV);
935
936 void *operator new(size_t S) { return User::operator new(S, 1); }
937
938 void destroyConstantImpl();
939 Value *handleOperandChangeImpl(Value *From, Value *To);
940
941public:
942 /// Return a NoCFIValue for the specified function.
943 static NoCFIValue *get(GlobalValue *GV);
944
945 /// Transparently provide more efficient getOperand methods.
946 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
947
948 GlobalValue *getGlobalValue() const {
949 return cast<GlobalValue>(Op<0>().get());
950 }
951
952 /// Methods for support type inquiry through isa, cast, and dyn_cast:
953 static bool classof(const Value *V) {
954 return V->getValueID() == NoCFIValueVal;
955 }
956};
957
958template <>
959struct OperandTraits<NoCFIValue> : public FixedNumOperandTraits<NoCFIValue, 1> {
960};
961
962DEFINE_TRANSPARENT_OPERAND_ACCESSORS(NoCFIValue, Value)NoCFIValue::op_iterator NoCFIValue::op_begin() { return OperandTraits
<NoCFIValue>::op_begin(this); } NoCFIValue::const_op_iterator
NoCFIValue::op_begin() const { return OperandTraits<NoCFIValue
>::op_begin(const_cast<NoCFIValue*>(this)); } NoCFIValue
::op_iterator NoCFIValue::op_end() { return OperandTraits<
NoCFIValue>::op_end(this); } NoCFIValue::const_op_iterator
NoCFIValue::op_end() const { return OperandTraits<NoCFIValue
>::op_end(const_cast<NoCFIValue*>(this)); } Value *NoCFIValue
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<NoCFIValue>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<NoCFIValue>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Constants.h", 962, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<NoCFIValue
>::op_begin(const_cast<NoCFIValue*>(this))[i_nocapture
].get()); } void NoCFIValue::setOperand(unsigned i_nocapture,
Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<NoCFIValue>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<NoCFIValue>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Constants.h", 962, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<NoCFIValue>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned NoCFIValue::getNumOperands() const
{ return OperandTraits<NoCFIValue>::operands(this); } template
<int Idx_nocapture> Use &NoCFIValue::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &NoCFIValue::Op() const { return
this->OpFrom<Idx_nocapture>(this); }
963
964//===----------------------------------------------------------------------===//
965/// A constant value that is initialized with an expression using
966/// other constant values.
967///
968/// This class uses the standard Instruction opcodes to define the various
969/// constant expressions. The Opcode field for the ConstantExpr class is
970/// maintained in the Value::SubclassData field.
971class ConstantExpr : public Constant {
972 friend struct ConstantExprKeyType;
973 friend class Constant;
974
975 void destroyConstantImpl();
976 Value *handleOperandChangeImpl(Value *From, Value *To);
977
978protected:
979 ConstantExpr(Type *ty, unsigned Opcode, Use *Ops, unsigned NumOps)
980 : Constant(ty, ConstantExprVal, Ops, NumOps) {
981 // Operation type (an Instruction opcode) is stored as the SubclassData.
982 setValueSubclassData(Opcode);
983 }
984
985 ~ConstantExpr() = default;
986
987public:
988 // Static methods to construct a ConstantExpr of different kinds. Note that
989 // these methods may return a object that is not an instance of the
990 // ConstantExpr class, because they will attempt to fold the constant
991 // expression into something simpler if possible.
992
993 /// getAlignOf constant expr - computes the alignment of a type in a target
994 /// independent way (Note: the return type is an i64).
995 static Constant *getAlignOf(Type *Ty);
996
997 /// getSizeOf constant expr - computes the (alloc) size of a type (in
998 /// address-units, not bits) in a target independent way (Note: the return
999 /// type is an i64).
1000 ///
1001 static Constant *getSizeOf(Type *Ty);
1002
1003 /// getOffsetOf constant expr - computes the offset of a struct field in a
1004 /// target independent way (Note: the return type is an i64).
1005 ///
1006 static Constant *getOffsetOf(StructType *STy, unsigned FieldNo);
1007
1008 /// getOffsetOf constant expr - This is a generalized form of getOffsetOf,
1009 /// which supports any aggregate type, and any Constant index.
1010 ///
1011 static Constant *getOffsetOf(Type *Ty, Constant *FieldNo);
1012
1013 static Constant *getNeg(Constant *C, bool HasNUW = false,
1014 bool HasNSW = false);
1015 static Constant *getFNeg(Constant *C);
1016 static Constant *getNot(Constant *C);
1017 static Constant *getAdd(Constant *C1, Constant *C2, bool HasNUW = false,
1018 bool HasNSW = false);
1019 static Constant *getFAdd(Constant *C1, Constant *C2);
1020 static Constant *getSub(Constant *C1, Constant *C2, bool HasNUW = false,
1021 bool HasNSW = false);
1022 static Constant *getFSub(Constant *C1, Constant *C2);
1023 static Constant *getMul(Constant *C1, Constant *C2, bool HasNUW = false,
1024 bool HasNSW = false);
1025 static Constant *getFMul(Constant *C1, Constant *C2);
1026 static Constant *getUDiv(Constant *C1, Constant *C2, bool isExact = false);
1027 static Constant *getSDiv(Constant *C1, Constant *C2, bool isExact = false);
1028 static Constant *getFDiv(Constant *C1, Constant *C2);
1029 static Constant *getURem(Constant *C1, Constant *C2);
1030 static Constant *getSRem(Constant *C1, Constant *C2);
1031 static Constant *getFRem(Constant *C1, Constant *C2);
1032 static Constant *getAnd(Constant *C1, Constant *C2);
1033 static Constant *getOr(Constant *C1, Constant *C2);
1034 static Constant *getXor(Constant *C1, Constant *C2);
1035 static Constant *getUMin(Constant *C1, Constant *C2);
1036 static Constant *getShl(Constant *C1, Constant *C2, bool HasNUW = false,
1037 bool HasNSW = false);
1038 static Constant *getLShr(Constant *C1, Constant *C2, bool isExact = false);
1039 static Constant *getAShr(Constant *C1, Constant *C2, bool isExact = false);
1040 static Constant *getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced = false);
1041 static Constant *getSExt(Constant *C, Type *Ty, bool OnlyIfReduced = false);
1042 static Constant *getZExt(Constant *C, Type *Ty, bool OnlyIfReduced = false);
1043 static Constant *getFPTrunc(Constant *C, Type *Ty,
1044 bool OnlyIfReduced = false);
1045 static Constant *getFPExtend(Constant *C, Type *Ty,
1046 bool OnlyIfReduced = false);
1047 static Constant *getUIToFP(Constant *C, Type *Ty, bool OnlyIfReduced = false);
1048 static Constant *getSIToFP(Constant *C, Type *Ty, bool OnlyIfReduced = false);
1049 static Constant *getFPToUI(Constant *C, Type *Ty, bool OnlyIfReduced = false);
1050 static Constant *getFPToSI(Constant *C, Type *Ty, bool OnlyIfReduced = false);
1051 static Constant *getPtrToInt(Constant *C, Type *Ty,
1052 bool OnlyIfReduced = false);
1053 static Constant *getIntToPtr(Constant *C, Type *Ty,
1054 bool OnlyIfReduced = false);
1055 static Constant *getBitCast(Constant *C, Type *Ty,
1056 bool OnlyIfReduced = false);
1057 static Constant *getAddrSpaceCast(Constant *C, Type *Ty,
1058 bool OnlyIfReduced = false);
1059
1060 static Constant *getNSWNeg(Constant *C) { return getNeg(C, false, true); }
1061 static Constant *getNUWNeg(Constant *C) { return getNeg(C, true, false); }
1062
1063 static Constant *getNSWAdd(Constant *C1, Constant *C2) {
1064 return getAdd(C1, C2, false, true);
1065 }
1066
1067 static Constant *getNUWAdd(Constant *C1, Constant *C2) {
1068 return getAdd(C1, C2, true, false);
1069 }
1070
1071 static Constant *getNSWSub(Constant *C1, Constant *C2) {
1072 return getSub(C1, C2, false, true);
1073 }
1074
1075 static Constant *getNUWSub(Constant *C1, Constant *C2) {
1076 return getSub(C1, C2, true, false);
1077 }
1078
1079 static Constant *getNSWMul(Constant *C1, Constant *C2) {
1080 return getMul(C1, C2, false, true);
1081 }
1082
1083 static Constant *getNUWMul(Constant *C1, Constant *C2) {
1084 return getMul(C1, C2, true, false);
1085 }
1086
1087 static Constant *getNSWShl(Constant *C1, Constant *C2) {
1088 return getShl(C1, C2, false, true);
1089 }
1090
1091 static Constant *getNUWShl(Constant *C1, Constant *C2) {
1092 return getShl(C1, C2, true, false);
1093 }
1094
1095 static Constant *getExactSDiv(Constant *C1, Constant *C2) {
1096 return getSDiv(C1, C2, true);
1097 }
1098
1099 static Constant *getExactUDiv(Constant *C1, Constant *C2) {
1100 return getUDiv(C1, C2, true);
1101 }
1102
1103 static Constant *getExactAShr(Constant *C1, Constant *C2) {
1104 return getAShr(C1, C2, true);
1105 }
1106
1107 static Constant *getExactLShr(Constant *C1, Constant *C2) {
1108 return getLShr(C1, C2, true);
1109 }
1110
1111 /// If C is a scalar/fixed width vector of known powers of 2, then this
1112 /// function returns a new scalar/fixed width vector obtained from logBase2
1113 /// of C. Undef vector elements are set to zero.
1114 /// Return a null pointer otherwise.
1115 static Constant *getExactLogBase2(Constant *C);
1116
1117 /// Return the identity constant for a binary opcode.
1118 /// The identity constant C is defined as X op C = X and C op X = X for every
1119 /// X when the binary operation is commutative. If the binop is not
1120 /// commutative, callers can acquire the operand 1 identity constant by
1121 /// setting AllowRHSConstant to true. For example, any shift has a zero
1122 /// identity constant for operand 1: X shift 0 = X.
1123 /// Return nullptr if the operator does not have an identity constant.
1124 static Constant *getBinOpIdentity(unsigned Opcode, Type *Ty,
1125 bool AllowRHSConstant = false);
1126
1127 /// Return the absorbing element for the given binary
1128 /// operation, i.e. a constant C such that X op C = C and C op X = C for
1129 /// every X. For example, this returns zero for integer multiplication.
1130 /// It returns null if the operator doesn't have an absorbing element.
1131 static Constant *getBinOpAbsorber(unsigned Opcode, Type *Ty);
1132
1133 /// Transparently provide more efficient getOperand methods.
1134 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant)public: inline Constant *getOperand(unsigned) const; inline void
setOperand(unsigned, Constant*); inline op_iterator op_begin
(); inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1135
1136 /// Convenience function for getting a Cast operation.
1137 ///
1138 /// \param ops The opcode for the conversion
1139 /// \param C The constant to be converted
1140 /// \param Ty The type to which the constant is converted
1141 /// \param OnlyIfReduced see \a getWithOperands() docs.
1142 static Constant *getCast(unsigned ops, Constant *C, Type *Ty,
1143 bool OnlyIfReduced = false);
1144
1145 // Create a ZExt or BitCast cast constant expression
1146 static Constant *
1147 getZExtOrBitCast(Constant *C, ///< The constant to zext or bitcast
1148 Type *Ty ///< The type to zext or bitcast C to
1149 );
1150
1151 // Create a SExt or BitCast cast constant expression
1152 static Constant *
1153 getSExtOrBitCast(Constant *C, ///< The constant to sext or bitcast
1154 Type *Ty ///< The type to sext or bitcast C to
1155 );
1156
1157 // Create a Trunc or BitCast cast constant expression
1158 static Constant *
1159 getTruncOrBitCast(Constant *C, ///< The constant to trunc or bitcast
1160 Type *Ty ///< The type to trunc or bitcast C to
1161 );
1162
1163 /// Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant
1164 /// expression.
1165 static Constant *
1166 getPointerCast(Constant *C, ///< The pointer value to be casted (operand 0)
1167 Type *Ty ///< The type to which cast should be made
1168 );
1169
1170 /// Create a BitCast or AddrSpaceCast for a pointer type depending on
1171 /// the address space.
1172 static Constant *getPointerBitCastOrAddrSpaceCast(
1173 Constant *C, ///< The constant to addrspacecast or bitcast
1174 Type *Ty ///< The type to bitcast or addrspacecast C to
1175 );
1176
1177 /// Create a ZExt, Bitcast or Trunc for integer -> integer casts
1178 static Constant *
1179 getIntegerCast(Constant *C, ///< The integer constant to be casted
1180 Type *Ty, ///< The integer type to cast to
1181 bool IsSigned ///< Whether C should be treated as signed or not
1182 );
1183
1184 /// Create a FPExt, Bitcast or FPTrunc for fp -> fp casts
1185 static Constant *getFPCast(Constant *C, ///< The integer constant to be casted
1186 Type *Ty ///< The integer type to cast to
1187 );
1188
1189 /// Return true if this is a convert constant expression
1190 bool isCast() const;
1191
1192 /// Return true if this is a compare constant expression
1193 bool isCompare() const;
1194
1195 /// Return true if this is an insertvalue or extractvalue expression,
1196 /// and the getIndices() method may be used.
1197 bool hasIndices() const;
1198
1199 /// Select constant expr
1200 ///
1201 /// \param OnlyIfReducedTy see \a getWithOperands() docs.
1202 static Constant *getSelect(Constant *C, Constant *V1, Constant *V2,
1203 Type *OnlyIfReducedTy = nullptr);
1204
1205 /// get - Return a unary operator constant expression,
1206 /// folding if possible.
1207 ///
1208 /// \param OnlyIfReducedTy see \a getWithOperands() docs.
1209 static Constant *get(unsigned Opcode, Constant *C1, unsigned Flags = 0,
1210 Type *OnlyIfReducedTy = nullptr);
1211
1212 /// get - Return a binary or shift operator constant expression,
1213 /// folding if possible.
1214 ///
1215 /// \param OnlyIfReducedTy see \a getWithOperands() docs.
1216 static Constant *get(unsigned Opcode, Constant *C1, Constant *C2,
1217 unsigned Flags = 0, Type *OnlyIfReducedTy = nullptr);
1218
1219 /// Return an ICmp or FCmp comparison operator constant expression.
1220 ///
1221 /// \param OnlyIfReduced see \a getWithOperands() docs.
1222 static Constant *getCompare(unsigned short pred, Constant *C1, Constant *C2,
1223 bool OnlyIfReduced = false);
1224
1225 /// get* - Return some common constants without having to
1226 /// specify the full Instruction::OPCODE identifier.
1227 ///
1228 static Constant *getICmp(unsigned short pred, Constant *LHS, Constant *RHS,
1229 bool OnlyIfReduced = false);
1230 static Constant *getFCmp(unsigned short pred, Constant *LHS, Constant *RHS,
1231 bool OnlyIfReduced = false);
1232
1233 /// Getelementptr form. Value* is only accepted for convenience;
1234 /// all elements must be Constants.
1235 ///
1236 /// \param InRangeIndex the inrange index if present or None.
1237 /// \param OnlyIfReducedTy see \a getWithOperands() docs.
1238 static Constant *getGetElementPtr(Type *Ty, Constant *C,
1239 ArrayRef<Constant *> IdxList,
1240 bool InBounds = false,
1241 Optional<unsigned> InRangeIndex = None,
1242 Type *OnlyIfReducedTy = nullptr) {
1243 return getGetElementPtr(
1244 Ty, C, makeArrayRef((Value *const *)IdxList.data(), IdxList.size()),
1245 InBounds, InRangeIndex, OnlyIfReducedTy);
1246 }
1247 static Constant *getGetElementPtr(Type *Ty, Constant *C, Constant *Idx,
1248 bool InBounds = false,
1249 Optional<unsigned> InRangeIndex = None,
1250 Type *OnlyIfReducedTy = nullptr) {
1251 // This form of the function only exists to avoid ambiguous overload
1252 // warnings about whether to convert Idx to ArrayRef<Constant *> or
1253 // ArrayRef<Value *>.
1254 return getGetElementPtr(Ty, C, cast<Value>(Idx), InBounds, InRangeIndex,
1255 OnlyIfReducedTy);
1256 }
1257 static Constant *getGetElementPtr(Type *Ty, Constant *C,
1258 ArrayRef<Value *> IdxList,
1259 bool InBounds = false,
1260 Optional<unsigned> InRangeIndex = None,
1261 Type *OnlyIfReducedTy = nullptr);
1262
1263 /// Create an "inbounds" getelementptr. See the documentation for the
1264 /// "inbounds" flag in LangRef.html for details.
1265 static Constant *getInBoundsGetElementPtr(Type *Ty, Constant *C,
1266 ArrayRef<Constant *> IdxList) {
1267 return getGetElementPtr(Ty, C, IdxList, true);
1268 }
1269 static Constant *getInBoundsGetElementPtr(Type *Ty, Constant *C,
1270 Constant *Idx) {
1271 // This form of the function only exists to avoid ambiguous overload
1272 // warnings about whether to convert Idx to ArrayRef<Constant *> or
1273 // ArrayRef<Value *>.
1274 return getGetElementPtr(Ty, C, Idx, true);
1275 }
1276 static Constant *getInBoundsGetElementPtr(Type *Ty, Constant *C,
1277 ArrayRef<Value *> IdxList) {
1278 return getGetElementPtr(Ty, C, IdxList, true);
1279 }
1280
1281 static Constant *getExtractElement(Constant *Vec, Constant *Idx,
1282 Type *OnlyIfReducedTy = nullptr);
1283 static Constant *getInsertElement(Constant *Vec, Constant *Elt, Constant *Idx,
1284 Type *OnlyIfReducedTy = nullptr);
1285 static Constant *getShuffleVector(Constant *V1, Constant *V2,
1286 ArrayRef<int> Mask,
1287 Type *OnlyIfReducedTy = nullptr);
1288 static Constant *getExtractValue(Constant *Agg, ArrayRef<unsigned> Idxs,
1289 Type *OnlyIfReducedTy = nullptr);
1290 static Constant *getInsertValue(Constant *Agg, Constant *Val,
1291 ArrayRef<unsigned> Idxs,
1292 Type *OnlyIfReducedTy = nullptr);
1293
1294 /// Return the opcode at the root of this constant expression
1295 unsigned getOpcode() const { return getSubclassDataFromValue(); }
1296
1297 /// Return the ICMP or FCMP predicate value. Assert if this is not an ICMP or
1298 /// FCMP constant expression.
1299 unsigned getPredicate() const;
1300
1301 /// Assert that this is an insertvalue or exactvalue
1302 /// expression and return the list of indices.
1303 ArrayRef<unsigned> getIndices() const;
1304
1305 /// Assert that this is a shufflevector and return the mask. See class
1306 /// ShuffleVectorInst for a description of the mask representation.
1307 ArrayRef<int> getShuffleMask() const;
1308
1309 /// Assert that this is a shufflevector and return the mask.
1310 ///
1311 /// TODO: This is a temporary hack until we update the bitcode format for
1312 /// shufflevector.
1313 Constant *getShuffleMaskForBitcode() const;
1314
1315 /// Return a string representation for an opcode.
1316 const char *getOpcodeName() const;
1317
1318 /// This returns the current constant expression with the operands replaced
1319 /// with the specified values. The specified array must have the same number
1320 /// of operands as our current one.
1321 Constant *getWithOperands(ArrayRef<Constant *> Ops) const {
1322 return getWithOperands(Ops, getType());
1323 }
1324
1325 /// Get the current expression with the operands replaced.
1326 ///
1327 /// Return the current constant expression with the operands replaced with \c
1328 /// Ops and the type with \c Ty. The new operands must have the same number
1329 /// as the current ones.
1330 ///
1331 /// If \c OnlyIfReduced is \c true, nullptr will be returned unless something
1332 /// gets constant-folded, the type changes, or the expression is otherwise
1333 /// canonicalized. This parameter should almost always be \c false.
1334 Constant *getWithOperands(ArrayRef<Constant *> Ops, Type *Ty,
1335 bool OnlyIfReduced = false,
1336 Type *SrcTy = nullptr) const;
1337
1338 /// Returns an Instruction which implements the same operation as this
1339 /// ConstantExpr. If \p InsertBefore is not null, the new instruction is
1340 /// inserted before it, otherwise it is not inserted into any basic block.
1341 ///
1342 /// A better approach to this could be to have a constructor for Instruction
1343 /// which would take a ConstantExpr parameter, but that would have spread
1344 /// implementation details of ConstantExpr outside of Constants.cpp, which
1345 /// would make it harder to remove ConstantExprs altogether.
1346 Instruction *getAsInstruction(Instruction *InsertBefore = nullptr) const;
1347
1348 /// Methods for support type inquiry through isa, cast, and dyn_cast:
1349 static bool classof(const Value *V) {
1350 return V->getValueID() == ConstantExprVal;
1351 }
1352
1353private:
1354 // Shadow Value::setValueSubclassData with a private forwarding method so that
1355 // subclasses cannot accidentally use it.
1356 void setValueSubclassData(unsigned short D) {
1357 Value::setValueSubclassData(D);
1358 }
1359};
1360
1361template <>
1362struct OperandTraits<ConstantExpr>
1363 : public VariadicOperandTraits<ConstantExpr, 1> {};
1364
1365DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantExpr, Constant)ConstantExpr::op_iterator ConstantExpr::op_begin() { return OperandTraits
<ConstantExpr>::op_begin(this); } ConstantExpr::const_op_iterator
ConstantExpr::op_begin() const { return OperandTraits<ConstantExpr
>::op_begin(const_cast<ConstantExpr*>(this)); } ConstantExpr
::op_iterator ConstantExpr::op_end() { return OperandTraits<
ConstantExpr>::op_end(this); } ConstantExpr::const_op_iterator
ConstantExpr::op_end() const { return OperandTraits<ConstantExpr
>::op_end(const_cast<ConstantExpr*>(this)); } Constant
*ConstantExpr::getOperand(unsigned i_nocapture) const { (static_cast
<bool> (i_nocapture < OperandTraits<ConstantExpr
>::operands(this) && "getOperand() out of range!")
? void (0) : __assert_fail ("i_nocapture < OperandTraits<ConstantExpr>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Constants.h", 1365, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Constant>( OperandTraits<ConstantExpr
>::op_begin(const_cast<ConstantExpr*>(this))[i_nocapture
].get()); } void ConstantExpr::setOperand(unsigned i_nocapture
, Constant *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ConstantExpr>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ConstantExpr>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Constants.h", 1365, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<ConstantExpr>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned ConstantExpr::getNumOperands() const
{ return OperandTraits<ConstantExpr>::operands(this); }
template <int Idx_nocapture> Use &ConstantExpr::Op
() { return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &ConstantExpr::Op() const
{ return this->OpFrom<Idx_nocapture>(this); }
1366
1367//===----------------------------------------------------------------------===//
1368/// 'undef' values are things that do not have specified contents.
1369/// These are used for a variety of purposes, including global variable
1370/// initializers and operands to instructions. 'undef' values can occur with
1371/// any first-class type.
1372///
1373/// Undef values aren't exactly constants; if they have multiple uses, they
1374/// can appear to have different bit patterns at each use. See
1375/// LangRef.html#undefvalues for details.
1376///
1377class UndefValue : public ConstantData {
1378 friend class Constant;
1379
1380 explicit UndefValue(Type *T) : ConstantData(T, UndefValueVal) {}
1381
1382 void destroyConstantImpl();
1383
1384protected:
1385 explicit UndefValue(Type *T, ValueTy vty) : ConstantData(T, vty) {}
1386
1387public:
1388 UndefValue(const UndefValue &) = delete;
1389
1390 /// Static factory methods - Return an 'undef' object of the specified type.
1391 static UndefValue *get(Type *T);
1392
1393 /// If this Undef has array or vector type, return a undef with the right
1394 /// element type.
1395 UndefValue *getSequentialElement() const;
1396
1397 /// If this undef has struct type, return a undef with the right element type
1398 /// for the specified element.
1399 UndefValue *getStructElement(unsigned Elt) const;
1400
1401 /// Return an undef of the right value for the specified GEP index if we can,
1402 /// otherwise return null (e.g. if C is a ConstantExpr).
1403 UndefValue *getElementValue(Constant *C) const;
1404
1405 /// Return an undef of the right value for the specified GEP index.
1406 UndefValue *getElementValue(unsigned Idx) const;
1407
1408 /// Return the number of elements in the array, vector, or struct.
1409 unsigned getNumElements() const;
1410
1411 /// Methods for support type inquiry through isa, cast, and dyn_cast:
1412 static bool classof(const Value *V) {
1413 return V->getValueID() == UndefValueVal ||
1414 V->getValueID() == PoisonValueVal;
1415 }
1416};
1417
1418//===----------------------------------------------------------------------===//
1419/// In order to facilitate speculative execution, many instructions do not
1420/// invoke immediate undefined behavior when provided with illegal operands,
1421/// and return a poison value instead.
1422///
1423/// see LangRef.html#poisonvalues for details.
1424///
1425class PoisonValue final : public UndefValue {
1426 friend class Constant;
1427
1428 explicit PoisonValue(Type *T) : UndefValue(T, PoisonValueVal) {}
1429
1430 void destroyConstantImpl();
1431
1432public:
1433 PoisonValue(const PoisonValue &) = delete;
1434
1435 /// Static factory methods - Return an 'poison' object of the specified type.
1436 static PoisonValue *get(Type *T);
1437
1438 /// If this poison has array or vector type, return a poison with the right
1439 /// element type.
1440 PoisonValue *getSequentialElement() const;
1441
1442 /// If this poison has struct type, return a poison with the right element
1443 /// type for the specified element.
1444 PoisonValue *getStructElement(unsigned Elt) const;
1445
1446 /// Return an poison of the right value for the specified GEP index if we can,
1447 /// otherwise return null (e.g. if C is a ConstantExpr).
1448 PoisonValue *getElementValue(Constant *C) const;
1449
1450 /// Return an poison of the right value for the specified GEP index.
1451 PoisonValue *getElementValue(unsigned Idx) const;
1452
1453 /// Methods for support type inquiry through isa, cast, and dyn_cast:
1454 static bool classof(const Value *V) {
1455 return V->getValueID() == PoisonValueVal;
1456 }
1457};
1458
1459} // end namespace llvm
1460
1461#endif // LLVM_IR_CONSTANTS_H