File: | llvm/lib/Transforms/InstCombine/InstCombineInternal.h |
Warning: | line 411, column 5 Forming reference to null pointer |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- InstructionCombining.cpp - Combine multiple instructions -----------===// | ||||||
2 | // | ||||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||
6 | // | ||||||
7 | //===----------------------------------------------------------------------===// | ||||||
8 | // | ||||||
9 | // InstructionCombining - Combine instructions to form fewer, simple | ||||||
10 | // instructions. This pass does not modify the CFG. This pass is where | ||||||
11 | // algebraic simplification happens. | ||||||
12 | // | ||||||
13 | // This pass combines things like: | ||||||
14 | // %Y = add i32 %X, 1 | ||||||
15 | // %Z = add i32 %Y, 1 | ||||||
16 | // into: | ||||||
17 | // %Z = add i32 %X, 2 | ||||||
18 | // | ||||||
19 | // This is a simple worklist driven algorithm. | ||||||
20 | // | ||||||
21 | // This pass guarantees that the following canonicalizations are performed on | ||||||
22 | // the program: | ||||||
23 | // 1. If a binary operator has a constant operand, it is moved to the RHS | ||||||
24 | // 2. Bitwise operators with constant operands are always grouped so that | ||||||
25 | // shifts are performed first, then or's, then and's, then xor's. | ||||||
26 | // 3. Compare instructions are converted from <,>,<=,>= to ==,!= if possible | ||||||
27 | // 4. All cmp instructions on boolean values are replaced with logical ops | ||||||
28 | // 5. add X, X is represented as (X*2) => (X << 1) | ||||||
29 | // 6. Multiplies with a power-of-two constant argument are transformed into | ||||||
30 | // shifts. | ||||||
31 | // ... etc. | ||||||
32 | // | ||||||
33 | //===----------------------------------------------------------------------===// | ||||||
34 | |||||||
35 | #include "InstCombineInternal.h" | ||||||
36 | #include "llvm-c/Initialization.h" | ||||||
37 | #include "llvm-c/Transforms/InstCombine.h" | ||||||
38 | #include "llvm/ADT/APInt.h" | ||||||
39 | #include "llvm/ADT/ArrayRef.h" | ||||||
40 | #include "llvm/ADT/DenseMap.h" | ||||||
41 | #include "llvm/ADT/None.h" | ||||||
42 | #include "llvm/ADT/SmallPtrSet.h" | ||||||
43 | #include "llvm/ADT/SmallVector.h" | ||||||
44 | #include "llvm/ADT/Statistic.h" | ||||||
45 | #include "llvm/ADT/TinyPtrVector.h" | ||||||
46 | #include "llvm/Analysis/AliasAnalysis.h" | ||||||
47 | #include "llvm/Analysis/AssumptionCache.h" | ||||||
48 | #include "llvm/Analysis/BasicAliasAnalysis.h" | ||||||
49 | #include "llvm/Analysis/BlockFrequencyInfo.h" | ||||||
50 | #include "llvm/Analysis/CFG.h" | ||||||
51 | #include "llvm/Analysis/ConstantFolding.h" | ||||||
52 | #include "llvm/Analysis/EHPersonalities.h" | ||||||
53 | #include "llvm/Analysis/GlobalsModRef.h" | ||||||
54 | #include "llvm/Analysis/InstructionSimplify.h" | ||||||
55 | #include "llvm/Analysis/LazyBlockFrequencyInfo.h" | ||||||
56 | #include "llvm/Analysis/LoopInfo.h" | ||||||
57 | #include "llvm/Analysis/MemoryBuiltins.h" | ||||||
58 | #include "llvm/Analysis/OptimizationRemarkEmitter.h" | ||||||
59 | #include "llvm/Analysis/ProfileSummaryInfo.h" | ||||||
60 | #include "llvm/Analysis/TargetFolder.h" | ||||||
61 | #include "llvm/Analysis/TargetLibraryInfo.h" | ||||||
62 | #include "llvm/Analysis/TargetTransformInfo.h" | ||||||
63 | #include "llvm/Analysis/ValueTracking.h" | ||||||
64 | #include "llvm/Analysis/VectorUtils.h" | ||||||
65 | #include "llvm/IR/BasicBlock.h" | ||||||
66 | #include "llvm/IR/CFG.h" | ||||||
67 | #include "llvm/IR/Constant.h" | ||||||
68 | #include "llvm/IR/Constants.h" | ||||||
69 | #include "llvm/IR/DIBuilder.h" | ||||||
70 | #include "llvm/IR/DataLayout.h" | ||||||
71 | #include "llvm/IR/DerivedTypes.h" | ||||||
72 | #include "llvm/IR/Dominators.h" | ||||||
73 | #include "llvm/IR/Function.h" | ||||||
74 | #include "llvm/IR/GetElementPtrTypeIterator.h" | ||||||
75 | #include "llvm/IR/IRBuilder.h" | ||||||
76 | #include "llvm/IR/InstrTypes.h" | ||||||
77 | #include "llvm/IR/Instruction.h" | ||||||
78 | #include "llvm/IR/Instructions.h" | ||||||
79 | #include "llvm/IR/IntrinsicInst.h" | ||||||
80 | #include "llvm/IR/Intrinsics.h" | ||||||
81 | #include "llvm/IR/LegacyPassManager.h" | ||||||
82 | #include "llvm/IR/Metadata.h" | ||||||
83 | #include "llvm/IR/Operator.h" | ||||||
84 | #include "llvm/IR/PassManager.h" | ||||||
85 | #include "llvm/IR/PatternMatch.h" | ||||||
86 | #include "llvm/IR/Type.h" | ||||||
87 | #include "llvm/IR/Use.h" | ||||||
88 | #include "llvm/IR/User.h" | ||||||
89 | #include "llvm/IR/Value.h" | ||||||
90 | #include "llvm/IR/ValueHandle.h" | ||||||
91 | #include "llvm/InitializePasses.h" | ||||||
92 | #include "llvm/Pass.h" | ||||||
93 | #include "llvm/Support/CBindingWrapping.h" | ||||||
94 | #include "llvm/Support/Casting.h" | ||||||
95 | #include "llvm/Support/CommandLine.h" | ||||||
96 | #include "llvm/Support/Compiler.h" | ||||||
97 | #include "llvm/Support/Debug.h" | ||||||
98 | #include "llvm/Support/DebugCounter.h" | ||||||
99 | #include "llvm/Support/ErrorHandling.h" | ||||||
100 | #include "llvm/Support/KnownBits.h" | ||||||
101 | #include "llvm/Support/raw_ostream.h" | ||||||
102 | #include "llvm/Transforms/InstCombine/InstCombine.h" | ||||||
103 | #include "llvm/Transforms/Utils/Local.h" | ||||||
104 | #include <algorithm> | ||||||
105 | #include <cassert> | ||||||
106 | #include <cstdint> | ||||||
107 | #include <memory> | ||||||
108 | #include <string> | ||||||
109 | #include <utility> | ||||||
110 | |||||||
111 | #define DEBUG_TYPE"instcombine" "instcombine" | ||||||
112 | #include "llvm/Transforms/Utils/InstructionWorklist.h" | ||||||
113 | |||||||
114 | using namespace llvm; | ||||||
115 | using namespace llvm::PatternMatch; | ||||||
116 | |||||||
117 | STATISTIC(NumWorklistIterations,static llvm::Statistic NumWorklistIterations = {"instcombine" , "NumWorklistIterations", "Number of instruction combining iterations performed" } | ||||||
118 | "Number of instruction combining iterations performed")static llvm::Statistic NumWorklistIterations = {"instcombine" , "NumWorklistIterations", "Number of instruction combining iterations performed" }; | ||||||
119 | |||||||
120 | STATISTIC(NumCombined , "Number of insts combined")static llvm::Statistic NumCombined = {"instcombine", "NumCombined" , "Number of insts combined"}; | ||||||
121 | STATISTIC(NumConstProp, "Number of constant folds")static llvm::Statistic NumConstProp = {"instcombine", "NumConstProp" , "Number of constant folds"}; | ||||||
122 | STATISTIC(NumDeadInst , "Number of dead inst eliminated")static llvm::Statistic NumDeadInst = {"instcombine", "NumDeadInst" , "Number of dead inst eliminated"}; | ||||||
123 | STATISTIC(NumSunkInst , "Number of instructions sunk")static llvm::Statistic NumSunkInst = {"instcombine", "NumSunkInst" , "Number of instructions sunk"}; | ||||||
124 | STATISTIC(NumExpand, "Number of expansions")static llvm::Statistic NumExpand = {"instcombine", "NumExpand" , "Number of expansions"}; | ||||||
125 | STATISTIC(NumFactor , "Number of factorizations")static llvm::Statistic NumFactor = {"instcombine", "NumFactor" , "Number of factorizations"}; | ||||||
126 | STATISTIC(NumReassoc , "Number of reassociations")static llvm::Statistic NumReassoc = {"instcombine", "NumReassoc" , "Number of reassociations"}; | ||||||
127 | DEBUG_COUNTER(VisitCounter, "instcombine-visit",static const unsigned VisitCounter = DebugCounter::registerCounter ("instcombine-visit", "Controls which instructions are visited" ) | ||||||
128 | "Controls which instructions are visited")static const unsigned VisitCounter = DebugCounter::registerCounter ("instcombine-visit", "Controls which instructions are visited" ); | ||||||
129 | |||||||
130 | // FIXME: these limits eventually should be as low as 2. | ||||||
131 | static constexpr unsigned InstCombineDefaultMaxIterations = 1000; | ||||||
132 | #ifndef NDEBUG | ||||||
133 | static constexpr unsigned InstCombineDefaultInfiniteLoopThreshold = 100; | ||||||
134 | #else | ||||||
135 | static constexpr unsigned InstCombineDefaultInfiniteLoopThreshold = 1000; | ||||||
136 | #endif | ||||||
137 | |||||||
138 | static cl::opt<bool> | ||||||
139 | EnableCodeSinking("instcombine-code-sinking", cl::desc("Enable code sinking"), | ||||||
140 | cl::init(true)); | ||||||
141 | |||||||
142 | static cl::opt<unsigned> LimitMaxIterations( | ||||||
143 | "instcombine-max-iterations", | ||||||
144 | cl::desc("Limit the maximum number of instruction combining iterations"), | ||||||
145 | cl::init(InstCombineDefaultMaxIterations)); | ||||||
146 | |||||||
147 | static cl::opt<unsigned> InfiniteLoopDetectionThreshold( | ||||||
148 | "instcombine-infinite-loop-threshold", | ||||||
149 | cl::desc("Number of instruction combining iterations considered an " | ||||||
150 | "infinite loop"), | ||||||
151 | cl::init(InstCombineDefaultInfiniteLoopThreshold), cl::Hidden); | ||||||
152 | |||||||
153 | static cl::opt<unsigned> | ||||||
154 | MaxArraySize("instcombine-maxarray-size", cl::init(1024), | ||||||
155 | cl::desc("Maximum array size considered when doing a combine")); | ||||||
156 | |||||||
157 | // FIXME: Remove this flag when it is no longer necessary to convert | ||||||
158 | // llvm.dbg.declare to avoid inaccurate debug info. Setting this to false | ||||||
159 | // increases variable availability at the cost of accuracy. Variables that | ||||||
160 | // cannot be promoted by mem2reg or SROA will be described as living in memory | ||||||
161 | // for their entire lifetime. However, passes like DSE and instcombine can | ||||||
162 | // delete stores to the alloca, leading to misleading and inaccurate debug | ||||||
163 | // information. This flag can be removed when those passes are fixed. | ||||||
164 | static cl::opt<unsigned> ShouldLowerDbgDeclare("instcombine-lower-dbg-declare", | ||||||
165 | cl::Hidden, cl::init(true)); | ||||||
166 | |||||||
167 | Optional<Instruction *> | ||||||
168 | InstCombiner::targetInstCombineIntrinsic(IntrinsicInst &II) { | ||||||
169 | // Handle target specific intrinsics | ||||||
170 | if (II.getCalledFunction()->isTargetIntrinsic()) { | ||||||
171 | return TTI.instCombineIntrinsic(*this, II); | ||||||
172 | } | ||||||
173 | return None; | ||||||
174 | } | ||||||
175 | |||||||
176 | Optional<Value *> InstCombiner::targetSimplifyDemandedUseBitsIntrinsic( | ||||||
177 | IntrinsicInst &II, APInt DemandedMask, KnownBits &Known, | ||||||
178 | bool &KnownBitsComputed) { | ||||||
179 | // Handle target specific intrinsics | ||||||
180 | if (II.getCalledFunction()->isTargetIntrinsic()) { | ||||||
181 | return TTI.simplifyDemandedUseBitsIntrinsic(*this, II, DemandedMask, Known, | ||||||
182 | KnownBitsComputed); | ||||||
183 | } | ||||||
184 | return None; | ||||||
185 | } | ||||||
186 | |||||||
187 | Optional<Value *> InstCombiner::targetSimplifyDemandedVectorEltsIntrinsic( | ||||||
188 | IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2, | ||||||
189 | APInt &UndefElts3, | ||||||
190 | std::function<void(Instruction *, unsigned, APInt, APInt &)> | ||||||
191 | SimplifyAndSetOp) { | ||||||
192 | // Handle target specific intrinsics | ||||||
193 | if (II.getCalledFunction()->isTargetIntrinsic()) { | ||||||
194 | return TTI.simplifyDemandedVectorEltsIntrinsic( | ||||||
195 | *this, II, DemandedElts, UndefElts, UndefElts2, UndefElts3, | ||||||
196 | SimplifyAndSetOp); | ||||||
197 | } | ||||||
198 | return None; | ||||||
199 | } | ||||||
200 | |||||||
201 | Value *InstCombinerImpl::EmitGEPOffset(User *GEP) { | ||||||
202 | return llvm::EmitGEPOffset(&Builder, DL, GEP); | ||||||
203 | } | ||||||
204 | |||||||
205 | /// Legal integers and common types are considered desirable. This is used to | ||||||
206 | /// avoid creating instructions with types that may not be supported well by the | ||||||
207 | /// the backend. | ||||||
208 | /// NOTE: This treats i8, i16 and i32 specially because they are common | ||||||
209 | /// types in frontend languages. | ||||||
210 | bool InstCombinerImpl::isDesirableIntType(unsigned BitWidth) const { | ||||||
211 | switch (BitWidth) { | ||||||
212 | case 8: | ||||||
213 | case 16: | ||||||
214 | case 32: | ||||||
215 | return true; | ||||||
216 | default: | ||||||
217 | return DL.isLegalInteger(BitWidth); | ||||||
218 | } | ||||||
219 | } | ||||||
220 | |||||||
221 | /// Return true if it is desirable to convert an integer computation from a | ||||||
222 | /// given bit width to a new bit width. | ||||||
223 | /// We don't want to convert from a legal to an illegal type or from a smaller | ||||||
224 | /// to a larger illegal type. A width of '1' is always treated as a desirable | ||||||
225 | /// type because i1 is a fundamental type in IR, and there are many specialized | ||||||
226 | /// optimizations for i1 types. Common/desirable widths are equally treated as | ||||||
227 | /// legal to convert to, in order to open up more combining opportunities. | ||||||
228 | bool InstCombinerImpl::shouldChangeType(unsigned FromWidth, | ||||||
229 | unsigned ToWidth) const { | ||||||
230 | bool FromLegal = FromWidth == 1 || DL.isLegalInteger(FromWidth); | ||||||
231 | bool ToLegal = ToWidth == 1 || DL.isLegalInteger(ToWidth); | ||||||
232 | |||||||
233 | // Convert to desirable widths even if they are not legal types. | ||||||
234 | // Only shrink types, to prevent infinite loops. | ||||||
235 | if (ToWidth < FromWidth && isDesirableIntType(ToWidth)) | ||||||
236 | return true; | ||||||
237 | |||||||
238 | // If this is a legal integer from type, and the result would be an illegal | ||||||
239 | // type, don't do the transformation. | ||||||
240 | if (FromLegal && !ToLegal) | ||||||
241 | return false; | ||||||
242 | |||||||
243 | // Otherwise, if both are illegal, do not increase the size of the result. We | ||||||
244 | // do allow things like i160 -> i64, but not i64 -> i160. | ||||||
245 | if (!FromLegal && !ToLegal && ToWidth > FromWidth) | ||||||
246 | return false; | ||||||
247 | |||||||
248 | return true; | ||||||
249 | } | ||||||
250 | |||||||
251 | /// Return true if it is desirable to convert a computation from 'From' to 'To'. | ||||||
252 | /// We don't want to convert from a legal to an illegal type or from a smaller | ||||||
253 | /// to a larger illegal type. i1 is always treated as a legal type because it is | ||||||
254 | /// a fundamental type in IR, and there are many specialized optimizations for | ||||||
255 | /// i1 types. | ||||||
256 | bool InstCombinerImpl::shouldChangeType(Type *From, Type *To) const { | ||||||
257 | // TODO: This could be extended to allow vectors. Datalayout changes might be | ||||||
258 | // needed to properly support that. | ||||||
259 | if (!From->isIntegerTy() || !To->isIntegerTy()) | ||||||
260 | return false; | ||||||
261 | |||||||
262 | unsigned FromWidth = From->getPrimitiveSizeInBits(); | ||||||
263 | unsigned ToWidth = To->getPrimitiveSizeInBits(); | ||||||
264 | return shouldChangeType(FromWidth, ToWidth); | ||||||
265 | } | ||||||
266 | |||||||
267 | // Return true, if No Signed Wrap should be maintained for I. | ||||||
268 | // The No Signed Wrap flag can be kept if the operation "B (I.getOpcode) C", | ||||||
269 | // where both B and C should be ConstantInts, results in a constant that does | ||||||
270 | // not overflow. This function only handles the Add and Sub opcodes. For | ||||||
271 | // all other opcodes, the function conservatively returns false. | ||||||
272 | static bool maintainNoSignedWrap(BinaryOperator &I, Value *B, Value *C) { | ||||||
273 | auto *OBO = dyn_cast<OverflowingBinaryOperator>(&I); | ||||||
274 | if (!OBO || !OBO->hasNoSignedWrap()) | ||||||
275 | return false; | ||||||
276 | |||||||
277 | // We reason about Add and Sub Only. | ||||||
278 | Instruction::BinaryOps Opcode = I.getOpcode(); | ||||||
279 | if (Opcode != Instruction::Add && Opcode != Instruction::Sub) | ||||||
280 | return false; | ||||||
281 | |||||||
282 | const APInt *BVal, *CVal; | ||||||
283 | if (!match(B, m_APInt(BVal)) || !match(C, m_APInt(CVal))) | ||||||
284 | return false; | ||||||
285 | |||||||
286 | bool Overflow = false; | ||||||
287 | if (Opcode == Instruction::Add) | ||||||
288 | (void)BVal->sadd_ov(*CVal, Overflow); | ||||||
289 | else | ||||||
290 | (void)BVal->ssub_ov(*CVal, Overflow); | ||||||
291 | |||||||
292 | return !Overflow; | ||||||
293 | } | ||||||
294 | |||||||
295 | static bool hasNoUnsignedWrap(BinaryOperator &I) { | ||||||
296 | auto *OBO = dyn_cast<OverflowingBinaryOperator>(&I); | ||||||
297 | return OBO && OBO->hasNoUnsignedWrap(); | ||||||
298 | } | ||||||
299 | |||||||
300 | static bool hasNoSignedWrap(BinaryOperator &I) { | ||||||
301 | auto *OBO = dyn_cast<OverflowingBinaryOperator>(&I); | ||||||
302 | return OBO && OBO->hasNoSignedWrap(); | ||||||
303 | } | ||||||
304 | |||||||
305 | /// Conservatively clears subclassOptionalData after a reassociation or | ||||||
306 | /// commutation. We preserve fast-math flags when applicable as they can be | ||||||
307 | /// preserved. | ||||||
308 | static void ClearSubclassDataAfterReassociation(BinaryOperator &I) { | ||||||
309 | FPMathOperator *FPMO = dyn_cast<FPMathOperator>(&I); | ||||||
310 | if (!FPMO) { | ||||||
311 | I.clearSubclassOptionalData(); | ||||||
312 | return; | ||||||
313 | } | ||||||
314 | |||||||
315 | FastMathFlags FMF = I.getFastMathFlags(); | ||||||
316 | I.clearSubclassOptionalData(); | ||||||
317 | I.setFastMathFlags(FMF); | ||||||
318 | } | ||||||
319 | |||||||
320 | /// Combine constant operands of associative operations either before or after a | ||||||
321 | /// cast to eliminate one of the associative operations: | ||||||
322 | /// (op (cast (op X, C2)), C1) --> (cast (op X, op (C1, C2))) | ||||||
323 | /// (op (cast (op X, C2)), C1) --> (op (cast X), op (C1, C2)) | ||||||
324 | static bool simplifyAssocCastAssoc(BinaryOperator *BinOp1, | ||||||
325 | InstCombinerImpl &IC) { | ||||||
326 | auto *Cast = dyn_cast<CastInst>(BinOp1->getOperand(0)); | ||||||
327 | if (!Cast || !Cast->hasOneUse()) | ||||||
328 | return false; | ||||||
329 | |||||||
330 | // TODO: Enhance logic for other casts and remove this check. | ||||||
331 | auto CastOpcode = Cast->getOpcode(); | ||||||
332 | if (CastOpcode != Instruction::ZExt) | ||||||
333 | return false; | ||||||
334 | |||||||
335 | // TODO: Enhance logic for other BinOps and remove this check. | ||||||
336 | if (!BinOp1->isBitwiseLogicOp()) | ||||||
337 | return false; | ||||||
338 | |||||||
339 | auto AssocOpcode = BinOp1->getOpcode(); | ||||||
340 | auto *BinOp2 = dyn_cast<BinaryOperator>(Cast->getOperand(0)); | ||||||
341 | if (!BinOp2 || !BinOp2->hasOneUse() || BinOp2->getOpcode() != AssocOpcode) | ||||||
342 | return false; | ||||||
343 | |||||||
344 | Constant *C1, *C2; | ||||||
345 | if (!match(BinOp1->getOperand(1), m_Constant(C1)) || | ||||||
346 | !match(BinOp2->getOperand(1), m_Constant(C2))) | ||||||
347 | return false; | ||||||
348 | |||||||
349 | // TODO: This assumes a zext cast. | ||||||
350 | // Eg, if it was a trunc, we'd cast C1 to the source type because casting C2 | ||||||
351 | // to the destination type might lose bits. | ||||||
352 | |||||||
353 | // Fold the constants together in the destination type: | ||||||
354 | // (op (cast (op X, C2)), C1) --> (op (cast X), FoldedC) | ||||||
355 | Type *DestTy = C1->getType(); | ||||||
356 | Constant *CastC2 = ConstantExpr::getCast(CastOpcode, C2, DestTy); | ||||||
357 | Constant *FoldedC = ConstantExpr::get(AssocOpcode, C1, CastC2); | ||||||
358 | IC.replaceOperand(*Cast, 0, BinOp2->getOperand(0)); | ||||||
359 | IC.replaceOperand(*BinOp1, 1, FoldedC); | ||||||
360 | return true; | ||||||
361 | } | ||||||
362 | |||||||
363 | // Simplifies IntToPtr/PtrToInt RoundTrip Cast To BitCast. | ||||||
364 | // inttoptr ( ptrtoint (x) ) --> x | ||||||
365 | Value *InstCombinerImpl::simplifyIntToPtrRoundTripCast(Value *Val) { | ||||||
366 | auto *IntToPtr = dyn_cast<IntToPtrInst>(Val); | ||||||
367 | if (IntToPtr && DL.getPointerTypeSizeInBits(IntToPtr->getDestTy()) == | ||||||
368 | DL.getTypeSizeInBits(IntToPtr->getSrcTy())) { | ||||||
369 | auto *PtrToInt = dyn_cast<PtrToIntInst>(IntToPtr->getOperand(0)); | ||||||
370 | Type *CastTy = IntToPtr->getDestTy(); | ||||||
371 | if (PtrToInt && | ||||||
372 | CastTy->getPointerAddressSpace() == | ||||||
373 | PtrToInt->getSrcTy()->getPointerAddressSpace() && | ||||||
374 | DL.getPointerTypeSizeInBits(PtrToInt->getSrcTy()) == | ||||||
375 | DL.getTypeSizeInBits(PtrToInt->getDestTy())) { | ||||||
376 | return CastInst::CreateBitOrPointerCast(PtrToInt->getOperand(0), CastTy, | ||||||
377 | "", PtrToInt); | ||||||
378 | } | ||||||
379 | } | ||||||
380 | return nullptr; | ||||||
381 | } | ||||||
382 | |||||||
383 | /// This performs a few simplifications for operators that are associative or | ||||||
384 | /// commutative: | ||||||
385 | /// | ||||||
386 | /// Commutative operators: | ||||||
387 | /// | ||||||
388 | /// 1. Order operands such that they are listed from right (least complex) to | ||||||
389 | /// left (most complex). This puts constants before unary operators before | ||||||
390 | /// binary operators. | ||||||
391 | /// | ||||||
392 | /// Associative operators: | ||||||
393 | /// | ||||||
394 | /// 2. Transform: "(A op B) op C" ==> "A op (B op C)" if "B op C" simplifies. | ||||||
395 | /// 3. Transform: "A op (B op C)" ==> "(A op B) op C" if "A op B" simplifies. | ||||||
396 | /// | ||||||
397 | /// Associative and commutative operators: | ||||||
398 | /// | ||||||
399 | /// 4. Transform: "(A op B) op C" ==> "(C op A) op B" if "C op A" simplifies. | ||||||
400 | /// 5. Transform: "A op (B op C)" ==> "B op (C op A)" if "C op A" simplifies. | ||||||
401 | /// 6. Transform: "(A op C1) op (B op C2)" ==> "(A op B) op (C1 op C2)" | ||||||
402 | /// if C1 and C2 are constants. | ||||||
403 | bool InstCombinerImpl::SimplifyAssociativeOrCommutative(BinaryOperator &I) { | ||||||
404 | Instruction::BinaryOps Opcode = I.getOpcode(); | ||||||
405 | bool Changed = false; | ||||||
406 | |||||||
407 | do { | ||||||
408 | // Order operands such that they are listed from right (least complex) to | ||||||
409 | // left (most complex). This puts constants before unary operators before | ||||||
410 | // binary operators. | ||||||
411 | if (I.isCommutative() && getComplexity(I.getOperand(0)) < | ||||||
412 | getComplexity(I.getOperand(1))) | ||||||
413 | Changed = !I.swapOperands(); | ||||||
414 | |||||||
415 | BinaryOperator *Op0 = dyn_cast<BinaryOperator>(I.getOperand(0)); | ||||||
416 | BinaryOperator *Op1 = dyn_cast<BinaryOperator>(I.getOperand(1)); | ||||||
417 | |||||||
418 | if (I.isAssociative()) { | ||||||
419 | // Transform: "(A op B) op C" ==> "A op (B op C)" if "B op C" simplifies. | ||||||
420 | if (Op0 && Op0->getOpcode() == Opcode) { | ||||||
421 | Value *A = Op0->getOperand(0); | ||||||
422 | Value *B = Op0->getOperand(1); | ||||||
423 | Value *C = I.getOperand(1); | ||||||
424 | |||||||
425 | // Does "B op C" simplify? | ||||||
426 | if (Value *V = SimplifyBinOp(Opcode, B, C, SQ.getWithInstruction(&I))) { | ||||||
427 | // It simplifies to V. Form "A op V". | ||||||
428 | replaceOperand(I, 0, A); | ||||||
429 | replaceOperand(I, 1, V); | ||||||
430 | bool IsNUW = hasNoUnsignedWrap(I) && hasNoUnsignedWrap(*Op0); | ||||||
431 | bool IsNSW = maintainNoSignedWrap(I, B, C) && hasNoSignedWrap(*Op0); | ||||||
432 | |||||||
433 | // Conservatively clear all optional flags since they may not be | ||||||
434 | // preserved by the reassociation. Reset nsw/nuw based on the above | ||||||
435 | // analysis. | ||||||
436 | ClearSubclassDataAfterReassociation(I); | ||||||
437 | |||||||
438 | // Note: this is only valid because SimplifyBinOp doesn't look at | ||||||
439 | // the operands to Op0. | ||||||
440 | if (IsNUW) | ||||||
441 | I.setHasNoUnsignedWrap(true); | ||||||
442 | |||||||
443 | if (IsNSW) | ||||||
444 | I.setHasNoSignedWrap(true); | ||||||
445 | |||||||
446 | Changed = true; | ||||||
447 | ++NumReassoc; | ||||||
448 | continue; | ||||||
449 | } | ||||||
450 | } | ||||||
451 | |||||||
452 | // Transform: "A op (B op C)" ==> "(A op B) op C" if "A op B" simplifies. | ||||||
453 | if (Op1 && Op1->getOpcode() == Opcode) { | ||||||
454 | Value *A = I.getOperand(0); | ||||||
455 | Value *B = Op1->getOperand(0); | ||||||
456 | Value *C = Op1->getOperand(1); | ||||||
457 | |||||||
458 | // Does "A op B" simplify? | ||||||
459 | if (Value *V = SimplifyBinOp(Opcode, A, B, SQ.getWithInstruction(&I))) { | ||||||
460 | // It simplifies to V. Form "V op C". | ||||||
461 | replaceOperand(I, 0, V); | ||||||
462 | replaceOperand(I, 1, C); | ||||||
463 | // Conservatively clear the optional flags, since they may not be | ||||||
464 | // preserved by the reassociation. | ||||||
465 | ClearSubclassDataAfterReassociation(I); | ||||||
466 | Changed = true; | ||||||
467 | ++NumReassoc; | ||||||
468 | continue; | ||||||
469 | } | ||||||
470 | } | ||||||
471 | } | ||||||
472 | |||||||
473 | if (I.isAssociative() && I.isCommutative()) { | ||||||
474 | if (simplifyAssocCastAssoc(&I, *this)) { | ||||||
475 | Changed = true; | ||||||
476 | ++NumReassoc; | ||||||
477 | continue; | ||||||
478 | } | ||||||
479 | |||||||
480 | // Transform: "(A op B) op C" ==> "(C op A) op B" if "C op A" simplifies. | ||||||
481 | if (Op0 && Op0->getOpcode() == Opcode) { | ||||||
482 | Value *A = Op0->getOperand(0); | ||||||
483 | Value *B = Op0->getOperand(1); | ||||||
484 | Value *C = I.getOperand(1); | ||||||
485 | |||||||
486 | // Does "C op A" simplify? | ||||||
487 | if (Value *V = SimplifyBinOp(Opcode, C, A, SQ.getWithInstruction(&I))) { | ||||||
488 | // It simplifies to V. Form "V op B". | ||||||
489 | replaceOperand(I, 0, V); | ||||||
490 | replaceOperand(I, 1, B); | ||||||
491 | // Conservatively clear the optional flags, since they may not be | ||||||
492 | // preserved by the reassociation. | ||||||
493 | ClearSubclassDataAfterReassociation(I); | ||||||
494 | Changed = true; | ||||||
495 | ++NumReassoc; | ||||||
496 | continue; | ||||||
497 | } | ||||||
498 | } | ||||||
499 | |||||||
500 | // Transform: "A op (B op C)" ==> "B op (C op A)" if "C op A" simplifies. | ||||||
501 | if (Op1 && Op1->getOpcode() == Opcode) { | ||||||
502 | Value *A = I.getOperand(0); | ||||||
503 | Value *B = Op1->getOperand(0); | ||||||
504 | Value *C = Op1->getOperand(1); | ||||||
505 | |||||||
506 | // Does "C op A" simplify? | ||||||
507 | if (Value *V = SimplifyBinOp(Opcode, C, A, SQ.getWithInstruction(&I))) { | ||||||
508 | // It simplifies to V. Form "B op V". | ||||||
509 | replaceOperand(I, 0, B); | ||||||
510 | replaceOperand(I, 1, V); | ||||||
511 | // Conservatively clear the optional flags, since they may not be | ||||||
512 | // preserved by the reassociation. | ||||||
513 | ClearSubclassDataAfterReassociation(I); | ||||||
514 | Changed = true; | ||||||
515 | ++NumReassoc; | ||||||
516 | continue; | ||||||
517 | } | ||||||
518 | } | ||||||
519 | |||||||
520 | // Transform: "(A op C1) op (B op C2)" ==> "(A op B) op (C1 op C2)" | ||||||
521 | // if C1 and C2 are constants. | ||||||
522 | Value *A, *B; | ||||||
523 | Constant *C1, *C2; | ||||||
524 | if (Op0 && Op1 && | ||||||
525 | Op0->getOpcode() == Opcode && Op1->getOpcode() == Opcode && | ||||||
526 | match(Op0, m_OneUse(m_BinOp(m_Value(A), m_Constant(C1)))) && | ||||||
527 | match(Op1, m_OneUse(m_BinOp(m_Value(B), m_Constant(C2))))) { | ||||||
528 | bool IsNUW = hasNoUnsignedWrap(I) && | ||||||
529 | hasNoUnsignedWrap(*Op0) && | ||||||
530 | hasNoUnsignedWrap(*Op1); | ||||||
531 | BinaryOperator *NewBO = (IsNUW && Opcode == Instruction::Add) ? | ||||||
532 | BinaryOperator::CreateNUW(Opcode, A, B) : | ||||||
533 | BinaryOperator::Create(Opcode, A, B); | ||||||
534 | |||||||
535 | if (isa<FPMathOperator>(NewBO)) { | ||||||
536 | FastMathFlags Flags = I.getFastMathFlags(); | ||||||
537 | Flags &= Op0->getFastMathFlags(); | ||||||
538 | Flags &= Op1->getFastMathFlags(); | ||||||
539 | NewBO->setFastMathFlags(Flags); | ||||||
540 | } | ||||||
541 | InsertNewInstWith(NewBO, I); | ||||||
542 | NewBO->takeName(Op1); | ||||||
543 | replaceOperand(I, 0, NewBO); | ||||||
544 | replaceOperand(I, 1, ConstantExpr::get(Opcode, C1, C2)); | ||||||
545 | // Conservatively clear the optional flags, since they may not be | ||||||
546 | // preserved by the reassociation. | ||||||
547 | ClearSubclassDataAfterReassociation(I); | ||||||
548 | if (IsNUW) | ||||||
549 | I.setHasNoUnsignedWrap(true); | ||||||
550 | |||||||
551 | Changed = true; | ||||||
552 | continue; | ||||||
553 | } | ||||||
554 | } | ||||||
555 | |||||||
556 | // No further simplifications. | ||||||
557 | return Changed; | ||||||
558 | } while (true); | ||||||
559 | } | ||||||
560 | |||||||
561 | /// Return whether "X LOp (Y ROp Z)" is always equal to | ||||||
562 | /// "(X LOp Y) ROp (X LOp Z)". | ||||||
563 | static bool leftDistributesOverRight(Instruction::BinaryOps LOp, | ||||||
564 | Instruction::BinaryOps ROp) { | ||||||
565 | // X & (Y | Z) <--> (X & Y) | (X & Z) | ||||||
566 | // X & (Y ^ Z) <--> (X & Y) ^ (X & Z) | ||||||
567 | if (LOp == Instruction::And) | ||||||
568 | return ROp == Instruction::Or || ROp == Instruction::Xor; | ||||||
569 | |||||||
570 | // X | (Y & Z) <--> (X | Y) & (X | Z) | ||||||
571 | if (LOp == Instruction::Or) | ||||||
572 | return ROp == Instruction::And; | ||||||
573 | |||||||
574 | // X * (Y + Z) <--> (X * Y) + (X * Z) | ||||||
575 | // X * (Y - Z) <--> (X * Y) - (X * Z) | ||||||
576 | if (LOp == Instruction::Mul) | ||||||
577 | return ROp == Instruction::Add || ROp == Instruction::Sub; | ||||||
578 | |||||||
579 | return false; | ||||||
580 | } | ||||||
581 | |||||||
582 | /// Return whether "(X LOp Y) ROp Z" is always equal to | ||||||
583 | /// "(X ROp Z) LOp (Y ROp Z)". | ||||||
584 | static bool rightDistributesOverLeft(Instruction::BinaryOps LOp, | ||||||
585 | Instruction::BinaryOps ROp) { | ||||||
586 | if (Instruction::isCommutative(ROp)) | ||||||
587 | return leftDistributesOverRight(ROp, LOp); | ||||||
588 | |||||||
589 | // (X {&|^} Y) >> Z <--> (X >> Z) {&|^} (Y >> Z) for all shifts. | ||||||
590 | return Instruction::isBitwiseLogicOp(LOp) && Instruction::isShift(ROp); | ||||||
591 | |||||||
592 | // TODO: It would be nice to handle division, aka "(X + Y)/Z = X/Z + Y/Z", | ||||||
593 | // but this requires knowing that the addition does not overflow and other | ||||||
594 | // such subtleties. | ||||||
595 | } | ||||||
596 | |||||||
597 | /// This function returns identity value for given opcode, which can be used to | ||||||
598 | /// factor patterns like (X * 2) + X ==> (X * 2) + (X * 1) ==> X * (2 + 1). | ||||||
599 | static Value *getIdentityValue(Instruction::BinaryOps Opcode, Value *V) { | ||||||
600 | if (isa<Constant>(V)) | ||||||
601 | return nullptr; | ||||||
602 | |||||||
603 | return ConstantExpr::getBinOpIdentity(Opcode, V->getType()); | ||||||
604 | } | ||||||
605 | |||||||
606 | /// This function predicates factorization using distributive laws. By default, | ||||||
607 | /// it just returns the 'Op' inputs. But for special-cases like | ||||||
608 | /// 'add(shl(X, 5), ...)', this function will have TopOpcode == Instruction::Add | ||||||
609 | /// and Op = shl(X, 5). The 'shl' is treated as the more general 'mul X, 32' to | ||||||
610 | /// allow more factorization opportunities. | ||||||
611 | static Instruction::BinaryOps | ||||||
612 | getBinOpsForFactorization(Instruction::BinaryOps TopOpcode, BinaryOperator *Op, | ||||||
613 | Value *&LHS, Value *&RHS) { | ||||||
614 | assert(Op && "Expected a binary operator")(static_cast <bool> (Op && "Expected a binary operator" ) ? void (0) : __assert_fail ("Op && \"Expected a binary operator\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 614, __extension__ __PRETTY_FUNCTION__)); | ||||||
615 | LHS = Op->getOperand(0); | ||||||
616 | RHS = Op->getOperand(1); | ||||||
617 | if (TopOpcode == Instruction::Add || TopOpcode == Instruction::Sub) { | ||||||
618 | Constant *C; | ||||||
619 | if (match(Op, m_Shl(m_Value(), m_Constant(C)))) { | ||||||
620 | // X << C --> X * (1 << C) | ||||||
621 | RHS = ConstantExpr::getShl(ConstantInt::get(Op->getType(), 1), C); | ||||||
622 | return Instruction::Mul; | ||||||
623 | } | ||||||
624 | // TODO: We can add other conversions e.g. shr => div etc. | ||||||
625 | } | ||||||
626 | return Op->getOpcode(); | ||||||
627 | } | ||||||
628 | |||||||
629 | /// This tries to simplify binary operations by factorizing out common terms | ||||||
630 | /// (e. g. "(A*B)+(A*C)" -> "A*(B+C)"). | ||||||
631 | Value *InstCombinerImpl::tryFactorization(BinaryOperator &I, | ||||||
632 | Instruction::BinaryOps InnerOpcode, | ||||||
633 | Value *A, Value *B, Value *C, | ||||||
634 | Value *D) { | ||||||
635 | assert(A && B && C && D && "All values must be provided")(static_cast <bool> (A && B && C && D && "All values must be provided") ? void (0) : __assert_fail ("A && B && C && D && \"All values must be provided\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 635, __extension__ __PRETTY_FUNCTION__)); | ||||||
636 | |||||||
637 | Value *V = nullptr; | ||||||
638 | Value *SimplifiedInst = nullptr; | ||||||
639 | Value *LHS = I.getOperand(0), *RHS = I.getOperand(1); | ||||||
640 | Instruction::BinaryOps TopLevelOpcode = I.getOpcode(); | ||||||
641 | |||||||
642 | // Does "X op' Y" always equal "Y op' X"? | ||||||
643 | bool InnerCommutative = Instruction::isCommutative(InnerOpcode); | ||||||
644 | |||||||
645 | // Does "X op' (Y op Z)" always equal "(X op' Y) op (X op' Z)"? | ||||||
646 | if (leftDistributesOverRight(InnerOpcode, TopLevelOpcode)) | ||||||
647 | // Does the instruction have the form "(A op' B) op (A op' D)" or, in the | ||||||
648 | // commutative case, "(A op' B) op (C op' A)"? | ||||||
649 | if (A == C || (InnerCommutative && A == D)) { | ||||||
650 | if (A != C) | ||||||
651 | std::swap(C, D); | ||||||
652 | // Consider forming "A op' (B op D)". | ||||||
653 | // If "B op D" simplifies then it can be formed with no cost. | ||||||
654 | V = SimplifyBinOp(TopLevelOpcode, B, D, SQ.getWithInstruction(&I)); | ||||||
655 | // If "B op D" doesn't simplify then only go on if both of the existing | ||||||
656 | // operations "A op' B" and "C op' D" will be zapped as no longer used. | ||||||
657 | if (!V && LHS->hasOneUse() && RHS->hasOneUse()) | ||||||
658 | V = Builder.CreateBinOp(TopLevelOpcode, B, D, RHS->getName()); | ||||||
659 | if (V) { | ||||||
660 | SimplifiedInst = Builder.CreateBinOp(InnerOpcode, A, V); | ||||||
661 | } | ||||||
662 | } | ||||||
663 | |||||||
664 | // Does "(X op Y) op' Z" always equal "(X op' Z) op (Y op' Z)"? | ||||||
665 | if (!SimplifiedInst && rightDistributesOverLeft(TopLevelOpcode, InnerOpcode)) | ||||||
666 | // Does the instruction have the form "(A op' B) op (C op' B)" or, in the | ||||||
667 | // commutative case, "(A op' B) op (B op' D)"? | ||||||
668 | if (B == D || (InnerCommutative && B == C)) { | ||||||
669 | if (B != D) | ||||||
670 | std::swap(C, D); | ||||||
671 | // Consider forming "(A op C) op' B". | ||||||
672 | // If "A op C" simplifies then it can be formed with no cost. | ||||||
673 | V = SimplifyBinOp(TopLevelOpcode, A, C, SQ.getWithInstruction(&I)); | ||||||
674 | |||||||
675 | // If "A op C" doesn't simplify then only go on if both of the existing | ||||||
676 | // operations "A op' B" and "C op' D" will be zapped as no longer used. | ||||||
677 | if (!V && LHS->hasOneUse() && RHS->hasOneUse()) | ||||||
678 | V = Builder.CreateBinOp(TopLevelOpcode, A, C, LHS->getName()); | ||||||
679 | if (V) { | ||||||
680 | SimplifiedInst = Builder.CreateBinOp(InnerOpcode, V, B); | ||||||
681 | } | ||||||
682 | } | ||||||
683 | |||||||
684 | if (SimplifiedInst) { | ||||||
685 | ++NumFactor; | ||||||
686 | SimplifiedInst->takeName(&I); | ||||||
687 | |||||||
688 | // Check if we can add NSW/NUW flags to SimplifiedInst. If so, set them. | ||||||
689 | if (BinaryOperator *BO = dyn_cast<BinaryOperator>(SimplifiedInst)) { | ||||||
690 | if (isa<OverflowingBinaryOperator>(SimplifiedInst)) { | ||||||
691 | bool HasNSW = false; | ||||||
692 | bool HasNUW = false; | ||||||
693 | if (isa<OverflowingBinaryOperator>(&I)) { | ||||||
694 | HasNSW = I.hasNoSignedWrap(); | ||||||
695 | HasNUW = I.hasNoUnsignedWrap(); | ||||||
696 | } | ||||||
697 | |||||||
698 | if (auto *LOBO = dyn_cast<OverflowingBinaryOperator>(LHS)) { | ||||||
699 | HasNSW &= LOBO->hasNoSignedWrap(); | ||||||
700 | HasNUW &= LOBO->hasNoUnsignedWrap(); | ||||||
701 | } | ||||||
702 | |||||||
703 | if (auto *ROBO = dyn_cast<OverflowingBinaryOperator>(RHS)) { | ||||||
704 | HasNSW &= ROBO->hasNoSignedWrap(); | ||||||
705 | HasNUW &= ROBO->hasNoUnsignedWrap(); | ||||||
706 | } | ||||||
707 | |||||||
708 | if (TopLevelOpcode == Instruction::Add && | ||||||
709 | InnerOpcode == Instruction::Mul) { | ||||||
710 | // We can propagate 'nsw' if we know that | ||||||
711 | // %Y = mul nsw i16 %X, C | ||||||
712 | // %Z = add nsw i16 %Y, %X | ||||||
713 | // => | ||||||
714 | // %Z = mul nsw i16 %X, C+1 | ||||||
715 | // | ||||||
716 | // iff C+1 isn't INT_MIN | ||||||
717 | const APInt *CInt; | ||||||
718 | if (match(V, m_APInt(CInt))) { | ||||||
719 | if (!CInt->isMinSignedValue()) | ||||||
720 | BO->setHasNoSignedWrap(HasNSW); | ||||||
721 | } | ||||||
722 | |||||||
723 | // nuw can be propagated with any constant or nuw value. | ||||||
724 | BO->setHasNoUnsignedWrap(HasNUW); | ||||||
725 | } | ||||||
726 | } | ||||||
727 | } | ||||||
728 | } | ||||||
729 | return SimplifiedInst; | ||||||
730 | } | ||||||
731 | |||||||
732 | /// This tries to simplify binary operations which some other binary operation | ||||||
733 | /// distributes over either by factorizing out common terms | ||||||
734 | /// (eg "(A*B)+(A*C)" -> "A*(B+C)") or expanding out if this results in | ||||||
735 | /// simplifications (eg: "A & (B | C) -> (A&B) | (A&C)" if this is a win). | ||||||
736 | /// Returns the simplified value, or null if it didn't simplify. | ||||||
737 | Value *InstCombinerImpl::SimplifyUsingDistributiveLaws(BinaryOperator &I) { | ||||||
738 | Value *LHS = I.getOperand(0), *RHS = I.getOperand(1); | ||||||
739 | BinaryOperator *Op0 = dyn_cast<BinaryOperator>(LHS); | ||||||
740 | BinaryOperator *Op1 = dyn_cast<BinaryOperator>(RHS); | ||||||
741 | Instruction::BinaryOps TopLevelOpcode = I.getOpcode(); | ||||||
742 | |||||||
743 | { | ||||||
744 | // Factorization. | ||||||
745 | Value *A, *B, *C, *D; | ||||||
746 | Instruction::BinaryOps LHSOpcode, RHSOpcode; | ||||||
747 | if (Op0) | ||||||
748 | LHSOpcode = getBinOpsForFactorization(TopLevelOpcode, Op0, A, B); | ||||||
749 | if (Op1) | ||||||
750 | RHSOpcode = getBinOpsForFactorization(TopLevelOpcode, Op1, C, D); | ||||||
751 | |||||||
752 | // The instruction has the form "(A op' B) op (C op' D)". Try to factorize | ||||||
753 | // a common term. | ||||||
754 | if (Op0 && Op1 && LHSOpcode == RHSOpcode) | ||||||
755 | if (Value *V = tryFactorization(I, LHSOpcode, A, B, C, D)) | ||||||
756 | return V; | ||||||
757 | |||||||
758 | // The instruction has the form "(A op' B) op (C)". Try to factorize common | ||||||
759 | // term. | ||||||
760 | if (Op0) | ||||||
761 | if (Value *Ident = getIdentityValue(LHSOpcode, RHS)) | ||||||
762 | if (Value *V = tryFactorization(I, LHSOpcode, A, B, RHS, Ident)) | ||||||
763 | return V; | ||||||
764 | |||||||
765 | // The instruction has the form "(B) op (C op' D)". Try to factorize common | ||||||
766 | // term. | ||||||
767 | if (Op1) | ||||||
768 | if (Value *Ident = getIdentityValue(RHSOpcode, LHS)) | ||||||
769 | if (Value *V = tryFactorization(I, RHSOpcode, LHS, Ident, C, D)) | ||||||
770 | return V; | ||||||
771 | } | ||||||
772 | |||||||
773 | // Expansion. | ||||||
774 | if (Op0 && rightDistributesOverLeft(Op0->getOpcode(), TopLevelOpcode)) { | ||||||
775 | // The instruction has the form "(A op' B) op C". See if expanding it out | ||||||
776 | // to "(A op C) op' (B op C)" results in simplifications. | ||||||
777 | Value *A = Op0->getOperand(0), *B = Op0->getOperand(1), *C = RHS; | ||||||
778 | Instruction::BinaryOps InnerOpcode = Op0->getOpcode(); // op' | ||||||
779 | |||||||
780 | // Disable the use of undef because it's not safe to distribute undef. | ||||||
781 | auto SQDistributive = SQ.getWithInstruction(&I).getWithoutUndef(); | ||||||
782 | Value *L = SimplifyBinOp(TopLevelOpcode, A, C, SQDistributive); | ||||||
783 | Value *R = SimplifyBinOp(TopLevelOpcode, B, C, SQDistributive); | ||||||
784 | |||||||
785 | // Do "A op C" and "B op C" both simplify? | ||||||
786 | if (L && R) { | ||||||
787 | // They do! Return "L op' R". | ||||||
788 | ++NumExpand; | ||||||
789 | C = Builder.CreateBinOp(InnerOpcode, L, R); | ||||||
790 | C->takeName(&I); | ||||||
791 | return C; | ||||||
792 | } | ||||||
793 | |||||||
794 | // Does "A op C" simplify to the identity value for the inner opcode? | ||||||
795 | if (L && L == ConstantExpr::getBinOpIdentity(InnerOpcode, L->getType())) { | ||||||
796 | // They do! Return "B op C". | ||||||
797 | ++NumExpand; | ||||||
798 | C = Builder.CreateBinOp(TopLevelOpcode, B, C); | ||||||
799 | C->takeName(&I); | ||||||
800 | return C; | ||||||
801 | } | ||||||
802 | |||||||
803 | // Does "B op C" simplify to the identity value for the inner opcode? | ||||||
804 | if (R && R == ConstantExpr::getBinOpIdentity(InnerOpcode, R->getType())) { | ||||||
805 | // They do! Return "A op C". | ||||||
806 | ++NumExpand; | ||||||
807 | C = Builder.CreateBinOp(TopLevelOpcode, A, C); | ||||||
808 | C->takeName(&I); | ||||||
809 | return C; | ||||||
810 | } | ||||||
811 | } | ||||||
812 | |||||||
813 | if (Op1 && leftDistributesOverRight(TopLevelOpcode, Op1->getOpcode())) { | ||||||
814 | // The instruction has the form "A op (B op' C)". See if expanding it out | ||||||
815 | // to "(A op B) op' (A op C)" results in simplifications. | ||||||
816 | Value *A = LHS, *B = Op1->getOperand(0), *C = Op1->getOperand(1); | ||||||
817 | Instruction::BinaryOps InnerOpcode = Op1->getOpcode(); // op' | ||||||
818 | |||||||
819 | // Disable the use of undef because it's not safe to distribute undef. | ||||||
820 | auto SQDistributive = SQ.getWithInstruction(&I).getWithoutUndef(); | ||||||
821 | Value *L = SimplifyBinOp(TopLevelOpcode, A, B, SQDistributive); | ||||||
822 | Value *R = SimplifyBinOp(TopLevelOpcode, A, C, SQDistributive); | ||||||
823 | |||||||
824 | // Do "A op B" and "A op C" both simplify? | ||||||
825 | if (L && R) { | ||||||
826 | // They do! Return "L op' R". | ||||||
827 | ++NumExpand; | ||||||
828 | A = Builder.CreateBinOp(InnerOpcode, L, R); | ||||||
829 | A->takeName(&I); | ||||||
830 | return A; | ||||||
831 | } | ||||||
832 | |||||||
833 | // Does "A op B" simplify to the identity value for the inner opcode? | ||||||
834 | if (L && L == ConstantExpr::getBinOpIdentity(InnerOpcode, L->getType())) { | ||||||
835 | // They do! Return "A op C". | ||||||
836 | ++NumExpand; | ||||||
837 | A = Builder.CreateBinOp(TopLevelOpcode, A, C); | ||||||
838 | A->takeName(&I); | ||||||
839 | return A; | ||||||
840 | } | ||||||
841 | |||||||
842 | // Does "A op C" simplify to the identity value for the inner opcode? | ||||||
843 | if (R && R == ConstantExpr::getBinOpIdentity(InnerOpcode, R->getType())) { | ||||||
844 | // They do! Return "A op B". | ||||||
845 | ++NumExpand; | ||||||
846 | A = Builder.CreateBinOp(TopLevelOpcode, A, B); | ||||||
847 | A->takeName(&I); | ||||||
848 | return A; | ||||||
849 | } | ||||||
850 | } | ||||||
851 | |||||||
852 | return SimplifySelectsFeedingBinaryOp(I, LHS, RHS); | ||||||
853 | } | ||||||
854 | |||||||
855 | Value *InstCombinerImpl::SimplifySelectsFeedingBinaryOp(BinaryOperator &I, | ||||||
856 | Value *LHS, | ||||||
857 | Value *RHS) { | ||||||
858 | Value *A, *B, *C, *D, *E, *F; | ||||||
859 | bool LHSIsSelect = match(LHS, m_Select(m_Value(A), m_Value(B), m_Value(C))); | ||||||
860 | bool RHSIsSelect = match(RHS, m_Select(m_Value(D), m_Value(E), m_Value(F))); | ||||||
861 | if (!LHSIsSelect && !RHSIsSelect) | ||||||
862 | return nullptr; | ||||||
863 | |||||||
864 | FastMathFlags FMF; | ||||||
865 | BuilderTy::FastMathFlagGuard Guard(Builder); | ||||||
866 | if (isa<FPMathOperator>(&I)) { | ||||||
867 | FMF = I.getFastMathFlags(); | ||||||
868 | Builder.setFastMathFlags(FMF); | ||||||
869 | } | ||||||
870 | |||||||
871 | Instruction::BinaryOps Opcode = I.getOpcode(); | ||||||
872 | SimplifyQuery Q = SQ.getWithInstruction(&I); | ||||||
873 | |||||||
874 | Value *Cond, *True = nullptr, *False = nullptr; | ||||||
875 | if (LHSIsSelect && RHSIsSelect && A == D) { | ||||||
876 | // (A ? B : C) op (A ? E : F) -> A ? (B op E) : (C op F) | ||||||
877 | Cond = A; | ||||||
878 | True = SimplifyBinOp(Opcode, B, E, FMF, Q); | ||||||
879 | False = SimplifyBinOp(Opcode, C, F, FMF, Q); | ||||||
880 | |||||||
881 | if (LHS->hasOneUse() && RHS->hasOneUse()) { | ||||||
882 | if (False && !True) | ||||||
883 | True = Builder.CreateBinOp(Opcode, B, E); | ||||||
884 | else if (True && !False) | ||||||
885 | False = Builder.CreateBinOp(Opcode, C, F); | ||||||
886 | } | ||||||
887 | } else if (LHSIsSelect && LHS->hasOneUse()) { | ||||||
888 | // (A ? B : C) op Y -> A ? (B op Y) : (C op Y) | ||||||
889 | Cond = A; | ||||||
890 | True = SimplifyBinOp(Opcode, B, RHS, FMF, Q); | ||||||
891 | False = SimplifyBinOp(Opcode, C, RHS, FMF, Q); | ||||||
892 | } else if (RHSIsSelect && RHS->hasOneUse()) { | ||||||
893 | // X op (D ? E : F) -> D ? (X op E) : (X op F) | ||||||
894 | Cond = D; | ||||||
895 | True = SimplifyBinOp(Opcode, LHS, E, FMF, Q); | ||||||
896 | False = SimplifyBinOp(Opcode, LHS, F, FMF, Q); | ||||||
897 | } | ||||||
898 | |||||||
899 | if (!True || !False) | ||||||
900 | return nullptr; | ||||||
901 | |||||||
902 | Value *SI = Builder.CreateSelect(Cond, True, False); | ||||||
903 | SI->takeName(&I); | ||||||
904 | return SI; | ||||||
905 | } | ||||||
906 | |||||||
907 | /// Freely adapt every user of V as-if V was changed to !V. | ||||||
908 | /// WARNING: only if canFreelyInvertAllUsersOf() said this can be done. | ||||||
909 | void InstCombinerImpl::freelyInvertAllUsersOf(Value *I) { | ||||||
910 | for (User *U : I->users()) { | ||||||
911 | switch (cast<Instruction>(U)->getOpcode()) { | ||||||
912 | case Instruction::Select: { | ||||||
913 | auto *SI = cast<SelectInst>(U); | ||||||
914 | SI->swapValues(); | ||||||
915 | SI->swapProfMetadata(); | ||||||
916 | break; | ||||||
917 | } | ||||||
918 | case Instruction::Br: | ||||||
919 | cast<BranchInst>(U)->swapSuccessors(); // swaps prof metadata too | ||||||
920 | break; | ||||||
921 | case Instruction::Xor: | ||||||
922 | replaceInstUsesWith(cast<Instruction>(*U), I); | ||||||
923 | break; | ||||||
924 | default: | ||||||
925 | llvm_unreachable("Got unexpected user - out of sync with "::llvm::llvm_unreachable_internal("Got unexpected user - out of sync with " "canFreelyInvertAllUsersOf() ?", "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 926) | ||||||
926 | "canFreelyInvertAllUsersOf() ?")::llvm::llvm_unreachable_internal("Got unexpected user - out of sync with " "canFreelyInvertAllUsersOf() ?", "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 926); | ||||||
927 | } | ||||||
928 | } | ||||||
929 | } | ||||||
930 | |||||||
931 | /// Given a 'sub' instruction, return the RHS of the instruction if the LHS is a | ||||||
932 | /// constant zero (which is the 'negate' form). | ||||||
933 | Value *InstCombinerImpl::dyn_castNegVal(Value *V) const { | ||||||
934 | Value *NegV; | ||||||
935 | if (match(V, m_Neg(m_Value(NegV)))) | ||||||
936 | return NegV; | ||||||
937 | |||||||
938 | // Constants can be considered to be negated values if they can be folded. | ||||||
939 | if (ConstantInt *C = dyn_cast<ConstantInt>(V)) | ||||||
940 | return ConstantExpr::getNeg(C); | ||||||
941 | |||||||
942 | if (ConstantDataVector *C = dyn_cast<ConstantDataVector>(V)) | ||||||
943 | if (C->getType()->getElementType()->isIntegerTy()) | ||||||
944 | return ConstantExpr::getNeg(C); | ||||||
945 | |||||||
946 | if (ConstantVector *CV = dyn_cast<ConstantVector>(V)) { | ||||||
947 | for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) { | ||||||
948 | Constant *Elt = CV->getAggregateElement(i); | ||||||
949 | if (!Elt) | ||||||
950 | return nullptr; | ||||||
951 | |||||||
952 | if (isa<UndefValue>(Elt)) | ||||||
953 | continue; | ||||||
954 | |||||||
955 | if (!isa<ConstantInt>(Elt)) | ||||||
956 | return nullptr; | ||||||
957 | } | ||||||
958 | return ConstantExpr::getNeg(CV); | ||||||
959 | } | ||||||
960 | |||||||
961 | // Negate integer vector splats. | ||||||
962 | if (auto *CV = dyn_cast<Constant>(V)) | ||||||
963 | if (CV->getType()->isVectorTy() && | ||||||
964 | CV->getType()->getScalarType()->isIntegerTy() && CV->getSplatValue()) | ||||||
965 | return ConstantExpr::getNeg(CV); | ||||||
966 | |||||||
967 | return nullptr; | ||||||
968 | } | ||||||
969 | |||||||
970 | static Value *foldOperationIntoSelectOperand(Instruction &I, Value *SO, | ||||||
971 | InstCombiner::BuilderTy &Builder) { | ||||||
972 | if (auto *Cast = dyn_cast<CastInst>(&I)) | ||||||
973 | return Builder.CreateCast(Cast->getOpcode(), SO, I.getType()); | ||||||
974 | |||||||
975 | if (auto *II = dyn_cast<IntrinsicInst>(&I)) { | ||||||
976 | assert(canConstantFoldCallTo(II, cast<Function>(II->getCalledOperand())) &&(static_cast <bool> (canConstantFoldCallTo(II, cast< Function>(II->getCalledOperand())) && "Expected constant-foldable intrinsic" ) ? void (0) : __assert_fail ("canConstantFoldCallTo(II, cast<Function>(II->getCalledOperand())) && \"Expected constant-foldable intrinsic\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 977, __extension__ __PRETTY_FUNCTION__)) | ||||||
977 | "Expected constant-foldable intrinsic")(static_cast <bool> (canConstantFoldCallTo(II, cast< Function>(II->getCalledOperand())) && "Expected constant-foldable intrinsic" ) ? void (0) : __assert_fail ("canConstantFoldCallTo(II, cast<Function>(II->getCalledOperand())) && \"Expected constant-foldable intrinsic\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 977, __extension__ __PRETTY_FUNCTION__)); | ||||||
978 | Intrinsic::ID IID = II->getIntrinsicID(); | ||||||
979 | if (II->arg_size() == 1) | ||||||
980 | return Builder.CreateUnaryIntrinsic(IID, SO); | ||||||
981 | |||||||
982 | // This works for real binary ops like min/max (where we always expect the | ||||||
983 | // constant operand to be canonicalized as op1) and unary ops with a bonus | ||||||
984 | // constant argument like ctlz/cttz. | ||||||
985 | // TODO: Handle non-commutative binary intrinsics as below for binops. | ||||||
986 | assert(II->arg_size() == 2 && "Expected binary intrinsic")(static_cast <bool> (II->arg_size() == 2 && "Expected binary intrinsic" ) ? void (0) : __assert_fail ("II->arg_size() == 2 && \"Expected binary intrinsic\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 986, __extension__ __PRETTY_FUNCTION__)); | ||||||
987 | assert(isa<Constant>(II->getArgOperand(1)) && "Expected constant operand")(static_cast <bool> (isa<Constant>(II->getArgOperand (1)) && "Expected constant operand") ? void (0) : __assert_fail ("isa<Constant>(II->getArgOperand(1)) && \"Expected constant operand\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 987, __extension__ __PRETTY_FUNCTION__)); | ||||||
988 | return Builder.CreateBinaryIntrinsic(IID, SO, II->getArgOperand(1)); | ||||||
989 | } | ||||||
990 | |||||||
991 | assert(I.isBinaryOp() && "Unexpected opcode for select folding")(static_cast <bool> (I.isBinaryOp() && "Unexpected opcode for select folding" ) ? void (0) : __assert_fail ("I.isBinaryOp() && \"Unexpected opcode for select folding\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 991, __extension__ __PRETTY_FUNCTION__)); | ||||||
992 | |||||||
993 | // Figure out if the constant is the left or the right argument. | ||||||
994 | bool ConstIsRHS = isa<Constant>(I.getOperand(1)); | ||||||
995 | Constant *ConstOperand = cast<Constant>(I.getOperand(ConstIsRHS)); | ||||||
996 | |||||||
997 | if (auto *SOC = dyn_cast<Constant>(SO)) { | ||||||
998 | if (ConstIsRHS) | ||||||
999 | return ConstantExpr::get(I.getOpcode(), SOC, ConstOperand); | ||||||
1000 | return ConstantExpr::get(I.getOpcode(), ConstOperand, SOC); | ||||||
1001 | } | ||||||
1002 | |||||||
1003 | Value *Op0 = SO, *Op1 = ConstOperand; | ||||||
1004 | if (!ConstIsRHS) | ||||||
1005 | std::swap(Op0, Op1); | ||||||
1006 | |||||||
1007 | auto *BO = cast<BinaryOperator>(&I); | ||||||
1008 | Value *RI = Builder.CreateBinOp(BO->getOpcode(), Op0, Op1, | ||||||
1009 | SO->getName() + ".op"); | ||||||
1010 | auto *FPInst = dyn_cast<Instruction>(RI); | ||||||
1011 | if (FPInst && isa<FPMathOperator>(FPInst)) | ||||||
1012 | FPInst->copyFastMathFlags(BO); | ||||||
1013 | return RI; | ||||||
1014 | } | ||||||
1015 | |||||||
1016 | Instruction *InstCombinerImpl::FoldOpIntoSelect(Instruction &Op, | ||||||
1017 | SelectInst *SI) { | ||||||
1018 | // Don't modify shared select instructions. | ||||||
1019 | if (!SI->hasOneUse()) | ||||||
1020 | return nullptr; | ||||||
1021 | |||||||
1022 | Value *TV = SI->getTrueValue(); | ||||||
1023 | Value *FV = SI->getFalseValue(); | ||||||
1024 | if (!(isa<Constant>(TV) || isa<Constant>(FV))) | ||||||
1025 | return nullptr; | ||||||
1026 | |||||||
1027 | // Bool selects with constant operands can be folded to logical ops. | ||||||
1028 | if (SI->getType()->isIntOrIntVectorTy(1)) | ||||||
1029 | return nullptr; | ||||||
1030 | |||||||
1031 | // If it's a bitcast involving vectors, make sure it has the same number of | ||||||
1032 | // elements on both sides. | ||||||
1033 | if (auto *BC = dyn_cast<BitCastInst>(&Op)) { | ||||||
1034 | VectorType *DestTy = dyn_cast<VectorType>(BC->getDestTy()); | ||||||
1035 | VectorType *SrcTy = dyn_cast<VectorType>(BC->getSrcTy()); | ||||||
1036 | |||||||
1037 | // Verify that either both or neither are vectors. | ||||||
1038 | if ((SrcTy == nullptr) != (DestTy == nullptr)) | ||||||
1039 | return nullptr; | ||||||
1040 | |||||||
1041 | // If vectors, verify that they have the same number of elements. | ||||||
1042 | if (SrcTy && SrcTy->getElementCount() != DestTy->getElementCount()) | ||||||
1043 | return nullptr; | ||||||
1044 | } | ||||||
1045 | |||||||
1046 | // Test if a CmpInst instruction is used exclusively by a select as | ||||||
1047 | // part of a minimum or maximum operation. If so, refrain from doing | ||||||
1048 | // any other folding. This helps out other analyses which understand | ||||||
1049 | // non-obfuscated minimum and maximum idioms, such as ScalarEvolution | ||||||
1050 | // and CodeGen. And in this case, at least one of the comparison | ||||||
1051 | // operands has at least one user besides the compare (the select), | ||||||
1052 | // which would often largely negate the benefit of folding anyway. | ||||||
1053 | if (auto *CI = dyn_cast<CmpInst>(SI->getCondition())) { | ||||||
1054 | if (CI->hasOneUse()) { | ||||||
1055 | Value *Op0 = CI->getOperand(0), *Op1 = CI->getOperand(1); | ||||||
1056 | |||||||
1057 | // FIXME: This is a hack to avoid infinite looping with min/max patterns. | ||||||
1058 | // We have to ensure that vector constants that only differ with | ||||||
1059 | // undef elements are treated as equivalent. | ||||||
1060 | auto areLooselyEqual = [](Value *A, Value *B) { | ||||||
1061 | if (A == B) | ||||||
1062 | return true; | ||||||
1063 | |||||||
1064 | // Test for vector constants. | ||||||
1065 | Constant *ConstA, *ConstB; | ||||||
1066 | if (!match(A, m_Constant(ConstA)) || !match(B, m_Constant(ConstB))) | ||||||
1067 | return false; | ||||||
1068 | |||||||
1069 | // TODO: Deal with FP constants? | ||||||
1070 | if (!A->getType()->isIntOrIntVectorTy() || A->getType() != B->getType()) | ||||||
1071 | return false; | ||||||
1072 | |||||||
1073 | // Compare for equality including undefs as equal. | ||||||
1074 | auto *Cmp = ConstantExpr::getCompare(ICmpInst::ICMP_EQ, ConstA, ConstB); | ||||||
1075 | const APInt *C; | ||||||
1076 | return match(Cmp, m_APIntAllowUndef(C)) && C->isOne(); | ||||||
1077 | }; | ||||||
1078 | |||||||
1079 | if ((areLooselyEqual(TV, Op0) && areLooselyEqual(FV, Op1)) || | ||||||
1080 | (areLooselyEqual(FV, Op0) && areLooselyEqual(TV, Op1))) | ||||||
1081 | return nullptr; | ||||||
1082 | } | ||||||
1083 | } | ||||||
1084 | |||||||
1085 | Value *NewTV = foldOperationIntoSelectOperand(Op, TV, Builder); | ||||||
1086 | Value *NewFV = foldOperationIntoSelectOperand(Op, FV, Builder); | ||||||
1087 | return SelectInst::Create(SI->getCondition(), NewTV, NewFV, "", nullptr, SI); | ||||||
1088 | } | ||||||
1089 | |||||||
1090 | static Value *foldOperationIntoPhiValue(BinaryOperator *I, Value *InV, | ||||||
1091 | InstCombiner::BuilderTy &Builder) { | ||||||
1092 | bool ConstIsRHS = isa<Constant>(I->getOperand(1)); | ||||||
1093 | Constant *C = cast<Constant>(I->getOperand(ConstIsRHS)); | ||||||
1094 | |||||||
1095 | if (auto *InC = dyn_cast<Constant>(InV)) { | ||||||
1096 | if (ConstIsRHS) | ||||||
1097 | return ConstantExpr::get(I->getOpcode(), InC, C); | ||||||
1098 | return ConstantExpr::get(I->getOpcode(), C, InC); | ||||||
1099 | } | ||||||
1100 | |||||||
1101 | Value *Op0 = InV, *Op1 = C; | ||||||
1102 | if (!ConstIsRHS) | ||||||
1103 | std::swap(Op0, Op1); | ||||||
1104 | |||||||
1105 | Value *RI = Builder.CreateBinOp(I->getOpcode(), Op0, Op1, "phi.bo"); | ||||||
1106 | auto *FPInst = dyn_cast<Instruction>(RI); | ||||||
1107 | if (FPInst && isa<FPMathOperator>(FPInst)) | ||||||
1108 | FPInst->copyFastMathFlags(I); | ||||||
1109 | return RI; | ||||||
1110 | } | ||||||
1111 | |||||||
1112 | Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) { | ||||||
1113 | unsigned NumPHIValues = PN->getNumIncomingValues(); | ||||||
1114 | if (NumPHIValues == 0) | ||||||
1115 | return nullptr; | ||||||
1116 | |||||||
1117 | // We normally only transform phis with a single use. However, if a PHI has | ||||||
1118 | // multiple uses and they are all the same operation, we can fold *all* of the | ||||||
1119 | // uses into the PHI. | ||||||
1120 | if (!PN->hasOneUse()) { | ||||||
1121 | // Walk the use list for the instruction, comparing them to I. | ||||||
1122 | for (User *U : PN->users()) { | ||||||
1123 | Instruction *UI = cast<Instruction>(U); | ||||||
1124 | if (UI != &I && !I.isIdenticalTo(UI)) | ||||||
1125 | return nullptr; | ||||||
1126 | } | ||||||
1127 | // Otherwise, we can replace *all* users with the new PHI we form. | ||||||
1128 | } | ||||||
1129 | |||||||
1130 | // Check to see if all of the operands of the PHI are simple constants | ||||||
1131 | // (constantint/constantfp/undef). If there is one non-constant value, | ||||||
1132 | // remember the BB it is in. If there is more than one or if *it* is a PHI, | ||||||
1133 | // bail out. We don't do arbitrary constant expressions here because moving | ||||||
1134 | // their computation can be expensive without a cost model. | ||||||
1135 | BasicBlock *NonConstBB = nullptr; | ||||||
1136 | for (unsigned i = 0; i != NumPHIValues; ++i) { | ||||||
1137 | Value *InVal = PN->getIncomingValue(i); | ||||||
1138 | // If I is a freeze instruction, count undef as a non-constant. | ||||||
1139 | if (match(InVal, m_ImmConstant()) && | ||||||
1140 | (!isa<FreezeInst>(I) || isGuaranteedNotToBeUndefOrPoison(InVal))) | ||||||
1141 | continue; | ||||||
1142 | |||||||
1143 | if (isa<PHINode>(InVal)) return nullptr; // Itself a phi. | ||||||
1144 | if (NonConstBB) return nullptr; // More than one non-const value. | ||||||
1145 | |||||||
1146 | NonConstBB = PN->getIncomingBlock(i); | ||||||
1147 | |||||||
1148 | // If the InVal is an invoke at the end of the pred block, then we can't | ||||||
1149 | // insert a computation after it without breaking the edge. | ||||||
1150 | if (isa<InvokeInst>(InVal)) | ||||||
1151 | if (cast<Instruction>(InVal)->getParent() == NonConstBB) | ||||||
1152 | return nullptr; | ||||||
1153 | |||||||
1154 | // If the incoming non-constant value is in I's block, we will remove one | ||||||
1155 | // instruction, but insert another equivalent one, leading to infinite | ||||||
1156 | // instcombine. | ||||||
1157 | if (isPotentiallyReachable(I.getParent(), NonConstBB, nullptr, &DT, LI)) | ||||||
1158 | return nullptr; | ||||||
1159 | } | ||||||
1160 | |||||||
1161 | // If there is exactly one non-constant value, we can insert a copy of the | ||||||
1162 | // operation in that block. However, if this is a critical edge, we would be | ||||||
1163 | // inserting the computation on some other paths (e.g. inside a loop). Only | ||||||
1164 | // do this if the pred block is unconditionally branching into the phi block. | ||||||
1165 | // Also, make sure that the pred block is not dead code. | ||||||
1166 | if (NonConstBB != nullptr) { | ||||||
1167 | BranchInst *BI = dyn_cast<BranchInst>(NonConstBB->getTerminator()); | ||||||
1168 | if (!BI || !BI->isUnconditional() || !DT.isReachableFromEntry(NonConstBB)) | ||||||
1169 | return nullptr; | ||||||
1170 | } | ||||||
1171 | |||||||
1172 | // Okay, we can do the transformation: create the new PHI node. | ||||||
1173 | PHINode *NewPN = PHINode::Create(I.getType(), PN->getNumIncomingValues()); | ||||||
1174 | InsertNewInstBefore(NewPN, *PN); | ||||||
1175 | NewPN->takeName(PN); | ||||||
1176 | |||||||
1177 | // If we are going to have to insert a new computation, do so right before the | ||||||
1178 | // predecessor's terminator. | ||||||
1179 | if (NonConstBB) | ||||||
1180 | Builder.SetInsertPoint(NonConstBB->getTerminator()); | ||||||
1181 | |||||||
1182 | // Next, add all of the operands to the PHI. | ||||||
1183 | if (SelectInst *SI = dyn_cast<SelectInst>(&I)) { | ||||||
1184 | // We only currently try to fold the condition of a select when it is a phi, | ||||||
1185 | // not the true/false values. | ||||||
1186 | Value *TrueV = SI->getTrueValue(); | ||||||
1187 | Value *FalseV = SI->getFalseValue(); | ||||||
1188 | BasicBlock *PhiTransBB = PN->getParent(); | ||||||
1189 | for (unsigned i = 0; i != NumPHIValues; ++i) { | ||||||
1190 | BasicBlock *ThisBB = PN->getIncomingBlock(i); | ||||||
1191 | Value *TrueVInPred = TrueV->DoPHITranslation(PhiTransBB, ThisBB); | ||||||
1192 | Value *FalseVInPred = FalseV->DoPHITranslation(PhiTransBB, ThisBB); | ||||||
1193 | Value *InV = nullptr; | ||||||
1194 | // Beware of ConstantExpr: it may eventually evaluate to getNullValue, | ||||||
1195 | // even if currently isNullValue gives false. | ||||||
1196 | Constant *InC = dyn_cast<Constant>(PN->getIncomingValue(i)); | ||||||
1197 | // For vector constants, we cannot use isNullValue to fold into | ||||||
1198 | // FalseVInPred versus TrueVInPred. When we have individual nonzero | ||||||
1199 | // elements in the vector, we will incorrectly fold InC to | ||||||
1200 | // `TrueVInPred`. | ||||||
1201 | if (InC && isa<ConstantInt>(InC)) | ||||||
1202 | InV = InC->isNullValue() ? FalseVInPred : TrueVInPred; | ||||||
1203 | else { | ||||||
1204 | // Generate the select in the same block as PN's current incoming block. | ||||||
1205 | // Note: ThisBB need not be the NonConstBB because vector constants | ||||||
1206 | // which are constants by definition are handled here. | ||||||
1207 | // FIXME: This can lead to an increase in IR generation because we might | ||||||
1208 | // generate selects for vector constant phi operand, that could not be | ||||||
1209 | // folded to TrueVInPred or FalseVInPred as done for ConstantInt. For | ||||||
1210 | // non-vector phis, this transformation was always profitable because | ||||||
1211 | // the select would be generated exactly once in the NonConstBB. | ||||||
1212 | Builder.SetInsertPoint(ThisBB->getTerminator()); | ||||||
1213 | InV = Builder.CreateSelect(PN->getIncomingValue(i), TrueVInPred, | ||||||
1214 | FalseVInPred, "phi.sel"); | ||||||
1215 | } | ||||||
1216 | NewPN->addIncoming(InV, ThisBB); | ||||||
1217 | } | ||||||
1218 | } else if (CmpInst *CI = dyn_cast<CmpInst>(&I)) { | ||||||
1219 | Constant *C = cast<Constant>(I.getOperand(1)); | ||||||
1220 | for (unsigned i = 0; i != NumPHIValues; ++i) { | ||||||
1221 | Value *InV = nullptr; | ||||||
1222 | if (auto *InC = dyn_cast<Constant>(PN->getIncomingValue(i))) | ||||||
1223 | InV = ConstantExpr::getCompare(CI->getPredicate(), InC, C); | ||||||
1224 | else | ||||||
1225 | InV = Builder.CreateCmp(CI->getPredicate(), PN->getIncomingValue(i), | ||||||
1226 | C, "phi.cmp"); | ||||||
1227 | NewPN->addIncoming(InV, PN->getIncomingBlock(i)); | ||||||
1228 | } | ||||||
1229 | } else if (auto *BO = dyn_cast<BinaryOperator>(&I)) { | ||||||
1230 | for (unsigned i = 0; i != NumPHIValues; ++i) { | ||||||
1231 | Value *InV = foldOperationIntoPhiValue(BO, PN->getIncomingValue(i), | ||||||
1232 | Builder); | ||||||
1233 | NewPN->addIncoming(InV, PN->getIncomingBlock(i)); | ||||||
1234 | } | ||||||
1235 | } else if (isa<FreezeInst>(&I)) { | ||||||
1236 | for (unsigned i = 0; i != NumPHIValues; ++i) { | ||||||
1237 | Value *InV; | ||||||
1238 | if (NonConstBB == PN->getIncomingBlock(i)) | ||||||
1239 | InV = Builder.CreateFreeze(PN->getIncomingValue(i), "phi.fr"); | ||||||
1240 | else | ||||||
1241 | InV = PN->getIncomingValue(i); | ||||||
1242 | NewPN->addIncoming(InV, PN->getIncomingBlock(i)); | ||||||
1243 | } | ||||||
1244 | } else { | ||||||
1245 | CastInst *CI = cast<CastInst>(&I); | ||||||
1246 | Type *RetTy = CI->getType(); | ||||||
1247 | for (unsigned i = 0; i != NumPHIValues; ++i) { | ||||||
1248 | Value *InV; | ||||||
1249 | if (Constant *InC = dyn_cast<Constant>(PN->getIncomingValue(i))) | ||||||
1250 | InV = ConstantExpr::getCast(CI->getOpcode(), InC, RetTy); | ||||||
1251 | else | ||||||
1252 | InV = Builder.CreateCast(CI->getOpcode(), PN->getIncomingValue(i), | ||||||
1253 | I.getType(), "phi.cast"); | ||||||
1254 | NewPN->addIncoming(InV, PN->getIncomingBlock(i)); | ||||||
1255 | } | ||||||
1256 | } | ||||||
1257 | |||||||
1258 | for (User *U : make_early_inc_range(PN->users())) { | ||||||
1259 | Instruction *User = cast<Instruction>(U); | ||||||
1260 | if (User == &I) continue; | ||||||
1261 | replaceInstUsesWith(*User, NewPN); | ||||||
1262 | eraseInstFromFunction(*User); | ||||||
1263 | } | ||||||
1264 | return replaceInstUsesWith(I, NewPN); | ||||||
1265 | } | ||||||
1266 | |||||||
1267 | Instruction *InstCombinerImpl::foldBinOpIntoSelectOrPhi(BinaryOperator &I) { | ||||||
1268 | if (!isa<Constant>(I.getOperand(1))) | ||||||
1269 | return nullptr; | ||||||
1270 | |||||||
1271 | if (auto *Sel = dyn_cast<SelectInst>(I.getOperand(0))) { | ||||||
1272 | if (Instruction *NewSel = FoldOpIntoSelect(I, Sel)) | ||||||
1273 | return NewSel; | ||||||
1274 | } else if (auto *PN = dyn_cast<PHINode>(I.getOperand(0))) { | ||||||
1275 | if (Instruction *NewPhi = foldOpIntoPhi(I, PN)) | ||||||
1276 | return NewPhi; | ||||||
1277 | } | ||||||
1278 | return nullptr; | ||||||
1279 | } | ||||||
1280 | |||||||
1281 | /// Given a pointer type and a constant offset, determine whether or not there | ||||||
1282 | /// is a sequence of GEP indices into the pointed type that will land us at the | ||||||
1283 | /// specified offset. If so, fill them into NewIndices and return the resultant | ||||||
1284 | /// element type, otherwise return null. | ||||||
1285 | Type * | ||||||
1286 | InstCombinerImpl::FindElementAtOffset(PointerType *PtrTy, int64_t IntOffset, | ||||||
1287 | SmallVectorImpl<Value *> &NewIndices) { | ||||||
1288 | Type *Ty = PtrTy->getElementType(); | ||||||
1289 | if (!Ty->isSized()) | ||||||
1290 | return nullptr; | ||||||
1291 | |||||||
1292 | APInt Offset(DL.getIndexTypeSizeInBits(PtrTy), IntOffset); | ||||||
1293 | SmallVector<APInt> Indices = DL.getGEPIndicesForOffset(Ty, Offset); | ||||||
1294 | if (!Offset.isZero()) | ||||||
1295 | return nullptr; | ||||||
1296 | |||||||
1297 | for (const APInt &Index : Indices) | ||||||
1298 | NewIndices.push_back(Builder.getInt(Index)); | ||||||
1299 | return Ty; | ||||||
1300 | } | ||||||
1301 | |||||||
1302 | static bool shouldMergeGEPs(GEPOperator &GEP, GEPOperator &Src) { | ||||||
1303 | // If this GEP has only 0 indices, it is the same pointer as | ||||||
1304 | // Src. If Src is not a trivial GEP too, don't combine | ||||||
1305 | // the indices. | ||||||
1306 | if (GEP.hasAllZeroIndices() && !Src.hasAllZeroIndices() && | ||||||
1307 | !Src.hasOneUse()) | ||||||
1308 | return false; | ||||||
1309 | return true; | ||||||
1310 | } | ||||||
1311 | |||||||
1312 | /// Return a value X such that Val = X * Scale, or null if none. | ||||||
1313 | /// If the multiplication is known not to overflow, then NoSignedWrap is set. | ||||||
1314 | Value *InstCombinerImpl::Descale(Value *Val, APInt Scale, bool &NoSignedWrap) { | ||||||
1315 | assert(isa<IntegerType>(Val->getType()) && "Can only descale integers!")(static_cast <bool> (isa<IntegerType>(Val->getType ()) && "Can only descale integers!") ? void (0) : __assert_fail ("isa<IntegerType>(Val->getType()) && \"Can only descale integers!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1315, __extension__ __PRETTY_FUNCTION__)); | ||||||
1316 | assert(cast<IntegerType>(Val->getType())->getBitWidth() ==(static_cast <bool> (cast<IntegerType>(Val->getType ())->getBitWidth() == Scale.getBitWidth() && "Scale not compatible with value!" ) ? void (0) : __assert_fail ("cast<IntegerType>(Val->getType())->getBitWidth() == Scale.getBitWidth() && \"Scale not compatible with value!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1317, __extension__ __PRETTY_FUNCTION__)) | ||||||
1317 | Scale.getBitWidth() && "Scale not compatible with value!")(static_cast <bool> (cast<IntegerType>(Val->getType ())->getBitWidth() == Scale.getBitWidth() && "Scale not compatible with value!" ) ? void (0) : __assert_fail ("cast<IntegerType>(Val->getType())->getBitWidth() == Scale.getBitWidth() && \"Scale not compatible with value!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1317, __extension__ __PRETTY_FUNCTION__)); | ||||||
1318 | |||||||
1319 | // If Val is zero or Scale is one then Val = Val * Scale. | ||||||
1320 | if (match(Val, m_Zero()) || Scale == 1) { | ||||||
1321 | NoSignedWrap = true; | ||||||
1322 | return Val; | ||||||
1323 | } | ||||||
1324 | |||||||
1325 | // If Scale is zero then it does not divide Val. | ||||||
1326 | if (Scale.isMinValue()) | ||||||
1327 | return nullptr; | ||||||
1328 | |||||||
1329 | // Look through chains of multiplications, searching for a constant that is | ||||||
1330 | // divisible by Scale. For example, descaling X*(Y*(Z*4)) by a factor of 4 | ||||||
1331 | // will find the constant factor 4 and produce X*(Y*Z). Descaling X*(Y*8) by | ||||||
1332 | // a factor of 4 will produce X*(Y*2). The principle of operation is to bore | ||||||
1333 | // down from Val: | ||||||
1334 | // | ||||||
1335 | // Val = M1 * X || Analysis starts here and works down | ||||||
1336 | // M1 = M2 * Y || Doesn't descend into terms with more | ||||||
1337 | // M2 = Z * 4 \/ than one use | ||||||
1338 | // | ||||||
1339 | // Then to modify a term at the bottom: | ||||||
1340 | // | ||||||
1341 | // Val = M1 * X | ||||||
1342 | // M1 = Z * Y || Replaced M2 with Z | ||||||
1343 | // | ||||||
1344 | // Then to work back up correcting nsw flags. | ||||||
1345 | |||||||
1346 | // Op - the term we are currently analyzing. Starts at Val then drills down. | ||||||
1347 | // Replaced with its descaled value before exiting from the drill down loop. | ||||||
1348 | Value *Op = Val; | ||||||
1349 | |||||||
1350 | // Parent - initially null, but after drilling down notes where Op came from. | ||||||
1351 | // In the example above, Parent is (Val, 0) when Op is M1, because M1 is the | ||||||
1352 | // 0'th operand of Val. | ||||||
1353 | std::pair<Instruction *, unsigned> Parent; | ||||||
1354 | |||||||
1355 | // Set if the transform requires a descaling at deeper levels that doesn't | ||||||
1356 | // overflow. | ||||||
1357 | bool RequireNoSignedWrap = false; | ||||||
1358 | |||||||
1359 | // Log base 2 of the scale. Negative if not a power of 2. | ||||||
1360 | int32_t logScale = Scale.exactLogBase2(); | ||||||
1361 | |||||||
1362 | for (;; Op = Parent.first->getOperand(Parent.second)) { // Drill down | ||||||
1363 | if (ConstantInt *CI = dyn_cast<ConstantInt>(Op)) { | ||||||
1364 | // If Op is a constant divisible by Scale then descale to the quotient. | ||||||
1365 | APInt Quotient(Scale), Remainder(Scale); // Init ensures right bitwidth. | ||||||
1366 | APInt::sdivrem(CI->getValue(), Scale, Quotient, Remainder); | ||||||
1367 | if (!Remainder.isMinValue()) | ||||||
1368 | // Not divisible by Scale. | ||||||
1369 | return nullptr; | ||||||
1370 | // Replace with the quotient in the parent. | ||||||
1371 | Op = ConstantInt::get(CI->getType(), Quotient); | ||||||
1372 | NoSignedWrap = true; | ||||||
1373 | break; | ||||||
1374 | } | ||||||
1375 | |||||||
1376 | if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Op)) { | ||||||
1377 | if (BO->getOpcode() == Instruction::Mul) { | ||||||
1378 | // Multiplication. | ||||||
1379 | NoSignedWrap = BO->hasNoSignedWrap(); | ||||||
1380 | if (RequireNoSignedWrap && !NoSignedWrap) | ||||||
1381 | return nullptr; | ||||||
1382 | |||||||
1383 | // There are three cases for multiplication: multiplication by exactly | ||||||
1384 | // the scale, multiplication by a constant different to the scale, and | ||||||
1385 | // multiplication by something else. | ||||||
1386 | Value *LHS = BO->getOperand(0); | ||||||
1387 | Value *RHS = BO->getOperand(1); | ||||||
1388 | |||||||
1389 | if (ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) { | ||||||
1390 | // Multiplication by a constant. | ||||||
1391 | if (CI->getValue() == Scale) { | ||||||
1392 | // Multiplication by exactly the scale, replace the multiplication | ||||||
1393 | // by its left-hand side in the parent. | ||||||
1394 | Op = LHS; | ||||||
1395 | break; | ||||||
1396 | } | ||||||
1397 | |||||||
1398 | // Otherwise drill down into the constant. | ||||||
1399 | if (!Op->hasOneUse()) | ||||||
1400 | return nullptr; | ||||||
1401 | |||||||
1402 | Parent = std::make_pair(BO, 1); | ||||||
1403 | continue; | ||||||
1404 | } | ||||||
1405 | |||||||
1406 | // Multiplication by something else. Drill down into the left-hand side | ||||||
1407 | // since that's where the reassociate pass puts the good stuff. | ||||||
1408 | if (!Op->hasOneUse()) | ||||||
1409 | return nullptr; | ||||||
1410 | |||||||
1411 | Parent = std::make_pair(BO, 0); | ||||||
1412 | continue; | ||||||
1413 | } | ||||||
1414 | |||||||
1415 | if (logScale > 0 && BO->getOpcode() == Instruction::Shl && | ||||||
1416 | isa<ConstantInt>(BO->getOperand(1))) { | ||||||
1417 | // Multiplication by a power of 2. | ||||||
1418 | NoSignedWrap = BO->hasNoSignedWrap(); | ||||||
1419 | if (RequireNoSignedWrap && !NoSignedWrap) | ||||||
1420 | return nullptr; | ||||||
1421 | |||||||
1422 | Value *LHS = BO->getOperand(0); | ||||||
1423 | int32_t Amt = cast<ConstantInt>(BO->getOperand(1))-> | ||||||
1424 | getLimitedValue(Scale.getBitWidth()); | ||||||
1425 | // Op = LHS << Amt. | ||||||
1426 | |||||||
1427 | if (Amt == logScale) { | ||||||
1428 | // Multiplication by exactly the scale, replace the multiplication | ||||||
1429 | // by its left-hand side in the parent. | ||||||
1430 | Op = LHS; | ||||||
1431 | break; | ||||||
1432 | } | ||||||
1433 | if (Amt < logScale || !Op->hasOneUse()) | ||||||
1434 | return nullptr; | ||||||
1435 | |||||||
1436 | // Multiplication by more than the scale. Reduce the multiplying amount | ||||||
1437 | // by the scale in the parent. | ||||||
1438 | Parent = std::make_pair(BO, 1); | ||||||
1439 | Op = ConstantInt::get(BO->getType(), Amt - logScale); | ||||||
1440 | break; | ||||||
1441 | } | ||||||
1442 | } | ||||||
1443 | |||||||
1444 | if (!Op->hasOneUse()) | ||||||
1445 | return nullptr; | ||||||
1446 | |||||||
1447 | if (CastInst *Cast = dyn_cast<CastInst>(Op)) { | ||||||
1448 | if (Cast->getOpcode() == Instruction::SExt) { | ||||||
1449 | // Op is sign-extended from a smaller type, descale in the smaller type. | ||||||
1450 | unsigned SmallSize = Cast->getSrcTy()->getPrimitiveSizeInBits(); | ||||||
1451 | APInt SmallScale = Scale.trunc(SmallSize); | ||||||
1452 | // Suppose Op = sext X, and we descale X as Y * SmallScale. We want to | ||||||
1453 | // descale Op as (sext Y) * Scale. In order to have | ||||||
1454 | // sext (Y * SmallScale) = (sext Y) * Scale | ||||||
1455 | // some conditions need to hold however: SmallScale must sign-extend to | ||||||
1456 | // Scale and the multiplication Y * SmallScale should not overflow. | ||||||
1457 | if (SmallScale.sext(Scale.getBitWidth()) != Scale) | ||||||
1458 | // SmallScale does not sign-extend to Scale. | ||||||
1459 | return nullptr; | ||||||
1460 | assert(SmallScale.exactLogBase2() == logScale)(static_cast <bool> (SmallScale.exactLogBase2() == logScale ) ? void (0) : __assert_fail ("SmallScale.exactLogBase2() == logScale" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1460, __extension__ __PRETTY_FUNCTION__)); | ||||||
1461 | // Require that Y * SmallScale must not overflow. | ||||||
1462 | RequireNoSignedWrap = true; | ||||||
1463 | |||||||
1464 | // Drill down through the cast. | ||||||
1465 | Parent = std::make_pair(Cast, 0); | ||||||
1466 | Scale = SmallScale; | ||||||
1467 | continue; | ||||||
1468 | } | ||||||
1469 | |||||||
1470 | if (Cast->getOpcode() == Instruction::Trunc) { | ||||||
1471 | // Op is truncated from a larger type, descale in the larger type. | ||||||
1472 | // Suppose Op = trunc X, and we descale X as Y * sext Scale. Then | ||||||
1473 | // trunc (Y * sext Scale) = (trunc Y) * Scale | ||||||
1474 | // always holds. However (trunc Y) * Scale may overflow even if | ||||||
1475 | // trunc (Y * sext Scale) does not, so nsw flags need to be cleared | ||||||
1476 | // from this point up in the expression (see later). | ||||||
1477 | if (RequireNoSignedWrap) | ||||||
1478 | return nullptr; | ||||||
1479 | |||||||
1480 | // Drill down through the cast. | ||||||
1481 | unsigned LargeSize = Cast->getSrcTy()->getPrimitiveSizeInBits(); | ||||||
1482 | Parent = std::make_pair(Cast, 0); | ||||||
1483 | Scale = Scale.sext(LargeSize); | ||||||
1484 | if (logScale + 1 == (int32_t)Cast->getType()->getPrimitiveSizeInBits()) | ||||||
1485 | logScale = -1; | ||||||
1486 | assert(Scale.exactLogBase2() == logScale)(static_cast <bool> (Scale.exactLogBase2() == logScale) ? void (0) : __assert_fail ("Scale.exactLogBase2() == logScale" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1486, __extension__ __PRETTY_FUNCTION__)); | ||||||
1487 | continue; | ||||||
1488 | } | ||||||
1489 | } | ||||||
1490 | |||||||
1491 | // Unsupported expression, bail out. | ||||||
1492 | return nullptr; | ||||||
1493 | } | ||||||
1494 | |||||||
1495 | // If Op is zero then Val = Op * Scale. | ||||||
1496 | if (match(Op, m_Zero())) { | ||||||
1497 | NoSignedWrap = true; | ||||||
1498 | return Op; | ||||||
1499 | } | ||||||
1500 | |||||||
1501 | // We know that we can successfully descale, so from here on we can safely | ||||||
1502 | // modify the IR. Op holds the descaled version of the deepest term in the | ||||||
1503 | // expression. NoSignedWrap is 'true' if multiplying Op by Scale is known | ||||||
1504 | // not to overflow. | ||||||
1505 | |||||||
1506 | if (!Parent.first) | ||||||
1507 | // The expression only had one term. | ||||||
1508 | return Op; | ||||||
1509 | |||||||
1510 | // Rewrite the parent using the descaled version of its operand. | ||||||
1511 | assert(Parent.first->hasOneUse() && "Drilled down when more than one use!")(static_cast <bool> (Parent.first->hasOneUse() && "Drilled down when more than one use!") ? void (0) : __assert_fail ("Parent.first->hasOneUse() && \"Drilled down when more than one use!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1511, __extension__ __PRETTY_FUNCTION__)); | ||||||
1512 | assert(Op != Parent.first->getOperand(Parent.second) &&(static_cast <bool> (Op != Parent.first->getOperand( Parent.second) && "Descaling was a no-op?") ? void (0 ) : __assert_fail ("Op != Parent.first->getOperand(Parent.second) && \"Descaling was a no-op?\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1513, __extension__ __PRETTY_FUNCTION__)) | ||||||
1513 | "Descaling was a no-op?")(static_cast <bool> (Op != Parent.first->getOperand( Parent.second) && "Descaling was a no-op?") ? void (0 ) : __assert_fail ("Op != Parent.first->getOperand(Parent.second) && \"Descaling was a no-op?\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1513, __extension__ __PRETTY_FUNCTION__)); | ||||||
1514 | replaceOperand(*Parent.first, Parent.second, Op); | ||||||
1515 | Worklist.push(Parent.first); | ||||||
1516 | |||||||
1517 | // Now work back up the expression correcting nsw flags. The logic is based | ||||||
1518 | // on the following observation: if X * Y is known not to overflow as a signed | ||||||
1519 | // multiplication, and Y is replaced by a value Z with smaller absolute value, | ||||||
1520 | // then X * Z will not overflow as a signed multiplication either. As we work | ||||||
1521 | // our way up, having NoSignedWrap 'true' means that the descaled value at the | ||||||
1522 | // current level has strictly smaller absolute value than the original. | ||||||
1523 | Instruction *Ancestor = Parent.first; | ||||||
1524 | do { | ||||||
1525 | if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Ancestor)) { | ||||||
1526 | // If the multiplication wasn't nsw then we can't say anything about the | ||||||
1527 | // value of the descaled multiplication, and we have to clear nsw flags | ||||||
1528 | // from this point on up. | ||||||
1529 | bool OpNoSignedWrap = BO->hasNoSignedWrap(); | ||||||
1530 | NoSignedWrap &= OpNoSignedWrap; | ||||||
1531 | if (NoSignedWrap != OpNoSignedWrap) { | ||||||
1532 | BO->setHasNoSignedWrap(NoSignedWrap); | ||||||
1533 | Worklist.push(Ancestor); | ||||||
1534 | } | ||||||
1535 | } else if (Ancestor->getOpcode() == Instruction::Trunc) { | ||||||
1536 | // The fact that the descaled input to the trunc has smaller absolute | ||||||
1537 | // value than the original input doesn't tell us anything useful about | ||||||
1538 | // the absolute values of the truncations. | ||||||
1539 | NoSignedWrap = false; | ||||||
1540 | } | ||||||
1541 | assert((Ancestor->getOpcode() != Instruction::SExt || NoSignedWrap) &&(static_cast <bool> ((Ancestor->getOpcode() != Instruction ::SExt || NoSignedWrap) && "Failed to keep proper track of nsw flags while drilling down?" ) ? void (0) : __assert_fail ("(Ancestor->getOpcode() != Instruction::SExt || NoSignedWrap) && \"Failed to keep proper track of nsw flags while drilling down?\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1542, __extension__ __PRETTY_FUNCTION__)) | ||||||
1542 | "Failed to keep proper track of nsw flags while drilling down?")(static_cast <bool> ((Ancestor->getOpcode() != Instruction ::SExt || NoSignedWrap) && "Failed to keep proper track of nsw flags while drilling down?" ) ? void (0) : __assert_fail ("(Ancestor->getOpcode() != Instruction::SExt || NoSignedWrap) && \"Failed to keep proper track of nsw flags while drilling down?\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1542, __extension__ __PRETTY_FUNCTION__)); | ||||||
1543 | |||||||
1544 | if (Ancestor == Val) | ||||||
1545 | // Got to the top, all done! | ||||||
1546 | return Val; | ||||||
1547 | |||||||
1548 | // Move up one level in the expression. | ||||||
1549 | assert(Ancestor->hasOneUse() && "Drilled down when more than one use!")(static_cast <bool> (Ancestor->hasOneUse() && "Drilled down when more than one use!") ? void (0) : __assert_fail ("Ancestor->hasOneUse() && \"Drilled down when more than one use!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1549, __extension__ __PRETTY_FUNCTION__)); | ||||||
1550 | Ancestor = Ancestor->user_back(); | ||||||
1551 | } while (true); | ||||||
1552 | } | ||||||
1553 | |||||||
1554 | Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) { | ||||||
1555 | if (!isa<VectorType>(Inst.getType())) | ||||||
1556 | return nullptr; | ||||||
1557 | |||||||
1558 | BinaryOperator::BinaryOps Opcode = Inst.getOpcode(); | ||||||
1559 | Value *LHS = Inst.getOperand(0), *RHS = Inst.getOperand(1); | ||||||
1560 | assert(cast<VectorType>(LHS->getType())->getElementCount() ==(static_cast <bool> (cast<VectorType>(LHS->getType ())->getElementCount() == cast<VectorType>(Inst.getType ())->getElementCount()) ? void (0) : __assert_fail ("cast<VectorType>(LHS->getType())->getElementCount() == cast<VectorType>(Inst.getType())->getElementCount()" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1561, __extension__ __PRETTY_FUNCTION__)) | ||||||
1561 | cast<VectorType>(Inst.getType())->getElementCount())(static_cast <bool> (cast<VectorType>(LHS->getType ())->getElementCount() == cast<VectorType>(Inst.getType ())->getElementCount()) ? void (0) : __assert_fail ("cast<VectorType>(LHS->getType())->getElementCount() == cast<VectorType>(Inst.getType())->getElementCount()" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1561, __extension__ __PRETTY_FUNCTION__)); | ||||||
1562 | assert(cast<VectorType>(RHS->getType())->getElementCount() ==(static_cast <bool> (cast<VectorType>(RHS->getType ())->getElementCount() == cast<VectorType>(Inst.getType ())->getElementCount()) ? void (0) : __assert_fail ("cast<VectorType>(RHS->getType())->getElementCount() == cast<VectorType>(Inst.getType())->getElementCount()" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1563, __extension__ __PRETTY_FUNCTION__)) | ||||||
1563 | cast<VectorType>(Inst.getType())->getElementCount())(static_cast <bool> (cast<VectorType>(RHS->getType ())->getElementCount() == cast<VectorType>(Inst.getType ())->getElementCount()) ? void (0) : __assert_fail ("cast<VectorType>(RHS->getType())->getElementCount() == cast<VectorType>(Inst.getType())->getElementCount()" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1563, __extension__ __PRETTY_FUNCTION__)); | ||||||
1564 | |||||||
1565 | // If both operands of the binop are vector concatenations, then perform the | ||||||
1566 | // narrow binop on each pair of the source operands followed by concatenation | ||||||
1567 | // of the results. | ||||||
1568 | Value *L0, *L1, *R0, *R1; | ||||||
1569 | ArrayRef<int> Mask; | ||||||
1570 | if (match(LHS, m_Shuffle(m_Value(L0), m_Value(L1), m_Mask(Mask))) && | ||||||
1571 | match(RHS, m_Shuffle(m_Value(R0), m_Value(R1), m_SpecificMask(Mask))) && | ||||||
1572 | LHS->hasOneUse() && RHS->hasOneUse() && | ||||||
1573 | cast<ShuffleVectorInst>(LHS)->isConcat() && | ||||||
1574 | cast<ShuffleVectorInst>(RHS)->isConcat()) { | ||||||
1575 | // This transform does not have the speculative execution constraint as | ||||||
1576 | // below because the shuffle is a concatenation. The new binops are | ||||||
1577 | // operating on exactly the same elements as the existing binop. | ||||||
1578 | // TODO: We could ease the mask requirement to allow different undef lanes, | ||||||
1579 | // but that requires an analysis of the binop-with-undef output value. | ||||||
1580 | Value *NewBO0 = Builder.CreateBinOp(Opcode, L0, R0); | ||||||
1581 | if (auto *BO = dyn_cast<BinaryOperator>(NewBO0)) | ||||||
1582 | BO->copyIRFlags(&Inst); | ||||||
1583 | Value *NewBO1 = Builder.CreateBinOp(Opcode, L1, R1); | ||||||
1584 | if (auto *BO = dyn_cast<BinaryOperator>(NewBO1)) | ||||||
1585 | BO->copyIRFlags(&Inst); | ||||||
1586 | return new ShuffleVectorInst(NewBO0, NewBO1, Mask); | ||||||
1587 | } | ||||||
1588 | |||||||
1589 | // It may not be safe to reorder shuffles and things like div, urem, etc. | ||||||
1590 | // because we may trap when executing those ops on unknown vector elements. | ||||||
1591 | // See PR20059. | ||||||
1592 | if (!isSafeToSpeculativelyExecute(&Inst)) | ||||||
1593 | return nullptr; | ||||||
1594 | |||||||
1595 | auto createBinOpShuffle = [&](Value *X, Value *Y, ArrayRef<int> M) { | ||||||
1596 | Value *XY = Builder.CreateBinOp(Opcode, X, Y); | ||||||
1597 | if (auto *BO = dyn_cast<BinaryOperator>(XY)) | ||||||
1598 | BO->copyIRFlags(&Inst); | ||||||
1599 | return new ShuffleVectorInst(XY, M); | ||||||
1600 | }; | ||||||
1601 | |||||||
1602 | // If both arguments of the binary operation are shuffles that use the same | ||||||
1603 | // mask and shuffle within a single vector, move the shuffle after the binop. | ||||||
1604 | Value *V1, *V2; | ||||||
1605 | if (match(LHS, m_Shuffle(m_Value(V1), m_Undef(), m_Mask(Mask))) && | ||||||
1606 | match(RHS, m_Shuffle(m_Value(V2), m_Undef(), m_SpecificMask(Mask))) && | ||||||
1607 | V1->getType() == V2->getType() && | ||||||
1608 | (LHS->hasOneUse() || RHS->hasOneUse() || LHS == RHS)) { | ||||||
1609 | // Op(shuffle(V1, Mask), shuffle(V2, Mask)) -> shuffle(Op(V1, V2), Mask) | ||||||
1610 | return createBinOpShuffle(V1, V2, Mask); | ||||||
1611 | } | ||||||
1612 | |||||||
1613 | // If both arguments of a commutative binop are select-shuffles that use the | ||||||
1614 | // same mask with commuted operands, the shuffles are unnecessary. | ||||||
1615 | if (Inst.isCommutative() && | ||||||
1616 | match(LHS, m_Shuffle(m_Value(V1), m_Value(V2), m_Mask(Mask))) && | ||||||
1617 | match(RHS, | ||||||
1618 | m_Shuffle(m_Specific(V2), m_Specific(V1), m_SpecificMask(Mask)))) { | ||||||
1619 | auto *LShuf = cast<ShuffleVectorInst>(LHS); | ||||||
1620 | auto *RShuf = cast<ShuffleVectorInst>(RHS); | ||||||
1621 | // TODO: Allow shuffles that contain undefs in the mask? | ||||||
1622 | // That is legal, but it reduces undef knowledge. | ||||||
1623 | // TODO: Allow arbitrary shuffles by shuffling after binop? | ||||||
1624 | // That might be legal, but we have to deal with poison. | ||||||
1625 | if (LShuf->isSelect() && | ||||||
1626 | !is_contained(LShuf->getShuffleMask(), UndefMaskElem) && | ||||||
1627 | RShuf->isSelect() && | ||||||
1628 | !is_contained(RShuf->getShuffleMask(), UndefMaskElem)) { | ||||||
1629 | // Example: | ||||||
1630 | // LHS = shuffle V1, V2, <0, 5, 6, 3> | ||||||
1631 | // RHS = shuffle V2, V1, <0, 5, 6, 3> | ||||||
1632 | // LHS + RHS --> (V10+V20, V21+V11, V22+V12, V13+V23) --> V1 + V2 | ||||||
1633 | Instruction *NewBO = BinaryOperator::Create(Opcode, V1, V2); | ||||||
1634 | NewBO->copyIRFlags(&Inst); | ||||||
1635 | return NewBO; | ||||||
1636 | } | ||||||
1637 | } | ||||||
1638 | |||||||
1639 | // If one argument is a shuffle within one vector and the other is a constant, | ||||||
1640 | // try moving the shuffle after the binary operation. This canonicalization | ||||||
1641 | // intends to move shuffles closer to other shuffles and binops closer to | ||||||
1642 | // other binops, so they can be folded. It may also enable demanded elements | ||||||
1643 | // transforms. | ||||||
1644 | Constant *C; | ||||||
1645 | auto *InstVTy = dyn_cast<FixedVectorType>(Inst.getType()); | ||||||
1646 | if (InstVTy && | ||||||
1647 | match(&Inst, | ||||||
1648 | m_c_BinOp(m_OneUse(m_Shuffle(m_Value(V1), m_Undef(), m_Mask(Mask))), | ||||||
1649 | m_ImmConstant(C))) && | ||||||
1650 | cast<FixedVectorType>(V1->getType())->getNumElements() <= | ||||||
1651 | InstVTy->getNumElements()) { | ||||||
1652 | assert(InstVTy->getScalarType() == V1->getType()->getScalarType() &&(static_cast <bool> (InstVTy->getScalarType() == V1-> getType()->getScalarType() && "Shuffle should not change scalar type" ) ? void (0) : __assert_fail ("InstVTy->getScalarType() == V1->getType()->getScalarType() && \"Shuffle should not change scalar type\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1653, __extension__ __PRETTY_FUNCTION__)) | ||||||
1653 | "Shuffle should not change scalar type")(static_cast <bool> (InstVTy->getScalarType() == V1-> getType()->getScalarType() && "Shuffle should not change scalar type" ) ? void (0) : __assert_fail ("InstVTy->getScalarType() == V1->getType()->getScalarType() && \"Shuffle should not change scalar type\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1653, __extension__ __PRETTY_FUNCTION__)); | ||||||
1654 | |||||||
1655 | // Find constant NewC that has property: | ||||||
1656 | // shuffle(NewC, ShMask) = C | ||||||
1657 | // If such constant does not exist (example: ShMask=<0,0> and C=<1,2>) | ||||||
1658 | // reorder is not possible. A 1-to-1 mapping is not required. Example: | ||||||
1659 | // ShMask = <1,1,2,2> and C = <5,5,6,6> --> NewC = <undef,5,6,undef> | ||||||
1660 | bool ConstOp1 = isa<Constant>(RHS); | ||||||
1661 | ArrayRef<int> ShMask = Mask; | ||||||
1662 | unsigned SrcVecNumElts = | ||||||
1663 | cast<FixedVectorType>(V1->getType())->getNumElements(); | ||||||
1664 | UndefValue *UndefScalar = UndefValue::get(C->getType()->getScalarType()); | ||||||
1665 | SmallVector<Constant *, 16> NewVecC(SrcVecNumElts, UndefScalar); | ||||||
1666 | bool MayChange = true; | ||||||
1667 | unsigned NumElts = InstVTy->getNumElements(); | ||||||
1668 | for (unsigned I = 0; I < NumElts; ++I) { | ||||||
1669 | Constant *CElt = C->getAggregateElement(I); | ||||||
1670 | if (ShMask[I] >= 0) { | ||||||
1671 | assert(ShMask[I] < (int)NumElts && "Not expecting narrowing shuffle")(static_cast <bool> (ShMask[I] < (int)NumElts && "Not expecting narrowing shuffle") ? void (0) : __assert_fail ("ShMask[I] < (int)NumElts && \"Not expecting narrowing shuffle\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1671, __extension__ __PRETTY_FUNCTION__)); | ||||||
1672 | Constant *NewCElt = NewVecC[ShMask[I]]; | ||||||
1673 | // Bail out if: | ||||||
1674 | // 1. The constant vector contains a constant expression. | ||||||
1675 | // 2. The shuffle needs an element of the constant vector that can't | ||||||
1676 | // be mapped to a new constant vector. | ||||||
1677 | // 3. This is a widening shuffle that copies elements of V1 into the | ||||||
1678 | // extended elements (extending with undef is allowed). | ||||||
1679 | if (!CElt || (!isa<UndefValue>(NewCElt) && NewCElt != CElt) || | ||||||
1680 | I >= SrcVecNumElts) { | ||||||
1681 | MayChange = false; | ||||||
1682 | break; | ||||||
1683 | } | ||||||
1684 | NewVecC[ShMask[I]] = CElt; | ||||||
1685 | } | ||||||
1686 | // If this is a widening shuffle, we must be able to extend with undef | ||||||
1687 | // elements. If the original binop does not produce an undef in the high | ||||||
1688 | // lanes, then this transform is not safe. | ||||||
1689 | // Similarly for undef lanes due to the shuffle mask, we can only | ||||||
1690 | // transform binops that preserve undef. | ||||||
1691 | // TODO: We could shuffle those non-undef constant values into the | ||||||
1692 | // result by using a constant vector (rather than an undef vector) | ||||||
1693 | // as operand 1 of the new binop, but that might be too aggressive | ||||||
1694 | // for target-independent shuffle creation. | ||||||
1695 | if (I >= SrcVecNumElts || ShMask[I] < 0) { | ||||||
1696 | Constant *MaybeUndef = | ||||||
1697 | ConstOp1 ? ConstantExpr::get(Opcode, UndefScalar, CElt) | ||||||
1698 | : ConstantExpr::get(Opcode, CElt, UndefScalar); | ||||||
1699 | if (!match(MaybeUndef, m_Undef())) { | ||||||
1700 | MayChange = false; | ||||||
1701 | break; | ||||||
1702 | } | ||||||
1703 | } | ||||||
1704 | } | ||||||
1705 | if (MayChange) { | ||||||
1706 | Constant *NewC = ConstantVector::get(NewVecC); | ||||||
1707 | // It may not be safe to execute a binop on a vector with undef elements | ||||||
1708 | // because the entire instruction can be folded to undef or create poison | ||||||
1709 | // that did not exist in the original code. | ||||||
1710 | if (Inst.isIntDivRem() || (Inst.isShift() && ConstOp1)) | ||||||
1711 | NewC = getSafeVectorConstantForBinop(Opcode, NewC, ConstOp1); | ||||||
1712 | |||||||
1713 | // Op(shuffle(V1, Mask), C) -> shuffle(Op(V1, NewC), Mask) | ||||||
1714 | // Op(C, shuffle(V1, Mask)) -> shuffle(Op(NewC, V1), Mask) | ||||||
1715 | Value *NewLHS = ConstOp1 ? V1 : NewC; | ||||||
1716 | Value *NewRHS = ConstOp1 ? NewC : V1; | ||||||
1717 | return createBinOpShuffle(NewLHS, NewRHS, Mask); | ||||||
1718 | } | ||||||
1719 | } | ||||||
1720 | |||||||
1721 | // Try to reassociate to sink a splat shuffle after a binary operation. | ||||||
1722 | if (Inst.isAssociative() && Inst.isCommutative()) { | ||||||
1723 | // Canonicalize shuffle operand as LHS. | ||||||
1724 | if (isa<ShuffleVectorInst>(RHS)) | ||||||
1725 | std::swap(LHS, RHS); | ||||||
1726 | |||||||
1727 | Value *X; | ||||||
1728 | ArrayRef<int> MaskC; | ||||||
1729 | int SplatIndex; | ||||||
1730 | BinaryOperator *BO; | ||||||
1731 | if (!match(LHS, | ||||||
1732 | m_OneUse(m_Shuffle(m_Value(X), m_Undef(), m_Mask(MaskC)))) || | ||||||
1733 | !match(MaskC, m_SplatOrUndefMask(SplatIndex)) || | ||||||
1734 | X->getType() != Inst.getType() || !match(RHS, m_OneUse(m_BinOp(BO))) || | ||||||
1735 | BO->getOpcode() != Opcode) | ||||||
1736 | return nullptr; | ||||||
1737 | |||||||
1738 | // FIXME: This may not be safe if the analysis allows undef elements. By | ||||||
1739 | // moving 'Y' before the splat shuffle, we are implicitly assuming | ||||||
1740 | // that it is not undef/poison at the splat index. | ||||||
1741 | Value *Y, *OtherOp; | ||||||
1742 | if (isSplatValue(BO->getOperand(0), SplatIndex)) { | ||||||
1743 | Y = BO->getOperand(0); | ||||||
1744 | OtherOp = BO->getOperand(1); | ||||||
1745 | } else if (isSplatValue(BO->getOperand(1), SplatIndex)) { | ||||||
1746 | Y = BO->getOperand(1); | ||||||
1747 | OtherOp = BO->getOperand(0); | ||||||
1748 | } else { | ||||||
1749 | return nullptr; | ||||||
1750 | } | ||||||
1751 | |||||||
1752 | // X and Y are splatted values, so perform the binary operation on those | ||||||
1753 | // values followed by a splat followed by the 2nd binary operation: | ||||||
1754 | // bo (splat X), (bo Y, OtherOp) --> bo (splat (bo X, Y)), OtherOp | ||||||
1755 | Value *NewBO = Builder.CreateBinOp(Opcode, X, Y); | ||||||
1756 | SmallVector<int, 8> NewMask(MaskC.size(), SplatIndex); | ||||||
1757 | Value *NewSplat = Builder.CreateShuffleVector(NewBO, NewMask); | ||||||
1758 | Instruction *R = BinaryOperator::Create(Opcode, NewSplat, OtherOp); | ||||||
1759 | |||||||
1760 | // Intersect FMF on both new binops. Other (poison-generating) flags are | ||||||
1761 | // dropped to be safe. | ||||||
1762 | if (isa<FPMathOperator>(R)) { | ||||||
1763 | R->copyFastMathFlags(&Inst); | ||||||
1764 | R->andIRFlags(BO); | ||||||
1765 | } | ||||||
1766 | if (auto *NewInstBO = dyn_cast<BinaryOperator>(NewBO)) | ||||||
1767 | NewInstBO->copyIRFlags(R); | ||||||
1768 | return R; | ||||||
1769 | } | ||||||
1770 | |||||||
1771 | return nullptr; | ||||||
1772 | } | ||||||
1773 | |||||||
1774 | /// Try to narrow the width of a binop if at least 1 operand is an extend of | ||||||
1775 | /// of a value. This requires a potentially expensive known bits check to make | ||||||
1776 | /// sure the narrow op does not overflow. | ||||||
1777 | Instruction *InstCombinerImpl::narrowMathIfNoOverflow(BinaryOperator &BO) { | ||||||
1778 | // We need at least one extended operand. | ||||||
1779 | Value *Op0 = BO.getOperand(0), *Op1 = BO.getOperand(1); | ||||||
1780 | |||||||
1781 | // If this is a sub, we swap the operands since we always want an extension | ||||||
1782 | // on the RHS. The LHS can be an extension or a constant. | ||||||
1783 | if (BO.getOpcode() == Instruction::Sub) | ||||||
1784 | std::swap(Op0, Op1); | ||||||
1785 | |||||||
1786 | Value *X; | ||||||
1787 | bool IsSext = match(Op0, m_SExt(m_Value(X))); | ||||||
1788 | if (!IsSext && !match(Op0, m_ZExt(m_Value(X)))) | ||||||
1789 | return nullptr; | ||||||
1790 | |||||||
1791 | // If both operands are the same extension from the same source type and we | ||||||
1792 | // can eliminate at least one (hasOneUse), this might work. | ||||||
1793 | CastInst::CastOps CastOpc = IsSext ? Instruction::SExt : Instruction::ZExt; | ||||||
1794 | Value *Y; | ||||||
1795 | if (!(match(Op1, m_ZExtOrSExt(m_Value(Y))) && X->getType() == Y->getType() && | ||||||
1796 | cast<Operator>(Op1)->getOpcode() == CastOpc && | ||||||
1797 | (Op0->hasOneUse() || Op1->hasOneUse()))) { | ||||||
1798 | // If that did not match, see if we have a suitable constant operand. | ||||||
1799 | // Truncating and extending must produce the same constant. | ||||||
1800 | Constant *WideC; | ||||||
1801 | if (!Op0->hasOneUse() || !match(Op1, m_Constant(WideC))) | ||||||
1802 | return nullptr; | ||||||
1803 | Constant *NarrowC = ConstantExpr::getTrunc(WideC, X->getType()); | ||||||
1804 | if (ConstantExpr::getCast(CastOpc, NarrowC, BO.getType()) != WideC) | ||||||
1805 | return nullptr; | ||||||
1806 | Y = NarrowC; | ||||||
1807 | } | ||||||
1808 | |||||||
1809 | // Swap back now that we found our operands. | ||||||
1810 | if (BO.getOpcode() == Instruction::Sub) | ||||||
1811 | std::swap(X, Y); | ||||||
1812 | |||||||
1813 | // Both operands have narrow versions. Last step: the math must not overflow | ||||||
1814 | // in the narrow width. | ||||||
1815 | if (!willNotOverflow(BO.getOpcode(), X, Y, BO, IsSext)) | ||||||
1816 | return nullptr; | ||||||
1817 | |||||||
1818 | // bo (ext X), (ext Y) --> ext (bo X, Y) | ||||||
1819 | // bo (ext X), C --> ext (bo X, C') | ||||||
1820 | Value *NarrowBO = Builder.CreateBinOp(BO.getOpcode(), X, Y, "narrow"); | ||||||
1821 | if (auto *NewBinOp = dyn_cast<BinaryOperator>(NarrowBO)) { | ||||||
1822 | if (IsSext) | ||||||
1823 | NewBinOp->setHasNoSignedWrap(); | ||||||
1824 | else | ||||||
1825 | NewBinOp->setHasNoUnsignedWrap(); | ||||||
1826 | } | ||||||
1827 | return CastInst::Create(CastOpc, NarrowBO, BO.getType()); | ||||||
1828 | } | ||||||
1829 | |||||||
1830 | static bool isMergedGEPInBounds(GEPOperator &GEP1, GEPOperator &GEP2) { | ||||||
1831 | // At least one GEP must be inbounds. | ||||||
1832 | if (!GEP1.isInBounds() && !GEP2.isInBounds()) | ||||||
1833 | return false; | ||||||
1834 | |||||||
1835 | return (GEP1.isInBounds() || GEP1.hasAllZeroIndices()) && | ||||||
1836 | (GEP2.isInBounds() || GEP2.hasAllZeroIndices()); | ||||||
1837 | } | ||||||
1838 | |||||||
1839 | /// Thread a GEP operation with constant indices through the constant true/false | ||||||
1840 | /// arms of a select. | ||||||
1841 | static Instruction *foldSelectGEP(GetElementPtrInst &GEP, | ||||||
1842 | InstCombiner::BuilderTy &Builder) { | ||||||
1843 | if (!GEP.hasAllConstantIndices()) | ||||||
1844 | return nullptr; | ||||||
1845 | |||||||
1846 | Instruction *Sel; | ||||||
1847 | Value *Cond; | ||||||
1848 | Constant *TrueC, *FalseC; | ||||||
1849 | if (!match(GEP.getPointerOperand(), m_Instruction(Sel)) || | ||||||
1850 | !match(Sel, | ||||||
1851 | m_Select(m_Value(Cond), m_Constant(TrueC), m_Constant(FalseC)))) | ||||||
1852 | return nullptr; | ||||||
1853 | |||||||
1854 | // gep (select Cond, TrueC, FalseC), IndexC --> select Cond, TrueC', FalseC' | ||||||
1855 | // Propagate 'inbounds' and metadata from existing instructions. | ||||||
1856 | // Note: using IRBuilder to create the constants for efficiency. | ||||||
1857 | SmallVector<Value *, 4> IndexC(GEP.indices()); | ||||||
1858 | bool IsInBounds = GEP.isInBounds(); | ||||||
1859 | Type *Ty = GEP.getSourceElementType(); | ||||||
1860 | Value *NewTrueC = IsInBounds ? Builder.CreateInBoundsGEP(Ty, TrueC, IndexC) | ||||||
1861 | : Builder.CreateGEP(Ty, TrueC, IndexC); | ||||||
1862 | Value *NewFalseC = IsInBounds ? Builder.CreateInBoundsGEP(Ty, FalseC, IndexC) | ||||||
1863 | : Builder.CreateGEP(Ty, FalseC, IndexC); | ||||||
1864 | return SelectInst::Create(Cond, NewTrueC, NewFalseC, "", nullptr, Sel); | ||||||
1865 | } | ||||||
1866 | |||||||
1867 | Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) { | ||||||
1868 | SmallVector<Value *, 8> Ops(GEP.operands()); | ||||||
1869 | Type *GEPType = GEP.getType(); | ||||||
1870 | Type *GEPEltType = GEP.getSourceElementType(); | ||||||
1871 | bool IsGEPSrcEleScalable = isa<ScalableVectorType>(GEPEltType); | ||||||
1872 | if (Value *V = SimplifyGEPInst(GEPEltType, Ops, GEP.isInBounds(), | ||||||
1873 | SQ.getWithInstruction(&GEP))) | ||||||
1874 | return replaceInstUsesWith(GEP, V); | ||||||
1875 | |||||||
1876 | // For vector geps, use the generic demanded vector support. | ||||||
1877 | // Skip if GEP return type is scalable. The number of elements is unknown at | ||||||
1878 | // compile-time. | ||||||
1879 | if (auto *GEPFVTy = dyn_cast<FixedVectorType>(GEPType)) { | ||||||
1880 | auto VWidth = GEPFVTy->getNumElements(); | ||||||
1881 | APInt UndefElts(VWidth, 0); | ||||||
1882 | APInt AllOnesEltMask(APInt::getAllOnes(VWidth)); | ||||||
1883 | if (Value *V = SimplifyDemandedVectorElts(&GEP, AllOnesEltMask, | ||||||
1884 | UndefElts)) { | ||||||
1885 | if (V != &GEP) | ||||||
1886 | return replaceInstUsesWith(GEP, V); | ||||||
1887 | return &GEP; | ||||||
1888 | } | ||||||
1889 | |||||||
1890 | // TODO: 1) Scalarize splat operands, 2) scalarize entire instruction if | ||||||
1891 | // possible (decide on canonical form for pointer broadcast), 3) exploit | ||||||
1892 | // undef elements to decrease demanded bits | ||||||
1893 | } | ||||||
1894 | |||||||
1895 | Value *PtrOp = GEP.getOperand(0); | ||||||
1896 | |||||||
1897 | // Eliminate unneeded casts for indices, and replace indices which displace | ||||||
1898 | // by multiples of a zero size type with zero. | ||||||
1899 | bool MadeChange = false; | ||||||
1900 | |||||||
1901 | // Index width may not be the same width as pointer width. | ||||||
1902 | // Data layout chooses the right type based on supported integer types. | ||||||
1903 | Type *NewScalarIndexTy = | ||||||
1904 | DL.getIndexType(GEP.getPointerOperandType()->getScalarType()); | ||||||
1905 | |||||||
1906 | gep_type_iterator GTI = gep_type_begin(GEP); | ||||||
1907 | for (User::op_iterator I = GEP.op_begin() + 1, E = GEP.op_end(); I != E; | ||||||
1908 | ++I, ++GTI) { | ||||||
1909 | // Skip indices into struct types. | ||||||
1910 | if (GTI.isStruct()) | ||||||
1911 | continue; | ||||||
1912 | |||||||
1913 | Type *IndexTy = (*I)->getType(); | ||||||
1914 | Type *NewIndexType = | ||||||
1915 | IndexTy->isVectorTy() | ||||||
1916 | ? VectorType::get(NewScalarIndexTy, | ||||||
1917 | cast<VectorType>(IndexTy)->getElementCount()) | ||||||
1918 | : NewScalarIndexTy; | ||||||
1919 | |||||||
1920 | // If the element type has zero size then any index over it is equivalent | ||||||
1921 | // to an index of zero, so replace it with zero if it is not zero already. | ||||||
1922 | Type *EltTy = GTI.getIndexedType(); | ||||||
1923 | if (EltTy->isSized() && DL.getTypeAllocSize(EltTy).isZero()) | ||||||
1924 | if (!isa<Constant>(*I) || !match(I->get(), m_Zero())) { | ||||||
1925 | *I = Constant::getNullValue(NewIndexType); | ||||||
1926 | MadeChange = true; | ||||||
1927 | } | ||||||
1928 | |||||||
1929 | if (IndexTy != NewIndexType) { | ||||||
1930 | // If we are using a wider index than needed for this platform, shrink | ||||||
1931 | // it to what we need. If narrower, sign-extend it to what we need. | ||||||
1932 | // This explicit cast can make subsequent optimizations more obvious. | ||||||
1933 | *I = Builder.CreateIntCast(*I, NewIndexType, true); | ||||||
1934 | MadeChange = true; | ||||||
1935 | } | ||||||
1936 | } | ||||||
1937 | if (MadeChange) | ||||||
1938 | return &GEP; | ||||||
1939 | |||||||
1940 | // Check to see if the inputs to the PHI node are getelementptr instructions. | ||||||
1941 | if (auto *PN = dyn_cast<PHINode>(PtrOp)) { | ||||||
1942 | auto *Op1 = dyn_cast<GetElementPtrInst>(PN->getOperand(0)); | ||||||
1943 | if (!Op1) | ||||||
1944 | return nullptr; | ||||||
1945 | |||||||
1946 | // Don't fold a GEP into itself through a PHI node. This can only happen | ||||||
1947 | // through the back-edge of a loop. Folding a GEP into itself means that | ||||||
1948 | // the value of the previous iteration needs to be stored in the meantime, | ||||||
1949 | // thus requiring an additional register variable to be live, but not | ||||||
1950 | // actually achieving anything (the GEP still needs to be executed once per | ||||||
1951 | // loop iteration). | ||||||
1952 | if (Op1 == &GEP) | ||||||
1953 | return nullptr; | ||||||
1954 | |||||||
1955 | int DI = -1; | ||||||
1956 | |||||||
1957 | for (auto I = PN->op_begin()+1, E = PN->op_end(); I !=E; ++I) { | ||||||
1958 | auto *Op2 = dyn_cast<GetElementPtrInst>(*I); | ||||||
1959 | if (!Op2 || Op1->getNumOperands() != Op2->getNumOperands()) | ||||||
1960 | return nullptr; | ||||||
1961 | |||||||
1962 | // As for Op1 above, don't try to fold a GEP into itself. | ||||||
1963 | if (Op2 == &GEP) | ||||||
1964 | return nullptr; | ||||||
1965 | |||||||
1966 | // Keep track of the type as we walk the GEP. | ||||||
1967 | Type *CurTy = nullptr; | ||||||
1968 | |||||||
1969 | for (unsigned J = 0, F = Op1->getNumOperands(); J != F; ++J) { | ||||||
1970 | if (Op1->getOperand(J)->getType() != Op2->getOperand(J)->getType()) | ||||||
1971 | return nullptr; | ||||||
1972 | |||||||
1973 | if (Op1->getOperand(J) != Op2->getOperand(J)) { | ||||||
1974 | if (DI == -1) { | ||||||
1975 | // We have not seen any differences yet in the GEPs feeding the | ||||||
1976 | // PHI yet, so we record this one if it is allowed to be a | ||||||
1977 | // variable. | ||||||
1978 | |||||||
1979 | // The first two arguments can vary for any GEP, the rest have to be | ||||||
1980 | // static for struct slots | ||||||
1981 | if (J > 1) { | ||||||
1982 | assert(CurTy && "No current type?")(static_cast <bool> (CurTy && "No current type?" ) ? void (0) : __assert_fail ("CurTy && \"No current type?\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 1982, __extension__ __PRETTY_FUNCTION__)); | ||||||
1983 | if (CurTy->isStructTy()) | ||||||
1984 | return nullptr; | ||||||
1985 | } | ||||||
1986 | |||||||
1987 | DI = J; | ||||||
1988 | } else { | ||||||
1989 | // The GEP is different by more than one input. While this could be | ||||||
1990 | // extended to support GEPs that vary by more than one variable it | ||||||
1991 | // doesn't make sense since it greatly increases the complexity and | ||||||
1992 | // would result in an R+R+R addressing mode which no backend | ||||||
1993 | // directly supports and would need to be broken into several | ||||||
1994 | // simpler instructions anyway. | ||||||
1995 | return nullptr; | ||||||
1996 | } | ||||||
1997 | } | ||||||
1998 | |||||||
1999 | // Sink down a layer of the type for the next iteration. | ||||||
2000 | if (J > 0) { | ||||||
2001 | if (J == 1) { | ||||||
2002 | CurTy = Op1->getSourceElementType(); | ||||||
2003 | } else { | ||||||
2004 | CurTy = | ||||||
2005 | GetElementPtrInst::getTypeAtIndex(CurTy, Op1->getOperand(J)); | ||||||
2006 | } | ||||||
2007 | } | ||||||
2008 | } | ||||||
2009 | } | ||||||
2010 | |||||||
2011 | // If not all GEPs are identical we'll have to create a new PHI node. | ||||||
2012 | // Check that the old PHI node has only one use so that it will get | ||||||
2013 | // removed. | ||||||
2014 | if (DI != -1 && !PN->hasOneUse()) | ||||||
2015 | return nullptr; | ||||||
2016 | |||||||
2017 | auto *NewGEP = cast<GetElementPtrInst>(Op1->clone()); | ||||||
2018 | if (DI == -1) { | ||||||
2019 | // All the GEPs feeding the PHI are identical. Clone one down into our | ||||||
2020 | // BB so that it can be merged with the current GEP. | ||||||
2021 | } else { | ||||||
2022 | // All the GEPs feeding the PHI differ at a single offset. Clone a GEP | ||||||
2023 | // into the current block so it can be merged, and create a new PHI to | ||||||
2024 | // set that index. | ||||||
2025 | PHINode *NewPN; | ||||||
2026 | { | ||||||
2027 | IRBuilderBase::InsertPointGuard Guard(Builder); | ||||||
2028 | Builder.SetInsertPoint(PN); | ||||||
2029 | NewPN = Builder.CreatePHI(Op1->getOperand(DI)->getType(), | ||||||
2030 | PN->getNumOperands()); | ||||||
2031 | } | ||||||
2032 | |||||||
2033 | for (auto &I : PN->operands()) | ||||||
2034 | NewPN->addIncoming(cast<GEPOperator>(I)->getOperand(DI), | ||||||
2035 | PN->getIncomingBlock(I)); | ||||||
2036 | |||||||
2037 | NewGEP->setOperand(DI, NewPN); | ||||||
2038 | } | ||||||
2039 | |||||||
2040 | GEP.getParent()->getInstList().insert( | ||||||
2041 | GEP.getParent()->getFirstInsertionPt(), NewGEP); | ||||||
2042 | replaceOperand(GEP, 0, NewGEP); | ||||||
2043 | PtrOp = NewGEP; | ||||||
2044 | } | ||||||
2045 | |||||||
2046 | // Combine Indices - If the source pointer to this getelementptr instruction | ||||||
2047 | // is a getelementptr instruction, combine the indices of the two | ||||||
2048 | // getelementptr instructions into a single instruction. | ||||||
2049 | if (auto *Src = dyn_cast<GEPOperator>(PtrOp)) { | ||||||
2050 | if (!shouldMergeGEPs(*cast<GEPOperator>(&GEP), *Src)) | ||||||
2051 | return nullptr; | ||||||
2052 | |||||||
2053 | if (Src->getNumOperands() == 2 && GEP.getNumOperands() == 2 && | ||||||
2054 | Src->hasOneUse()) { | ||||||
2055 | Value *GO1 = GEP.getOperand(1); | ||||||
2056 | Value *SO1 = Src->getOperand(1); | ||||||
2057 | |||||||
2058 | if (LI) { | ||||||
2059 | // Try to reassociate loop invariant GEP chains to enable LICM. | ||||||
2060 | if (Loop *L = LI->getLoopFor(GEP.getParent())) { | ||||||
2061 | // Reassociate the two GEPs if SO1 is variant in the loop and GO1 is | ||||||
2062 | // invariant: this breaks the dependence between GEPs and allows LICM | ||||||
2063 | // to hoist the invariant part out of the loop. | ||||||
2064 | if (L->isLoopInvariant(GO1) && !L->isLoopInvariant(SO1)) { | ||||||
2065 | // We have to be careful here. | ||||||
2066 | // We have something like: | ||||||
2067 | // %src = getelementptr <ty>, <ty>* %base, <ty> %idx | ||||||
2068 | // %gep = getelementptr <ty>, <ty>* %src, <ty> %idx2 | ||||||
2069 | // If we just swap idx & idx2 then we could inadvertantly | ||||||
2070 | // change %src from a vector to a scalar, or vice versa. | ||||||
2071 | // Cases: | ||||||
2072 | // 1) %base a scalar & idx a scalar & idx2 a vector | ||||||
2073 | // => Swapping idx & idx2 turns %src into a vector type. | ||||||
2074 | // 2) %base a scalar & idx a vector & idx2 a scalar | ||||||
2075 | // => Swapping idx & idx2 turns %src in a scalar type | ||||||
2076 | // 3) %base, %idx, and %idx2 are scalars | ||||||
2077 | // => %src & %gep are scalars | ||||||
2078 | // => swapping idx & idx2 is safe | ||||||
2079 | // 4) %base a vector | ||||||
2080 | // => %src is a vector | ||||||
2081 | // => swapping idx & idx2 is safe. | ||||||
2082 | auto *SO0 = Src->getOperand(0); | ||||||
2083 | auto *SO0Ty = SO0->getType(); | ||||||
2084 | if (!isa<VectorType>(GEPType) || // case 3 | ||||||
2085 | isa<VectorType>(SO0Ty)) { // case 4 | ||||||
2086 | Src->setOperand(1, GO1); | ||||||
2087 | GEP.setOperand(1, SO1); | ||||||
2088 | return &GEP; | ||||||
2089 | } else { | ||||||
2090 | // Case 1 or 2 | ||||||
2091 | // -- have to recreate %src & %gep | ||||||
2092 | // put NewSrc at same location as %src | ||||||
2093 | Builder.SetInsertPoint(cast<Instruction>(PtrOp)); | ||||||
2094 | Value *NewSrc = | ||||||
2095 | Builder.CreateGEP(GEPEltType, SO0, GO1, Src->getName()); | ||||||
2096 | // Propagate 'inbounds' if the new source was not constant-folded. | ||||||
2097 | if (auto *NewSrcGEPI = dyn_cast<GetElementPtrInst>(NewSrc)) | ||||||
2098 | NewSrcGEPI->setIsInBounds(Src->isInBounds()); | ||||||
2099 | GetElementPtrInst *NewGEP = | ||||||
2100 | GetElementPtrInst::Create(GEPEltType, NewSrc, {SO1}); | ||||||
2101 | NewGEP->setIsInBounds(GEP.isInBounds()); | ||||||
2102 | return NewGEP; | ||||||
2103 | } | ||||||
2104 | } | ||||||
2105 | } | ||||||
2106 | } | ||||||
2107 | } | ||||||
2108 | |||||||
2109 | // Note that if our source is a gep chain itself then we wait for that | ||||||
2110 | // chain to be resolved before we perform this transformation. This | ||||||
2111 | // avoids us creating a TON of code in some cases. | ||||||
2112 | if (auto *SrcGEP = dyn_cast<GEPOperator>(Src->getOperand(0))) | ||||||
2113 | if (SrcGEP->getNumOperands() == 2 && shouldMergeGEPs(*Src, *SrcGEP)) | ||||||
2114 | return nullptr; // Wait until our source is folded to completion. | ||||||
2115 | |||||||
2116 | SmallVector<Value*, 8> Indices; | ||||||
2117 | |||||||
2118 | // Find out whether the last index in the source GEP is a sequential idx. | ||||||
2119 | bool EndsWithSequential = false; | ||||||
2120 | for (gep_type_iterator I = gep_type_begin(*Src), E = gep_type_end(*Src); | ||||||
2121 | I != E; ++I) | ||||||
2122 | EndsWithSequential = I.isSequential(); | ||||||
2123 | |||||||
2124 | // Can we combine the two pointer arithmetics offsets? | ||||||
2125 | if (EndsWithSequential) { | ||||||
2126 | // Replace: gep (gep %P, long B), long A, ... | ||||||
2127 | // With: T = long A+B; gep %P, T, ... | ||||||
2128 | Value *SO1 = Src->getOperand(Src->getNumOperands()-1); | ||||||
2129 | Value *GO1 = GEP.getOperand(1); | ||||||
2130 | |||||||
2131 | // If they aren't the same type, then the input hasn't been processed | ||||||
2132 | // by the loop above yet (which canonicalizes sequential index types to | ||||||
2133 | // intptr_t). Just avoid transforming this until the input has been | ||||||
2134 | // normalized. | ||||||
2135 | if (SO1->getType() != GO1->getType()) | ||||||
2136 | return nullptr; | ||||||
2137 | |||||||
2138 | Value *Sum = | ||||||
2139 | SimplifyAddInst(GO1, SO1, false, false, SQ.getWithInstruction(&GEP)); | ||||||
2140 | // Only do the combine when we are sure the cost after the | ||||||
2141 | // merge is never more than that before the merge. | ||||||
2142 | if (Sum == nullptr) | ||||||
2143 | return nullptr; | ||||||
2144 | |||||||
2145 | // Update the GEP in place if possible. | ||||||
2146 | if (Src->getNumOperands() == 2) { | ||||||
2147 | GEP.setIsInBounds(isMergedGEPInBounds(*Src, *cast<GEPOperator>(&GEP))); | ||||||
2148 | replaceOperand(GEP, 0, Src->getOperand(0)); | ||||||
2149 | replaceOperand(GEP, 1, Sum); | ||||||
2150 | return &GEP; | ||||||
2151 | } | ||||||
2152 | Indices.append(Src->op_begin()+1, Src->op_end()-1); | ||||||
2153 | Indices.push_back(Sum); | ||||||
2154 | Indices.append(GEP.op_begin()+2, GEP.op_end()); | ||||||
2155 | } else if (isa<Constant>(*GEP.idx_begin()) && | ||||||
2156 | cast<Constant>(*GEP.idx_begin())->isNullValue() && | ||||||
2157 | Src->getNumOperands() != 1) { | ||||||
2158 | // Otherwise we can do the fold if the first index of the GEP is a zero | ||||||
2159 | Indices.append(Src->op_begin()+1, Src->op_end()); | ||||||
2160 | Indices.append(GEP.idx_begin()+1, GEP.idx_end()); | ||||||
2161 | } | ||||||
2162 | |||||||
2163 | if (!Indices.empty()) | ||||||
2164 | return isMergedGEPInBounds(*Src, *cast<GEPOperator>(&GEP)) | ||||||
2165 | ? GetElementPtrInst::CreateInBounds( | ||||||
2166 | Src->getSourceElementType(), Src->getOperand(0), Indices, | ||||||
2167 | GEP.getName()) | ||||||
2168 | : GetElementPtrInst::Create(Src->getSourceElementType(), | ||||||
2169 | Src->getOperand(0), Indices, | ||||||
2170 | GEP.getName()); | ||||||
2171 | } | ||||||
2172 | |||||||
2173 | // Skip if GEP source element type is scalable. The type alloc size is unknown | ||||||
2174 | // at compile-time. | ||||||
2175 | if (GEP.getNumIndices() == 1 && !IsGEPSrcEleScalable) { | ||||||
2176 | unsigned AS = GEP.getPointerAddressSpace(); | ||||||
2177 | if (GEP.getOperand(1)->getType()->getScalarSizeInBits() == | ||||||
2178 | DL.getIndexSizeInBits(AS)) { | ||||||
2179 | uint64_t TyAllocSize = DL.getTypeAllocSize(GEPEltType).getFixedSize(); | ||||||
2180 | |||||||
2181 | bool Matched = false; | ||||||
2182 | uint64_t C; | ||||||
2183 | Value *V = nullptr; | ||||||
2184 | if (TyAllocSize == 1) { | ||||||
2185 | V = GEP.getOperand(1); | ||||||
2186 | Matched = true; | ||||||
2187 | } else if (match(GEP.getOperand(1), | ||||||
2188 | m_AShr(m_Value(V), m_ConstantInt(C)))) { | ||||||
2189 | if (TyAllocSize == 1ULL << C) | ||||||
2190 | Matched = true; | ||||||
2191 | } else if (match(GEP.getOperand(1), | ||||||
2192 | m_SDiv(m_Value(V), m_ConstantInt(C)))) { | ||||||
2193 | if (TyAllocSize == C) | ||||||
2194 | Matched = true; | ||||||
2195 | } | ||||||
2196 | |||||||
2197 | // Canonicalize (gep i8* X, (ptrtoint Y)-(ptrtoint X)) to (bitcast Y), but | ||||||
2198 | // only if both point to the same underlying object (otherwise provenance | ||||||
2199 | // is not necessarily retained). | ||||||
2200 | Value *Y; | ||||||
2201 | Value *X = GEP.getOperand(0); | ||||||
2202 | if (Matched && | ||||||
2203 | match(V, m_Sub(m_PtrToInt(m_Value(Y)), m_PtrToInt(m_Specific(X)))) && | ||||||
2204 | getUnderlyingObject(X) == getUnderlyingObject(Y)) | ||||||
2205 | return CastInst::CreatePointerBitCastOrAddrSpaceCast(Y, GEPType); | ||||||
2206 | } | ||||||
2207 | } | ||||||
2208 | |||||||
2209 | // We do not handle pointer-vector geps here. | ||||||
2210 | if (GEPType->isVectorTy()) | ||||||
2211 | return nullptr; | ||||||
2212 | |||||||
2213 | // Handle gep(bitcast x) and gep(gep x, 0, 0, 0). | ||||||
2214 | Value *StrippedPtr = PtrOp->stripPointerCasts(); | ||||||
2215 | PointerType *StrippedPtrTy = cast<PointerType>(StrippedPtr->getType()); | ||||||
2216 | |||||||
2217 | if (StrippedPtr != PtrOp) { | ||||||
2218 | bool HasZeroPointerIndex = false; | ||||||
2219 | Type *StrippedPtrEltTy = StrippedPtrTy->getElementType(); | ||||||
2220 | |||||||
2221 | if (auto *C = dyn_cast<ConstantInt>(GEP.getOperand(1))) | ||||||
2222 | HasZeroPointerIndex = C->isZero(); | ||||||
2223 | |||||||
2224 | // Transform: GEP (bitcast [10 x i8]* X to [0 x i8]*), i32 0, ... | ||||||
2225 | // into : GEP [10 x i8]* X, i32 0, ... | ||||||
2226 | // | ||||||
2227 | // Likewise, transform: GEP (bitcast i8* X to [0 x i8]*), i32 0, ... | ||||||
2228 | // into : GEP i8* X, ... | ||||||
2229 | // | ||||||
2230 | // This occurs when the program declares an array extern like "int X[];" | ||||||
2231 | if (HasZeroPointerIndex) { | ||||||
2232 | if (auto *CATy = dyn_cast<ArrayType>(GEPEltType)) { | ||||||
2233 | // GEP (bitcast i8* X to [0 x i8]*), i32 0, ... ? | ||||||
2234 | if (CATy->getElementType() == StrippedPtrEltTy) { | ||||||
2235 | // -> GEP i8* X, ... | ||||||
2236 | SmallVector<Value *, 8> Idx(drop_begin(GEP.indices())); | ||||||
2237 | GetElementPtrInst *Res = GetElementPtrInst::Create( | ||||||
2238 | StrippedPtrEltTy, StrippedPtr, Idx, GEP.getName()); | ||||||
2239 | Res->setIsInBounds(GEP.isInBounds()); | ||||||
2240 | if (StrippedPtrTy->getAddressSpace() == GEP.getAddressSpace()) | ||||||
2241 | return Res; | ||||||
2242 | // Insert Res, and create an addrspacecast. | ||||||
2243 | // e.g., | ||||||
2244 | // GEP (addrspacecast i8 addrspace(1)* X to [0 x i8]*), i32 0, ... | ||||||
2245 | // -> | ||||||
2246 | // %0 = GEP i8 addrspace(1)* X, ... | ||||||
2247 | // addrspacecast i8 addrspace(1)* %0 to i8* | ||||||
2248 | return new AddrSpaceCastInst(Builder.Insert(Res), GEPType); | ||||||
2249 | } | ||||||
2250 | |||||||
2251 | if (auto *XATy = dyn_cast<ArrayType>(StrippedPtrEltTy)) { | ||||||
2252 | // GEP (bitcast [10 x i8]* X to [0 x i8]*), i32 0, ... ? | ||||||
2253 | if (CATy->getElementType() == XATy->getElementType()) { | ||||||
2254 | // -> GEP [10 x i8]* X, i32 0, ... | ||||||
2255 | // At this point, we know that the cast source type is a pointer | ||||||
2256 | // to an array of the same type as the destination pointer | ||||||
2257 | // array. Because the array type is never stepped over (there | ||||||
2258 | // is a leading zero) we can fold the cast into this GEP. | ||||||
2259 | if (StrippedPtrTy->getAddressSpace() == GEP.getAddressSpace()) { | ||||||
2260 | GEP.setSourceElementType(XATy); | ||||||
2261 | return replaceOperand(GEP, 0, StrippedPtr); | ||||||
2262 | } | ||||||
2263 | // Cannot replace the base pointer directly because StrippedPtr's | ||||||
2264 | // address space is different. Instead, create a new GEP followed by | ||||||
2265 | // an addrspacecast. | ||||||
2266 | // e.g., | ||||||
2267 | // GEP (addrspacecast [10 x i8] addrspace(1)* X to [0 x i8]*), | ||||||
2268 | // i32 0, ... | ||||||
2269 | // -> | ||||||
2270 | // %0 = GEP [10 x i8] addrspace(1)* X, ... | ||||||
2271 | // addrspacecast i8 addrspace(1)* %0 to i8* | ||||||
2272 | SmallVector<Value *, 8> Idx(GEP.indices()); | ||||||
2273 | Value *NewGEP = | ||||||
2274 | GEP.isInBounds() | ||||||
2275 | ? Builder.CreateInBoundsGEP(StrippedPtrEltTy, StrippedPtr, | ||||||
2276 | Idx, GEP.getName()) | ||||||
2277 | : Builder.CreateGEP(StrippedPtrEltTy, StrippedPtr, Idx, | ||||||
2278 | GEP.getName()); | ||||||
2279 | return new AddrSpaceCastInst(NewGEP, GEPType); | ||||||
2280 | } | ||||||
2281 | } | ||||||
2282 | } | ||||||
2283 | } else if (GEP.getNumOperands() == 2 && !IsGEPSrcEleScalable) { | ||||||
2284 | // Skip if GEP source element type is scalable. The type alloc size is | ||||||
2285 | // unknown at compile-time. | ||||||
2286 | // Transform things like: %t = getelementptr i32* | ||||||
2287 | // bitcast ([2 x i32]* %str to i32*), i32 %V into: %t1 = getelementptr [2 | ||||||
2288 | // x i32]* %str, i32 0, i32 %V; bitcast | ||||||
2289 | if (StrippedPtrEltTy->isArrayTy() && | ||||||
2290 | DL.getTypeAllocSize(StrippedPtrEltTy->getArrayElementType()) == | ||||||
2291 | DL.getTypeAllocSize(GEPEltType)) { | ||||||
2292 | Type *IdxType = DL.getIndexType(GEPType); | ||||||
2293 | Value *Idx[2] = { Constant::getNullValue(IdxType), GEP.getOperand(1) }; | ||||||
2294 | Value *NewGEP = | ||||||
2295 | GEP.isInBounds() | ||||||
2296 | ? Builder.CreateInBoundsGEP(StrippedPtrEltTy, StrippedPtr, Idx, | ||||||
2297 | GEP.getName()) | ||||||
2298 | : Builder.CreateGEP(StrippedPtrEltTy, StrippedPtr, Idx, | ||||||
2299 | GEP.getName()); | ||||||
2300 | |||||||
2301 | // V and GEP are both pointer types --> BitCast | ||||||
2302 | return CastInst::CreatePointerBitCastOrAddrSpaceCast(NewGEP, GEPType); | ||||||
2303 | } | ||||||
2304 | |||||||
2305 | // Transform things like: | ||||||
2306 | // %V = mul i64 %N, 4 | ||||||
2307 | // %t = getelementptr i8* bitcast (i32* %arr to i8*), i32 %V | ||||||
2308 | // into: %t1 = getelementptr i32* %arr, i32 %N; bitcast | ||||||
2309 | if (GEPEltType->isSized() && StrippedPtrEltTy->isSized()) { | ||||||
2310 | // Check that changing the type amounts to dividing the index by a scale | ||||||
2311 | // factor. | ||||||
2312 | uint64_t ResSize = DL.getTypeAllocSize(GEPEltType).getFixedSize(); | ||||||
2313 | uint64_t SrcSize = DL.getTypeAllocSize(StrippedPtrEltTy).getFixedSize(); | ||||||
2314 | if (ResSize && SrcSize % ResSize == 0) { | ||||||
2315 | Value *Idx = GEP.getOperand(1); | ||||||
2316 | unsigned BitWidth = Idx->getType()->getPrimitiveSizeInBits(); | ||||||
2317 | uint64_t Scale = SrcSize / ResSize; | ||||||
2318 | |||||||
2319 | // Earlier transforms ensure that the index has the right type | ||||||
2320 | // according to Data Layout, which considerably simplifies the | ||||||
2321 | // logic by eliminating implicit casts. | ||||||
2322 | assert(Idx->getType() == DL.getIndexType(GEPType) &&(static_cast <bool> (Idx->getType() == DL.getIndexType (GEPType) && "Index type does not match the Data Layout preferences" ) ? void (0) : __assert_fail ("Idx->getType() == DL.getIndexType(GEPType) && \"Index type does not match the Data Layout preferences\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 2323, __extension__ __PRETTY_FUNCTION__)) | ||||||
2323 | "Index type does not match the Data Layout preferences")(static_cast <bool> (Idx->getType() == DL.getIndexType (GEPType) && "Index type does not match the Data Layout preferences" ) ? void (0) : __assert_fail ("Idx->getType() == DL.getIndexType(GEPType) && \"Index type does not match the Data Layout preferences\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 2323, __extension__ __PRETTY_FUNCTION__)); | ||||||
2324 | |||||||
2325 | bool NSW; | ||||||
2326 | if (Value *NewIdx = Descale(Idx, APInt(BitWidth, Scale), NSW)) { | ||||||
2327 | // Successfully decomposed Idx as NewIdx * Scale, form a new GEP. | ||||||
2328 | // If the multiplication NewIdx * Scale may overflow then the new | ||||||
2329 | // GEP may not be "inbounds". | ||||||
2330 | Value *NewGEP = | ||||||
2331 | GEP.isInBounds() && NSW | ||||||
2332 | ? Builder.CreateInBoundsGEP(StrippedPtrEltTy, StrippedPtr, | ||||||
2333 | NewIdx, GEP.getName()) | ||||||
2334 | : Builder.CreateGEP(StrippedPtrEltTy, StrippedPtr, NewIdx, | ||||||
2335 | GEP.getName()); | ||||||
2336 | |||||||
2337 | // The NewGEP must be pointer typed, so must the old one -> BitCast | ||||||
2338 | return CastInst::CreatePointerBitCastOrAddrSpaceCast(NewGEP, | ||||||
2339 | GEPType); | ||||||
2340 | } | ||||||
2341 | } | ||||||
2342 | } | ||||||
2343 | |||||||
2344 | // Similarly, transform things like: | ||||||
2345 | // getelementptr i8* bitcast ([100 x double]* X to i8*), i32 %tmp | ||||||
2346 | // (where tmp = 8*tmp2) into: | ||||||
2347 | // getelementptr [100 x double]* %arr, i32 0, i32 %tmp2; bitcast | ||||||
2348 | if (GEPEltType->isSized() && StrippedPtrEltTy->isSized() && | ||||||
2349 | StrippedPtrEltTy->isArrayTy()) { | ||||||
2350 | // Check that changing to the array element type amounts to dividing the | ||||||
2351 | // index by a scale factor. | ||||||
2352 | uint64_t ResSize = DL.getTypeAllocSize(GEPEltType).getFixedSize(); | ||||||
2353 | uint64_t ArrayEltSize = | ||||||
2354 | DL.getTypeAllocSize(StrippedPtrEltTy->getArrayElementType()) | ||||||
2355 | .getFixedSize(); | ||||||
2356 | if (ResSize && ArrayEltSize % ResSize == 0) { | ||||||
2357 | Value *Idx = GEP.getOperand(1); | ||||||
2358 | unsigned BitWidth = Idx->getType()->getPrimitiveSizeInBits(); | ||||||
2359 | uint64_t Scale = ArrayEltSize / ResSize; | ||||||
2360 | |||||||
2361 | // Earlier transforms ensure that the index has the right type | ||||||
2362 | // according to the Data Layout, which considerably simplifies | ||||||
2363 | // the logic by eliminating implicit casts. | ||||||
2364 | assert(Idx->getType() == DL.getIndexType(GEPType) &&(static_cast <bool> (Idx->getType() == DL.getIndexType (GEPType) && "Index type does not match the Data Layout preferences" ) ? void (0) : __assert_fail ("Idx->getType() == DL.getIndexType(GEPType) && \"Index type does not match the Data Layout preferences\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 2365, __extension__ __PRETTY_FUNCTION__)) | ||||||
2365 | "Index type does not match the Data Layout preferences")(static_cast <bool> (Idx->getType() == DL.getIndexType (GEPType) && "Index type does not match the Data Layout preferences" ) ? void (0) : __assert_fail ("Idx->getType() == DL.getIndexType(GEPType) && \"Index type does not match the Data Layout preferences\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 2365, __extension__ __PRETTY_FUNCTION__)); | ||||||
2366 | |||||||
2367 | bool NSW; | ||||||
2368 | if (Value *NewIdx = Descale(Idx, APInt(BitWidth, Scale), NSW)) { | ||||||
2369 | // Successfully decomposed Idx as NewIdx * Scale, form a new GEP. | ||||||
2370 | // If the multiplication NewIdx * Scale may overflow then the new | ||||||
2371 | // GEP may not be "inbounds". | ||||||
2372 | Type *IndTy = DL.getIndexType(GEPType); | ||||||
2373 | Value *Off[2] = {Constant::getNullValue(IndTy), NewIdx}; | ||||||
2374 | |||||||
2375 | Value *NewGEP = | ||||||
2376 | GEP.isInBounds() && NSW | ||||||
2377 | ? Builder.CreateInBoundsGEP(StrippedPtrEltTy, StrippedPtr, | ||||||
2378 | Off, GEP.getName()) | ||||||
2379 | : Builder.CreateGEP(StrippedPtrEltTy, StrippedPtr, Off, | ||||||
2380 | GEP.getName()); | ||||||
2381 | // The NewGEP must be pointer typed, so must the old one -> BitCast | ||||||
2382 | return CastInst::CreatePointerBitCastOrAddrSpaceCast(NewGEP, | ||||||
2383 | GEPType); | ||||||
2384 | } | ||||||
2385 | } | ||||||
2386 | } | ||||||
2387 | } | ||||||
2388 | } | ||||||
2389 | |||||||
2390 | // addrspacecast between types is canonicalized as a bitcast, then an | ||||||
2391 | // addrspacecast. To take advantage of the below bitcast + struct GEP, look | ||||||
2392 | // through the addrspacecast. | ||||||
2393 | Value *ASCStrippedPtrOp = PtrOp; | ||||||
2394 | if (auto *ASC = dyn_cast<AddrSpaceCastInst>(PtrOp)) { | ||||||
2395 | // X = bitcast A addrspace(1)* to B addrspace(1)* | ||||||
2396 | // Y = addrspacecast A addrspace(1)* to B addrspace(2)* | ||||||
2397 | // Z = gep Y, <...constant indices...> | ||||||
2398 | // Into an addrspacecasted GEP of the struct. | ||||||
2399 | if (auto *BC = dyn_cast<BitCastInst>(ASC->getOperand(0))) | ||||||
2400 | ASCStrippedPtrOp = BC; | ||||||
2401 | } | ||||||
2402 | |||||||
2403 | if (auto *BCI = dyn_cast<BitCastInst>(ASCStrippedPtrOp)) { | ||||||
2404 | Value *SrcOp = BCI->getOperand(0); | ||||||
2405 | PointerType *SrcType = cast<PointerType>(BCI->getSrcTy()); | ||||||
2406 | Type *SrcEltType = SrcType->getElementType(); | ||||||
2407 | |||||||
2408 | // GEP directly using the source operand if this GEP is accessing an element | ||||||
2409 | // of a bitcasted pointer to vector or array of the same dimensions: | ||||||
2410 | // gep (bitcast <c x ty>* X to [c x ty]*), Y, Z --> gep X, Y, Z | ||||||
2411 | // gep (bitcast [c x ty]* X to <c x ty>*), Y, Z --> gep X, Y, Z | ||||||
2412 | auto areMatchingArrayAndVecTypes = [](Type *ArrTy, Type *VecTy, | ||||||
2413 | const DataLayout &DL) { | ||||||
2414 | auto *VecVTy = cast<FixedVectorType>(VecTy); | ||||||
2415 | return ArrTy->getArrayElementType() == VecVTy->getElementType() && | ||||||
2416 | ArrTy->getArrayNumElements() == VecVTy->getNumElements() && | ||||||
2417 | DL.getTypeAllocSize(ArrTy) == DL.getTypeAllocSize(VecTy); | ||||||
2418 | }; | ||||||
2419 | if (GEP.getNumOperands() == 3 && | ||||||
2420 | ((GEPEltType->isArrayTy() && isa<FixedVectorType>(SrcEltType) && | ||||||
2421 | areMatchingArrayAndVecTypes(GEPEltType, SrcEltType, DL)) || | ||||||
2422 | (isa<FixedVectorType>(GEPEltType) && SrcEltType->isArrayTy() && | ||||||
2423 | areMatchingArrayAndVecTypes(SrcEltType, GEPEltType, DL)))) { | ||||||
2424 | |||||||
2425 | // Create a new GEP here, as using `setOperand()` followed by | ||||||
2426 | // `setSourceElementType()` won't actually update the type of the | ||||||
2427 | // existing GEP Value. Causing issues if this Value is accessed when | ||||||
2428 | // constructing an AddrSpaceCastInst | ||||||
2429 | Value *NGEP = | ||||||
2430 | GEP.isInBounds() | ||||||
2431 | ? Builder.CreateInBoundsGEP(SrcEltType, SrcOp, {Ops[1], Ops[2]}) | ||||||
2432 | : Builder.CreateGEP(SrcEltType, SrcOp, {Ops[1], Ops[2]}); | ||||||
2433 | NGEP->takeName(&GEP); | ||||||
2434 | |||||||
2435 | // Preserve GEP address space to satisfy users | ||||||
2436 | if (NGEP->getType()->getPointerAddressSpace() != GEP.getAddressSpace()) | ||||||
2437 | return new AddrSpaceCastInst(NGEP, GEPType); | ||||||
2438 | |||||||
2439 | return replaceInstUsesWith(GEP, NGEP); | ||||||
2440 | } | ||||||
2441 | |||||||
2442 | // See if we can simplify: | ||||||
2443 | // X = bitcast A* to B* | ||||||
2444 | // Y = gep X, <...constant indices...> | ||||||
2445 | // into a gep of the original struct. This is important for SROA and alias | ||||||
2446 | // analysis of unions. If "A" is also a bitcast, wait for A/X to be merged. | ||||||
2447 | unsigned OffsetBits = DL.getIndexTypeSizeInBits(GEPType); | ||||||
2448 | APInt Offset(OffsetBits, 0); | ||||||
2449 | |||||||
2450 | // If the bitcast argument is an allocation, The bitcast is for convertion | ||||||
2451 | // to actual type of allocation. Removing such bitcasts, results in having | ||||||
2452 | // GEPs with i8* base and pure byte offsets. That means GEP is not aware of | ||||||
2453 | // struct or array hierarchy. | ||||||
2454 | // By avoiding such GEPs, phi translation and MemoryDependencyAnalysis have | ||||||
2455 | // a better chance to succeed. | ||||||
2456 | if (!isa<BitCastInst>(SrcOp) && GEP.accumulateConstantOffset(DL, Offset) && | ||||||
2457 | !isAllocationFn(SrcOp, &TLI)) { | ||||||
2458 | // If this GEP instruction doesn't move the pointer, just replace the GEP | ||||||
2459 | // with a bitcast of the real input to the dest type. | ||||||
2460 | if (!Offset) { | ||||||
2461 | // If the bitcast is of an allocation, and the allocation will be | ||||||
2462 | // converted to match the type of the cast, don't touch this. | ||||||
2463 | if (isa<AllocaInst>(SrcOp)) { | ||||||
2464 | // See if the bitcast simplifies, if so, don't nuke this GEP yet. | ||||||
2465 | if (Instruction *I = visitBitCast(*BCI)) { | ||||||
2466 | if (I != BCI) { | ||||||
2467 | I->takeName(BCI); | ||||||
2468 | BCI->getParent()->getInstList().insert(BCI->getIterator(), I); | ||||||
2469 | replaceInstUsesWith(*BCI, I); | ||||||
2470 | } | ||||||
2471 | return &GEP; | ||||||
2472 | } | ||||||
2473 | } | ||||||
2474 | |||||||
2475 | if (SrcType->getPointerAddressSpace() != GEP.getAddressSpace()) | ||||||
2476 | return new AddrSpaceCastInst(SrcOp, GEPType); | ||||||
2477 | return new BitCastInst(SrcOp, GEPType); | ||||||
2478 | } | ||||||
2479 | |||||||
2480 | // Otherwise, if the offset is non-zero, we need to find out if there is a | ||||||
2481 | // field at Offset in 'A's type. If so, we can pull the cast through the | ||||||
2482 | // GEP. | ||||||
2483 | SmallVector<Value*, 8> NewIndices; | ||||||
2484 | if (FindElementAtOffset(SrcType, Offset.getSExtValue(), NewIndices)) { | ||||||
2485 | Value *NGEP = | ||||||
2486 | GEP.isInBounds() | ||||||
2487 | ? Builder.CreateInBoundsGEP(SrcEltType, SrcOp, NewIndices) | ||||||
2488 | : Builder.CreateGEP(SrcEltType, SrcOp, NewIndices); | ||||||
2489 | |||||||
2490 | if (NGEP->getType() == GEPType) | ||||||
2491 | return replaceInstUsesWith(GEP, NGEP); | ||||||
2492 | NGEP->takeName(&GEP); | ||||||
2493 | |||||||
2494 | if (NGEP->getType()->getPointerAddressSpace() != GEP.getAddressSpace()) | ||||||
2495 | return new AddrSpaceCastInst(NGEP, GEPType); | ||||||
2496 | return new BitCastInst(NGEP, GEPType); | ||||||
2497 | } | ||||||
2498 | } | ||||||
2499 | } | ||||||
2500 | |||||||
2501 | if (!GEP.isInBounds()) { | ||||||
2502 | unsigned IdxWidth = | ||||||
2503 | DL.getIndexSizeInBits(PtrOp->getType()->getPointerAddressSpace()); | ||||||
2504 | APInt BasePtrOffset(IdxWidth, 0); | ||||||
2505 | Value *UnderlyingPtrOp = | ||||||
2506 | PtrOp->stripAndAccumulateInBoundsConstantOffsets(DL, | ||||||
2507 | BasePtrOffset); | ||||||
2508 | if (auto *AI = dyn_cast<AllocaInst>(UnderlyingPtrOp)) { | ||||||
2509 | if (GEP.accumulateConstantOffset(DL, BasePtrOffset) && | ||||||
2510 | BasePtrOffset.isNonNegative()) { | ||||||
2511 | APInt AllocSize( | ||||||
2512 | IdxWidth, | ||||||
2513 | DL.getTypeAllocSize(AI->getAllocatedType()).getKnownMinSize()); | ||||||
2514 | if (BasePtrOffset.ule(AllocSize)) { | ||||||
2515 | return GetElementPtrInst::CreateInBounds( | ||||||
2516 | GEP.getSourceElementType(), PtrOp, makeArrayRef(Ops).slice(1), | ||||||
2517 | GEP.getName()); | ||||||
2518 | } | ||||||
2519 | } | ||||||
2520 | } | ||||||
2521 | } | ||||||
2522 | |||||||
2523 | if (Instruction *R = foldSelectGEP(GEP, Builder)) | ||||||
2524 | return R; | ||||||
2525 | |||||||
2526 | return nullptr; | ||||||
2527 | } | ||||||
2528 | |||||||
2529 | static bool isNeverEqualToUnescapedAlloc(Value *V, const TargetLibraryInfo *TLI, | ||||||
2530 | Instruction *AI) { | ||||||
2531 | if (isa<ConstantPointerNull>(V)) | ||||||
2532 | return true; | ||||||
2533 | if (auto *LI = dyn_cast<LoadInst>(V)) | ||||||
2534 | return isa<GlobalVariable>(LI->getPointerOperand()); | ||||||
2535 | // Two distinct allocations will never be equal. | ||||||
2536 | // We rely on LookThroughBitCast in isAllocLikeFn being false, since looking | ||||||
2537 | // through bitcasts of V can cause | ||||||
2538 | // the result statement below to be true, even when AI and V (ex: | ||||||
2539 | // i8* ->i32* ->i8* of AI) are the same allocations. | ||||||
2540 | return isAllocLikeFn(V, TLI) && V != AI; | ||||||
2541 | } | ||||||
2542 | |||||||
2543 | static bool isAllocSiteRemovable(Instruction *AI, | ||||||
2544 | SmallVectorImpl<WeakTrackingVH> &Users, | ||||||
2545 | const TargetLibraryInfo *TLI) { | ||||||
2546 | SmallVector<Instruction*, 4> Worklist; | ||||||
2547 | Worklist.push_back(AI); | ||||||
2548 | |||||||
2549 | do { | ||||||
2550 | Instruction *PI = Worklist.pop_back_val(); | ||||||
2551 | for (User *U : PI->users()) { | ||||||
2552 | Instruction *I = cast<Instruction>(U); | ||||||
2553 | switch (I->getOpcode()) { | ||||||
2554 | default: | ||||||
2555 | // Give up the moment we see something we can't handle. | ||||||
2556 | return false; | ||||||
2557 | |||||||
2558 | case Instruction::AddrSpaceCast: | ||||||
2559 | case Instruction::BitCast: | ||||||
2560 | case Instruction::GetElementPtr: | ||||||
2561 | Users.emplace_back(I); | ||||||
2562 | Worklist.push_back(I); | ||||||
2563 | continue; | ||||||
2564 | |||||||
2565 | case Instruction::ICmp: { | ||||||
2566 | ICmpInst *ICI = cast<ICmpInst>(I); | ||||||
2567 | // We can fold eq/ne comparisons with null to false/true, respectively. | ||||||
2568 | // We also fold comparisons in some conditions provided the alloc has | ||||||
2569 | // not escaped (see isNeverEqualToUnescapedAlloc). | ||||||
2570 | if (!ICI->isEquality()) | ||||||
2571 | return false; | ||||||
2572 | unsigned OtherIndex = (ICI->getOperand(0) == PI) ? 1 : 0; | ||||||
2573 | if (!isNeverEqualToUnescapedAlloc(ICI->getOperand(OtherIndex), TLI, AI)) | ||||||
2574 | return false; | ||||||
2575 | Users.emplace_back(I); | ||||||
2576 | continue; | ||||||
2577 | } | ||||||
2578 | |||||||
2579 | case Instruction::Call: | ||||||
2580 | // Ignore no-op and store intrinsics. | ||||||
2581 | if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) { | ||||||
2582 | switch (II->getIntrinsicID()) { | ||||||
2583 | default: | ||||||
2584 | return false; | ||||||
2585 | |||||||
2586 | case Intrinsic::memmove: | ||||||
2587 | case Intrinsic::memcpy: | ||||||
2588 | case Intrinsic::memset: { | ||||||
2589 | MemIntrinsic *MI = cast<MemIntrinsic>(II); | ||||||
2590 | if (MI->isVolatile() || MI->getRawDest() != PI) | ||||||
2591 | return false; | ||||||
2592 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||
2593 | } | ||||||
2594 | case Intrinsic::assume: | ||||||
2595 | case Intrinsic::invariant_start: | ||||||
2596 | case Intrinsic::invariant_end: | ||||||
2597 | case Intrinsic::lifetime_start: | ||||||
2598 | case Intrinsic::lifetime_end: | ||||||
2599 | case Intrinsic::objectsize: | ||||||
2600 | Users.emplace_back(I); | ||||||
2601 | continue; | ||||||
2602 | case Intrinsic::launder_invariant_group: | ||||||
2603 | case Intrinsic::strip_invariant_group: | ||||||
2604 | Users.emplace_back(I); | ||||||
2605 | Worklist.push_back(I); | ||||||
2606 | continue; | ||||||
2607 | } | ||||||
2608 | } | ||||||
2609 | |||||||
2610 | if (isFreeCall(I, TLI)) { | ||||||
2611 | Users.emplace_back(I); | ||||||
2612 | continue; | ||||||
2613 | } | ||||||
2614 | |||||||
2615 | if (isReallocLikeFn(I, TLI, true)) { | ||||||
2616 | Users.emplace_back(I); | ||||||
2617 | Worklist.push_back(I); | ||||||
2618 | continue; | ||||||
2619 | } | ||||||
2620 | |||||||
2621 | return false; | ||||||
2622 | |||||||
2623 | case Instruction::Store: { | ||||||
2624 | StoreInst *SI = cast<StoreInst>(I); | ||||||
2625 | if (SI->isVolatile() || SI->getPointerOperand() != PI) | ||||||
2626 | return false; | ||||||
2627 | Users.emplace_back(I); | ||||||
2628 | continue; | ||||||
2629 | } | ||||||
2630 | } | ||||||
2631 | llvm_unreachable("missing a return?")::llvm::llvm_unreachable_internal("missing a return?", "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 2631); | ||||||
2632 | } | ||||||
2633 | } while (!Worklist.empty()); | ||||||
2634 | return true; | ||||||
2635 | } | ||||||
2636 | |||||||
2637 | Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) { | ||||||
2638 | // If we have a malloc call which is only used in any amount of comparisons to | ||||||
2639 | // null and free calls, delete the calls and replace the comparisons with true | ||||||
2640 | // or false as appropriate. | ||||||
2641 | |||||||
2642 | // This is based on the principle that we can substitute our own allocation | ||||||
2643 | // function (which will never return null) rather than knowledge of the | ||||||
2644 | // specific function being called. In some sense this can change the permitted | ||||||
2645 | // outputs of a program (when we convert a malloc to an alloca, the fact that | ||||||
2646 | // the allocation is now on the stack is potentially visible, for example), | ||||||
2647 | // but we believe in a permissible manner. | ||||||
2648 | SmallVector<WeakTrackingVH, 64> Users; | ||||||
2649 | |||||||
2650 | // If we are removing an alloca with a dbg.declare, insert dbg.value calls | ||||||
2651 | // before each store. | ||||||
2652 | SmallVector<DbgVariableIntrinsic *, 8> DVIs; | ||||||
2653 | std::unique_ptr<DIBuilder> DIB; | ||||||
2654 | if (isa<AllocaInst>(MI)) { | ||||||
2655 | findDbgUsers(DVIs, &MI); | ||||||
2656 | DIB.reset(new DIBuilder(*MI.getModule(), /*AllowUnresolved=*/false)); | ||||||
2657 | } | ||||||
2658 | |||||||
2659 | if (isAllocSiteRemovable(&MI, Users, &TLI)) { | ||||||
2660 | for (unsigned i = 0, e = Users.size(); i != e; ++i) { | ||||||
2661 | // Lowering all @llvm.objectsize calls first because they may | ||||||
2662 | // use a bitcast/GEP of the alloca we are removing. | ||||||
2663 | if (!Users[i]) | ||||||
2664 | continue; | ||||||
2665 | |||||||
2666 | Instruction *I = cast<Instruction>(&*Users[i]); | ||||||
2667 | |||||||
2668 | if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) { | ||||||
2669 | if (II->getIntrinsicID() == Intrinsic::objectsize) { | ||||||
2670 | Value *Result = | ||||||
2671 | lowerObjectSizeCall(II, DL, &TLI, /*MustSucceed=*/true); | ||||||
2672 | replaceInstUsesWith(*I, Result); | ||||||
2673 | eraseInstFromFunction(*I); | ||||||
2674 | Users[i] = nullptr; // Skip examining in the next loop. | ||||||
2675 | } | ||||||
2676 | } | ||||||
2677 | } | ||||||
2678 | for (unsigned i = 0, e = Users.size(); i != e; ++i) { | ||||||
2679 | if (!Users[i]) | ||||||
2680 | continue; | ||||||
2681 | |||||||
2682 | Instruction *I = cast<Instruction>(&*Users[i]); | ||||||
2683 | |||||||
2684 | if (ICmpInst *C = dyn_cast<ICmpInst>(I)) { | ||||||
2685 | replaceInstUsesWith(*C, | ||||||
2686 | ConstantInt::get(Type::getInt1Ty(C->getContext()), | ||||||
2687 | C->isFalseWhenEqual())); | ||||||
2688 | } else if (auto *SI = dyn_cast<StoreInst>(I)) { | ||||||
2689 | for (auto *DVI : DVIs) | ||||||
2690 | if (DVI->isAddressOfVariable()) | ||||||
2691 | ConvertDebugDeclareToDebugValue(DVI, SI, *DIB); | ||||||
2692 | } else { | ||||||
2693 | // Casts, GEP, or anything else: we're about to delete this instruction, | ||||||
2694 | // so it can not have any valid uses. | ||||||
2695 | replaceInstUsesWith(*I, PoisonValue::get(I->getType())); | ||||||
2696 | } | ||||||
2697 | eraseInstFromFunction(*I); | ||||||
2698 | } | ||||||
2699 | |||||||
2700 | if (InvokeInst *II = dyn_cast<InvokeInst>(&MI)) { | ||||||
2701 | // Replace invoke with a NOP intrinsic to maintain the original CFG | ||||||
2702 | Module *M = II->getModule(); | ||||||
2703 | Function *F = Intrinsic::getDeclaration(M, Intrinsic::donothing); | ||||||
2704 | InvokeInst::Create(F, II->getNormalDest(), II->getUnwindDest(), | ||||||
2705 | None, "", II->getParent()); | ||||||
2706 | } | ||||||
2707 | |||||||
2708 | // Remove debug intrinsics which describe the value contained within the | ||||||
2709 | // alloca. In addition to removing dbg.{declare,addr} which simply point to | ||||||
2710 | // the alloca, remove dbg.value(<alloca>, ..., DW_OP_deref)'s as well, e.g.: | ||||||
2711 | // | ||||||
2712 | // ``` | ||||||
2713 | // define void @foo(i32 %0) { | ||||||
2714 | // %a = alloca i32 ; Deleted. | ||||||
2715 | // store i32 %0, i32* %a | ||||||
2716 | // dbg.value(i32 %0, "arg0") ; Not deleted. | ||||||
2717 | // dbg.value(i32* %a, "arg0", DW_OP_deref) ; Deleted. | ||||||
2718 | // call void @trivially_inlinable_no_op(i32* %a) | ||||||
2719 | // ret void | ||||||
2720 | // } | ||||||
2721 | // ``` | ||||||
2722 | // | ||||||
2723 | // This may not be required if we stop describing the contents of allocas | ||||||
2724 | // using dbg.value(<alloca>, ..., DW_OP_deref), but we currently do this in | ||||||
2725 | // the LowerDbgDeclare utility. | ||||||
2726 | // | ||||||
2727 | // If there is a dead store to `%a` in @trivially_inlinable_no_op, the | ||||||
2728 | // "arg0" dbg.value may be stale after the call. However, failing to remove | ||||||
2729 | // the DW_OP_deref dbg.value causes large gaps in location coverage. | ||||||
2730 | for (auto *DVI : DVIs) | ||||||
2731 | if (DVI->isAddressOfVariable() || DVI->getExpression()->startsWithDeref()) | ||||||
2732 | DVI->eraseFromParent(); | ||||||
2733 | |||||||
2734 | return eraseInstFromFunction(MI); | ||||||
2735 | } | ||||||
2736 | return nullptr; | ||||||
2737 | } | ||||||
2738 | |||||||
2739 | /// Move the call to free before a NULL test. | ||||||
2740 | /// | ||||||
2741 | /// Check if this free is accessed after its argument has been test | ||||||
2742 | /// against NULL (property 0). | ||||||
2743 | /// If yes, it is legal to move this call in its predecessor block. | ||||||
2744 | /// | ||||||
2745 | /// The move is performed only if the block containing the call to free | ||||||
2746 | /// will be removed, i.e.: | ||||||
2747 | /// 1. it has only one predecessor P, and P has two successors | ||||||
2748 | /// 2. it contains the call, noops, and an unconditional branch | ||||||
2749 | /// 3. its successor is the same as its predecessor's successor | ||||||
2750 | /// | ||||||
2751 | /// The profitability is out-of concern here and this function should | ||||||
2752 | /// be called only if the caller knows this transformation would be | ||||||
2753 | /// profitable (e.g., for code size). | ||||||
2754 | static Instruction *tryToMoveFreeBeforeNullTest(CallInst &FI, | ||||||
2755 | const DataLayout &DL) { | ||||||
2756 | Value *Op = FI.getArgOperand(0); | ||||||
2757 | BasicBlock *FreeInstrBB = FI.getParent(); | ||||||
2758 | BasicBlock *PredBB = FreeInstrBB->getSinglePredecessor(); | ||||||
2759 | |||||||
2760 | // Validate part of constraint #1: Only one predecessor | ||||||
2761 | // FIXME: We can extend the number of predecessor, but in that case, we | ||||||
2762 | // would duplicate the call to free in each predecessor and it may | ||||||
2763 | // not be profitable even for code size. | ||||||
2764 | if (!PredBB) | ||||||
2765 | return nullptr; | ||||||
2766 | |||||||
2767 | // Validate constraint #2: Does this block contains only the call to | ||||||
2768 | // free, noops, and an unconditional branch? | ||||||
2769 | BasicBlock *SuccBB; | ||||||
2770 | Instruction *FreeInstrBBTerminator = FreeInstrBB->getTerminator(); | ||||||
2771 | if (!match(FreeInstrBBTerminator, m_UnconditionalBr(SuccBB))) | ||||||
2772 | return nullptr; | ||||||
2773 | |||||||
2774 | // If there are only 2 instructions in the block, at this point, | ||||||
2775 | // this is the call to free and unconditional. | ||||||
2776 | // If there are more than 2 instructions, check that they are noops | ||||||
2777 | // i.e., they won't hurt the performance of the generated code. | ||||||
2778 | if (FreeInstrBB->size() != 2) { | ||||||
2779 | for (const Instruction &Inst : FreeInstrBB->instructionsWithoutDebug()) { | ||||||
2780 | if (&Inst == &FI || &Inst == FreeInstrBBTerminator) | ||||||
2781 | continue; | ||||||
2782 | auto *Cast = dyn_cast<CastInst>(&Inst); | ||||||
2783 | if (!Cast || !Cast->isNoopCast(DL)) | ||||||
2784 | return nullptr; | ||||||
2785 | } | ||||||
2786 | } | ||||||
2787 | // Validate the rest of constraint #1 by matching on the pred branch. | ||||||
2788 | Instruction *TI = PredBB->getTerminator(); | ||||||
2789 | BasicBlock *TrueBB, *FalseBB; | ||||||
2790 | ICmpInst::Predicate Pred; | ||||||
2791 | if (!match(TI, m_Br(m_ICmp(Pred, | ||||||
2792 | m_CombineOr(m_Specific(Op), | ||||||
2793 | m_Specific(Op->stripPointerCasts())), | ||||||
2794 | m_Zero()), | ||||||
2795 | TrueBB, FalseBB))) | ||||||
2796 | return nullptr; | ||||||
2797 | if (Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE) | ||||||
2798 | return nullptr; | ||||||
2799 | |||||||
2800 | // Validate constraint #3: Ensure the null case just falls through. | ||||||
2801 | if (SuccBB != (Pred == ICmpInst::ICMP_EQ ? TrueBB : FalseBB)) | ||||||
2802 | return nullptr; | ||||||
2803 | assert(FreeInstrBB == (Pred == ICmpInst::ICMP_EQ ? FalseBB : TrueBB) &&(static_cast <bool> (FreeInstrBB == (Pred == ICmpInst:: ICMP_EQ ? FalseBB : TrueBB) && "Broken CFG: missing edge from predecessor to successor" ) ? void (0) : __assert_fail ("FreeInstrBB == (Pred == ICmpInst::ICMP_EQ ? FalseBB : TrueBB) && \"Broken CFG: missing edge from predecessor to successor\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 2804, __extension__ __PRETTY_FUNCTION__)) | ||||||
2804 | "Broken CFG: missing edge from predecessor to successor")(static_cast <bool> (FreeInstrBB == (Pred == ICmpInst:: ICMP_EQ ? FalseBB : TrueBB) && "Broken CFG: missing edge from predecessor to successor" ) ? void (0) : __assert_fail ("FreeInstrBB == (Pred == ICmpInst::ICMP_EQ ? FalseBB : TrueBB) && \"Broken CFG: missing edge from predecessor to successor\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 2804, __extension__ __PRETTY_FUNCTION__)); | ||||||
2805 | |||||||
2806 | // At this point, we know that everything in FreeInstrBB can be moved | ||||||
2807 | // before TI. | ||||||
2808 | for (Instruction &Instr : llvm::make_early_inc_range(*FreeInstrBB)) { | ||||||
2809 | if (&Instr == FreeInstrBBTerminator) | ||||||
2810 | break; | ||||||
2811 | Instr.moveBefore(TI); | ||||||
2812 | } | ||||||
2813 | assert(FreeInstrBB->size() == 1 &&(static_cast <bool> (FreeInstrBB->size() == 1 && "Only the branch instruction should remain") ? void (0) : __assert_fail ("FreeInstrBB->size() == 1 && \"Only the branch instruction should remain\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 2814, __extension__ __PRETTY_FUNCTION__)) | ||||||
2814 | "Only the branch instruction should remain")(static_cast <bool> (FreeInstrBB->size() == 1 && "Only the branch instruction should remain") ? void (0) : __assert_fail ("FreeInstrBB->size() == 1 && \"Only the branch instruction should remain\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 2814, __extension__ __PRETTY_FUNCTION__)); | ||||||
2815 | return &FI; | ||||||
2816 | } | ||||||
2817 | |||||||
2818 | Instruction *InstCombinerImpl::visitFree(CallInst &FI) { | ||||||
2819 | Value *Op = FI.getArgOperand(0); | ||||||
2820 | |||||||
2821 | // free undef -> unreachable. | ||||||
2822 | if (isa<UndefValue>(Op)) { | ||||||
2823 | // Leave a marker since we can't modify the CFG here. | ||||||
2824 | CreateNonTerminatorUnreachable(&FI); | ||||||
2825 | return eraseInstFromFunction(FI); | ||||||
2826 | } | ||||||
2827 | |||||||
2828 | // If we have 'free null' delete the instruction. This can happen in stl code | ||||||
2829 | // when lots of inlining happens. | ||||||
2830 | if (isa<ConstantPointerNull>(Op)) | ||||||
2831 | return eraseInstFromFunction(FI); | ||||||
2832 | |||||||
2833 | // If we had free(realloc(...)) with no intervening uses, then eliminate the | ||||||
2834 | // realloc() entirely. | ||||||
2835 | if (CallInst *CI = dyn_cast<CallInst>(Op)) { | ||||||
2836 | if (CI->hasOneUse() && isReallocLikeFn(CI, &TLI, true)) { | ||||||
2837 | return eraseInstFromFunction( | ||||||
2838 | *replaceInstUsesWith(*CI, CI->getOperand(0))); | ||||||
2839 | } | ||||||
2840 | } | ||||||
2841 | |||||||
2842 | // If we optimize for code size, try to move the call to free before the null | ||||||
2843 | // test so that simplify cfg can remove the empty block and dead code | ||||||
2844 | // elimination the branch. I.e., helps to turn something like: | ||||||
2845 | // if (foo) free(foo); | ||||||
2846 | // into | ||||||
2847 | // free(foo); | ||||||
2848 | // | ||||||
2849 | // Note that we can only do this for 'free' and not for any flavor of | ||||||
2850 | // 'operator delete'; there is no 'operator delete' symbol for which we are | ||||||
2851 | // permitted to invent a call, even if we're passing in a null pointer. | ||||||
2852 | if (MinimizeSize) { | ||||||
2853 | LibFunc Func; | ||||||
2854 | if (TLI.getLibFunc(FI, Func) && TLI.has(Func) && Func == LibFunc_free) | ||||||
2855 | if (Instruction *I = tryToMoveFreeBeforeNullTest(FI, DL)) | ||||||
2856 | return I; | ||||||
2857 | } | ||||||
2858 | |||||||
2859 | return nullptr; | ||||||
2860 | } | ||||||
2861 | |||||||
2862 | static bool isMustTailCall(Value *V) { | ||||||
2863 | if (auto *CI = dyn_cast<CallInst>(V)) | ||||||
2864 | return CI->isMustTailCall(); | ||||||
2865 | return false; | ||||||
2866 | } | ||||||
2867 | |||||||
2868 | Instruction *InstCombinerImpl::visitReturnInst(ReturnInst &RI) { | ||||||
2869 | if (RI.getNumOperands() == 0) // ret void | ||||||
2870 | return nullptr; | ||||||
2871 | |||||||
2872 | Value *ResultOp = RI.getOperand(0); | ||||||
2873 | Type *VTy = ResultOp->getType(); | ||||||
2874 | if (!VTy->isIntegerTy() || isa<Constant>(ResultOp)) | ||||||
2875 | return nullptr; | ||||||
2876 | |||||||
2877 | // Don't replace result of musttail calls. | ||||||
2878 | if (isMustTailCall(ResultOp)) | ||||||
2879 | return nullptr; | ||||||
2880 | |||||||
2881 | // There might be assume intrinsics dominating this return that completely | ||||||
2882 | // determine the value. If so, constant fold it. | ||||||
2883 | KnownBits Known = computeKnownBits(ResultOp, 0, &RI); | ||||||
2884 | if (Known.isConstant()) | ||||||
2885 | return replaceOperand(RI, 0, | ||||||
2886 | Constant::getIntegerValue(VTy, Known.getConstant())); | ||||||
2887 | |||||||
2888 | return nullptr; | ||||||
2889 | } | ||||||
2890 | |||||||
2891 | // WARNING: keep in sync with SimplifyCFGOpt::simplifyUnreachable()! | ||||||
2892 | Instruction *InstCombinerImpl::visitUnreachableInst(UnreachableInst &I) { | ||||||
2893 | // Try to remove the previous instruction if it must lead to unreachable. | ||||||
2894 | // This includes instructions like stores and "llvm.assume" that may not get | ||||||
2895 | // removed by simple dead code elimination. | ||||||
2896 | while (Instruction *Prev = I.getPrevNonDebugInstruction()) { | ||||||
2897 | // While we theoretically can erase EH, that would result in a block that | ||||||
2898 | // used to start with an EH no longer starting with EH, which is invalid. | ||||||
2899 | // To make it valid, we'd need to fixup predecessors to no longer refer to | ||||||
2900 | // this block, but that changes CFG, which is not allowed in InstCombine. | ||||||
2901 | if (Prev->isEHPad()) | ||||||
2902 | return nullptr; // Can not drop any more instructions. We're done here. | ||||||
2903 | |||||||
2904 | if (!isGuaranteedToTransferExecutionToSuccessor(Prev)) | ||||||
2905 | return nullptr; // Can not drop any more instructions. We're done here. | ||||||
2906 | // Otherwise, this instruction can be freely erased, | ||||||
2907 | // even if it is not side-effect free. | ||||||
2908 | |||||||
2909 | // A value may still have uses before we process it here (for example, in | ||||||
2910 | // another unreachable block), so convert those to poison. | ||||||
2911 | replaceInstUsesWith(*Prev, PoisonValue::get(Prev->getType())); | ||||||
2912 | eraseInstFromFunction(*Prev); | ||||||
2913 | } | ||||||
2914 | assert(I.getParent()->sizeWithoutDebug() == 1 && "The block is now empty.")(static_cast <bool> (I.getParent()->sizeWithoutDebug () == 1 && "The block is now empty.") ? void (0) : __assert_fail ("I.getParent()->sizeWithoutDebug() == 1 && \"The block is now empty.\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 2914, __extension__ __PRETTY_FUNCTION__)); | ||||||
2915 | // FIXME: recurse into unconditional predecessors? | ||||||
2916 | return nullptr; | ||||||
2917 | } | ||||||
2918 | |||||||
2919 | Instruction *InstCombinerImpl::visitUnconditionalBranchInst(BranchInst &BI) { | ||||||
2920 | assert(BI.isUnconditional() && "Only for unconditional branches.")(static_cast <bool> (BI.isUnconditional() && "Only for unconditional branches." ) ? void (0) : __assert_fail ("BI.isUnconditional() && \"Only for unconditional branches.\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 2920, __extension__ __PRETTY_FUNCTION__)); | ||||||
2921 | |||||||
2922 | // If this store is the second-to-last instruction in the basic block | ||||||
2923 | // (excluding debug info and bitcasts of pointers) and if the block ends with | ||||||
2924 | // an unconditional branch, try to move the store to the successor block. | ||||||
2925 | |||||||
2926 | auto GetLastSinkableStore = [](BasicBlock::iterator BBI) { | ||||||
2927 | auto IsNoopInstrForStoreMerging = [](BasicBlock::iterator BBI) { | ||||||
2928 | return isa<DbgInfoIntrinsic>(BBI) || | ||||||
2929 | (isa<BitCastInst>(BBI) && BBI->getType()->isPointerTy()); | ||||||
2930 | }; | ||||||
2931 | |||||||
2932 | BasicBlock::iterator FirstInstr = BBI->getParent()->begin(); | ||||||
2933 | do { | ||||||
2934 | if (BBI != FirstInstr) | ||||||
2935 | --BBI; | ||||||
2936 | } while (BBI != FirstInstr && IsNoopInstrForStoreMerging(BBI)); | ||||||
2937 | |||||||
2938 | return dyn_cast<StoreInst>(BBI); | ||||||
2939 | }; | ||||||
2940 | |||||||
2941 | if (StoreInst *SI = GetLastSinkableStore(BasicBlock::iterator(BI))) | ||||||
2942 | if (mergeStoreIntoSuccessor(*SI)) | ||||||
2943 | return &BI; | ||||||
2944 | |||||||
2945 | return nullptr; | ||||||
2946 | } | ||||||
2947 | |||||||
2948 | Instruction *InstCombinerImpl::visitBranchInst(BranchInst &BI) { | ||||||
2949 | if (BI.isUnconditional()) | ||||||
2950 | return visitUnconditionalBranchInst(BI); | ||||||
2951 | |||||||
2952 | // Change br (not X), label True, label False to: br X, label False, True | ||||||
2953 | Value *X = nullptr; | ||||||
2954 | if (match(&BI, m_Br(m_Not(m_Value(X)), m_BasicBlock(), m_BasicBlock())) && | ||||||
2955 | !isa<Constant>(X)) { | ||||||
2956 | // Swap Destinations and condition... | ||||||
2957 | BI.swapSuccessors(); | ||||||
2958 | return replaceOperand(BI, 0, X); | ||||||
2959 | } | ||||||
2960 | |||||||
2961 | // If the condition is irrelevant, remove the use so that other | ||||||
2962 | // transforms on the condition become more effective. | ||||||
2963 | if (!isa<ConstantInt>(BI.getCondition()) && | ||||||
2964 | BI.getSuccessor(0) == BI.getSuccessor(1)) | ||||||
2965 | return replaceOperand( | ||||||
2966 | BI, 0, ConstantInt::getFalse(BI.getCondition()->getType())); | ||||||
2967 | |||||||
2968 | // Canonicalize, for example, fcmp_one -> fcmp_oeq. | ||||||
2969 | CmpInst::Predicate Pred; | ||||||
2970 | if (match(&BI, m_Br(m_OneUse(m_FCmp(Pred, m_Value(), m_Value())), | ||||||
2971 | m_BasicBlock(), m_BasicBlock())) && | ||||||
2972 | !isCanonicalPredicate(Pred)) { | ||||||
2973 | // Swap destinations and condition. | ||||||
2974 | CmpInst *Cond = cast<CmpInst>(BI.getCondition()); | ||||||
2975 | Cond->setPredicate(CmpInst::getInversePredicate(Pred)); | ||||||
2976 | BI.swapSuccessors(); | ||||||
2977 | Worklist.push(Cond); | ||||||
2978 | return &BI; | ||||||
2979 | } | ||||||
2980 | |||||||
2981 | return nullptr; | ||||||
2982 | } | ||||||
2983 | |||||||
2984 | Instruction *InstCombinerImpl::visitSwitchInst(SwitchInst &SI) { | ||||||
2985 | Value *Cond = SI.getCondition(); | ||||||
2986 | Value *Op0; | ||||||
2987 | ConstantInt *AddRHS; | ||||||
2988 | if (match(Cond, m_Add(m_Value(Op0), m_ConstantInt(AddRHS)))) { | ||||||
2989 | // Change 'switch (X+4) case 1:' into 'switch (X) case -3'. | ||||||
2990 | for (auto Case : SI.cases()) { | ||||||
2991 | Constant *NewCase = ConstantExpr::getSub(Case.getCaseValue(), AddRHS); | ||||||
2992 | assert(isa<ConstantInt>(NewCase) &&(static_cast <bool> (isa<ConstantInt>(NewCase) && "Result of expression should be constant") ? void (0) : __assert_fail ("isa<ConstantInt>(NewCase) && \"Result of expression should be constant\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 2993, __extension__ __PRETTY_FUNCTION__)) | ||||||
2993 | "Result of expression should be constant")(static_cast <bool> (isa<ConstantInt>(NewCase) && "Result of expression should be constant") ? void (0) : __assert_fail ("isa<ConstantInt>(NewCase) && \"Result of expression should be constant\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 2993, __extension__ __PRETTY_FUNCTION__)); | ||||||
2994 | Case.setValue(cast<ConstantInt>(NewCase)); | ||||||
2995 | } | ||||||
2996 | return replaceOperand(SI, 0, Op0); | ||||||
2997 | } | ||||||
2998 | |||||||
2999 | KnownBits Known = computeKnownBits(Cond, 0, &SI); | ||||||
3000 | unsigned LeadingKnownZeros = Known.countMinLeadingZeros(); | ||||||
3001 | unsigned LeadingKnownOnes = Known.countMinLeadingOnes(); | ||||||
3002 | |||||||
3003 | // Compute the number of leading bits we can ignore. | ||||||
3004 | // TODO: A better way to determine this would use ComputeNumSignBits(). | ||||||
3005 | for (auto &C : SI.cases()) { | ||||||
3006 | LeadingKnownZeros = std::min( | ||||||
3007 | LeadingKnownZeros, C.getCaseValue()->getValue().countLeadingZeros()); | ||||||
3008 | LeadingKnownOnes = std::min( | ||||||
3009 | LeadingKnownOnes, C.getCaseValue()->getValue().countLeadingOnes()); | ||||||
3010 | } | ||||||
3011 | |||||||
3012 | unsigned NewWidth = Known.getBitWidth() - std::max(LeadingKnownZeros, LeadingKnownOnes); | ||||||
3013 | |||||||
3014 | // Shrink the condition operand if the new type is smaller than the old type. | ||||||
3015 | // But do not shrink to a non-standard type, because backend can't generate | ||||||
3016 | // good code for that yet. | ||||||
3017 | // TODO: We can make it aggressive again after fixing PR39569. | ||||||
3018 | if (NewWidth > 0 && NewWidth < Known.getBitWidth() && | ||||||
3019 | shouldChangeType(Known.getBitWidth(), NewWidth)) { | ||||||
3020 | IntegerType *Ty = IntegerType::get(SI.getContext(), NewWidth); | ||||||
3021 | Builder.SetInsertPoint(&SI); | ||||||
3022 | Value *NewCond = Builder.CreateTrunc(Cond, Ty, "trunc"); | ||||||
3023 | |||||||
3024 | for (auto Case : SI.cases()) { | ||||||
3025 | APInt TruncatedCase = Case.getCaseValue()->getValue().trunc(NewWidth); | ||||||
3026 | Case.setValue(ConstantInt::get(SI.getContext(), TruncatedCase)); | ||||||
3027 | } | ||||||
3028 | return replaceOperand(SI, 0, NewCond); | ||||||
3029 | } | ||||||
3030 | |||||||
3031 | return nullptr; | ||||||
3032 | } | ||||||
3033 | |||||||
3034 | Instruction *InstCombinerImpl::visitExtractValueInst(ExtractValueInst &EV) { | ||||||
3035 | Value *Agg = EV.getAggregateOperand(); | ||||||
3036 | |||||||
3037 | if (!EV.hasIndices()) | ||||||
3038 | return replaceInstUsesWith(EV, Agg); | ||||||
3039 | |||||||
3040 | if (Value *V = SimplifyExtractValueInst(Agg, EV.getIndices(), | ||||||
3041 | SQ.getWithInstruction(&EV))) | ||||||
3042 | return replaceInstUsesWith(EV, V); | ||||||
3043 | |||||||
3044 | if (InsertValueInst *IV = dyn_cast<InsertValueInst>(Agg)) { | ||||||
3045 | // We're extracting from an insertvalue instruction, compare the indices | ||||||
3046 | const unsigned *exti, *exte, *insi, *inse; | ||||||
3047 | for (exti = EV.idx_begin(), insi = IV->idx_begin(), | ||||||
3048 | exte = EV.idx_end(), inse = IV->idx_end(); | ||||||
3049 | exti != exte && insi != inse; | ||||||
3050 | ++exti, ++insi) { | ||||||
3051 | if (*insi != *exti) | ||||||
3052 | // The insert and extract both reference distinctly different elements. | ||||||
3053 | // This means the extract is not influenced by the insert, and we can | ||||||
3054 | // replace the aggregate operand of the extract with the aggregate | ||||||
3055 | // operand of the insert. i.e., replace | ||||||
3056 | // %I = insertvalue { i32, { i32 } } %A, { i32 } { i32 42 }, 1 | ||||||
3057 | // %E = extractvalue { i32, { i32 } } %I, 0 | ||||||
3058 | // with | ||||||
3059 | // %E = extractvalue { i32, { i32 } } %A, 0 | ||||||
3060 | return ExtractValueInst::Create(IV->getAggregateOperand(), | ||||||
3061 | EV.getIndices()); | ||||||
3062 | } | ||||||
3063 | if (exti == exte && insi == inse) | ||||||
3064 | // Both iterators are at the end: Index lists are identical. Replace | ||||||
3065 | // %B = insertvalue { i32, { i32 } } %A, i32 42, 1, 0 | ||||||
3066 | // %C = extractvalue { i32, { i32 } } %B, 1, 0 | ||||||
3067 | // with "i32 42" | ||||||
3068 | return replaceInstUsesWith(EV, IV->getInsertedValueOperand()); | ||||||
3069 | if (exti == exte) { | ||||||
3070 | // The extract list is a prefix of the insert list. i.e. replace | ||||||
3071 | // %I = insertvalue { i32, { i32 } } %A, i32 42, 1, 0 | ||||||
3072 | // %E = extractvalue { i32, { i32 } } %I, 1 | ||||||
3073 | // with | ||||||
3074 | // %X = extractvalue { i32, { i32 } } %A, 1 | ||||||
3075 | // %E = insertvalue { i32 } %X, i32 42, 0 | ||||||
3076 | // by switching the order of the insert and extract (though the | ||||||
3077 | // insertvalue should be left in, since it may have other uses). | ||||||
3078 | Value *NewEV = Builder.CreateExtractValue(IV->getAggregateOperand(), | ||||||
3079 | EV.getIndices()); | ||||||
3080 | return InsertValueInst::Create(NewEV, IV->getInsertedValueOperand(), | ||||||
3081 | makeArrayRef(insi, inse)); | ||||||
3082 | } | ||||||
3083 | if (insi == inse) | ||||||
3084 | // The insert list is a prefix of the extract list | ||||||
3085 | // We can simply remove the common indices from the extract and make it | ||||||
3086 | // operate on the inserted value instead of the insertvalue result. | ||||||
3087 | // i.e., replace | ||||||
3088 | // %I = insertvalue { i32, { i32 } } %A, { i32 } { i32 42 }, 1 | ||||||
3089 | // %E = extractvalue { i32, { i32 } } %I, 1, 0 | ||||||
3090 | // with | ||||||
3091 | // %E extractvalue { i32 } { i32 42 }, 0 | ||||||
3092 | return ExtractValueInst::Create(IV->getInsertedValueOperand(), | ||||||
3093 | makeArrayRef(exti, exte)); | ||||||
3094 | } | ||||||
3095 | if (WithOverflowInst *WO = dyn_cast<WithOverflowInst>(Agg)) { | ||||||
3096 | // We're extracting from an overflow intrinsic, see if we're the only user, | ||||||
3097 | // which allows us to simplify multiple result intrinsics to simpler | ||||||
3098 | // things that just get one value. | ||||||
3099 | if (WO->hasOneUse()) { | ||||||
3100 | // Check if we're grabbing only the result of a 'with overflow' intrinsic | ||||||
3101 | // and replace it with a traditional binary instruction. | ||||||
3102 | if (*EV.idx_begin() == 0) { | ||||||
3103 | Instruction::BinaryOps BinOp = WO->getBinaryOp(); | ||||||
3104 | Value *LHS = WO->getLHS(), *RHS = WO->getRHS(); | ||||||
3105 | // Replace the old instruction's uses with poison. | ||||||
3106 | replaceInstUsesWith(*WO, PoisonValue::get(WO->getType())); | ||||||
3107 | eraseInstFromFunction(*WO); | ||||||
3108 | return BinaryOperator::Create(BinOp, LHS, RHS); | ||||||
3109 | } | ||||||
3110 | |||||||
3111 | assert(*EV.idx_begin() == 1 &&(static_cast <bool> (*EV.idx_begin() == 1 && "unexpected extract index for overflow inst" ) ? void (0) : __assert_fail ("*EV.idx_begin() == 1 && \"unexpected extract index for overflow inst\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3112, __extension__ __PRETTY_FUNCTION__)) | ||||||
3112 | "unexpected extract index for overflow inst")(static_cast <bool> (*EV.idx_begin() == 1 && "unexpected extract index for overflow inst" ) ? void (0) : __assert_fail ("*EV.idx_begin() == 1 && \"unexpected extract index for overflow inst\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3112, __extension__ __PRETTY_FUNCTION__)); | ||||||
3113 | |||||||
3114 | // If only the overflow result is used, and the right hand side is a | ||||||
3115 | // constant (or constant splat), we can remove the intrinsic by directly | ||||||
3116 | // checking for overflow. | ||||||
3117 | const APInt *C; | ||||||
3118 | if (match(WO->getRHS(), m_APInt(C))) { | ||||||
3119 | // Compute the no-wrap range [X,Y) for LHS given RHS=C, then | ||||||
3120 | // check for the inverted range using range offset trick (i.e. | ||||||
3121 | // use a subtract to shift the range to bottom of either the | ||||||
3122 | // signed or unsigned domain and then use a single compare to | ||||||
3123 | // check range membership). | ||||||
3124 | ConstantRange NWR = | ||||||
3125 | ConstantRange::makeExactNoWrapRegion(WO->getBinaryOp(), *C, | ||||||
3126 | WO->getNoWrapKind()); | ||||||
3127 | APInt Min = WO->isSigned() ? NWR.getSignedMin() : NWR.getUnsignedMin(); | ||||||
3128 | NWR = NWR.subtract(Min); | ||||||
3129 | |||||||
3130 | CmpInst::Predicate Pred; | ||||||
3131 | APInt NewRHSC; | ||||||
3132 | if (NWR.getEquivalentICmp(Pred, NewRHSC)) { | ||||||
3133 | auto *OpTy = WO->getRHS()->getType(); | ||||||
3134 | auto *NewLHS = Builder.CreateSub(WO->getLHS(), | ||||||
3135 | ConstantInt::get(OpTy, Min)); | ||||||
3136 | return new ICmpInst(ICmpInst::getInversePredicate(Pred), NewLHS, | ||||||
3137 | ConstantInt::get(OpTy, NewRHSC)); | ||||||
3138 | } | ||||||
3139 | } | ||||||
3140 | } | ||||||
3141 | } | ||||||
3142 | if (LoadInst *L = dyn_cast<LoadInst>(Agg)) | ||||||
3143 | // If the (non-volatile) load only has one use, we can rewrite this to a | ||||||
3144 | // load from a GEP. This reduces the size of the load. If a load is used | ||||||
3145 | // only by extractvalue instructions then this either must have been | ||||||
3146 | // optimized before, or it is a struct with padding, in which case we | ||||||
3147 | // don't want to do the transformation as it loses padding knowledge. | ||||||
3148 | if (L->isSimple() && L->hasOneUse()) { | ||||||
3149 | // extractvalue has integer indices, getelementptr has Value*s. Convert. | ||||||
3150 | SmallVector<Value*, 4> Indices; | ||||||
3151 | // Prefix an i32 0 since we need the first element. | ||||||
3152 | Indices.push_back(Builder.getInt32(0)); | ||||||
3153 | for (unsigned Idx : EV.indices()) | ||||||
3154 | Indices.push_back(Builder.getInt32(Idx)); | ||||||
3155 | |||||||
3156 | // We need to insert these at the location of the old load, not at that of | ||||||
3157 | // the extractvalue. | ||||||
3158 | Builder.SetInsertPoint(L); | ||||||
3159 | Value *GEP = Builder.CreateInBoundsGEP(L->getType(), | ||||||
3160 | L->getPointerOperand(), Indices); | ||||||
3161 | Instruction *NL = Builder.CreateLoad(EV.getType(), GEP); | ||||||
3162 | // Whatever aliasing information we had for the orignal load must also | ||||||
3163 | // hold for the smaller load, so propagate the annotations. | ||||||
3164 | NL->setAAMetadata(L->getAAMetadata()); | ||||||
3165 | // Returning the load directly will cause the main loop to insert it in | ||||||
3166 | // the wrong spot, so use replaceInstUsesWith(). | ||||||
3167 | return replaceInstUsesWith(EV, NL); | ||||||
3168 | } | ||||||
3169 | // We could simplify extracts from other values. Note that nested extracts may | ||||||
3170 | // already be simplified implicitly by the above: extract (extract (insert) ) | ||||||
3171 | // will be translated into extract ( insert ( extract ) ) first and then just | ||||||
3172 | // the value inserted, if appropriate. Similarly for extracts from single-use | ||||||
3173 | // loads: extract (extract (load)) will be translated to extract (load (gep)) | ||||||
3174 | // and if again single-use then via load (gep (gep)) to load (gep). | ||||||
3175 | // However, double extracts from e.g. function arguments or return values | ||||||
3176 | // aren't handled yet. | ||||||
3177 | return nullptr; | ||||||
3178 | } | ||||||
3179 | |||||||
3180 | /// Return 'true' if the given typeinfo will match anything. | ||||||
3181 | static bool isCatchAll(EHPersonality Personality, Constant *TypeInfo) { | ||||||
3182 | switch (Personality) { | ||||||
3183 | case EHPersonality::GNU_C: | ||||||
3184 | case EHPersonality::GNU_C_SjLj: | ||||||
3185 | case EHPersonality::Rust: | ||||||
3186 | // The GCC C EH and Rust personality only exists to support cleanups, so | ||||||
3187 | // it's not clear what the semantics of catch clauses are. | ||||||
3188 | return false; | ||||||
3189 | case EHPersonality::Unknown: | ||||||
3190 | return false; | ||||||
3191 | case EHPersonality::GNU_Ada: | ||||||
3192 | // While __gnat_all_others_value will match any Ada exception, it doesn't | ||||||
3193 | // match foreign exceptions (or didn't, before gcc-4.7). | ||||||
3194 | return false; | ||||||
3195 | case EHPersonality::GNU_CXX: | ||||||
3196 | case EHPersonality::GNU_CXX_SjLj: | ||||||
3197 | case EHPersonality::GNU_ObjC: | ||||||
3198 | case EHPersonality::MSVC_X86SEH: | ||||||
3199 | case EHPersonality::MSVC_TableSEH: | ||||||
3200 | case EHPersonality::MSVC_CXX: | ||||||
3201 | case EHPersonality::CoreCLR: | ||||||
3202 | case EHPersonality::Wasm_CXX: | ||||||
3203 | case EHPersonality::XL_CXX: | ||||||
3204 | return TypeInfo->isNullValue(); | ||||||
3205 | } | ||||||
3206 | llvm_unreachable("invalid enum")::llvm::llvm_unreachable_internal("invalid enum", "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3206); | ||||||
3207 | } | ||||||
3208 | |||||||
3209 | static bool shorter_filter(const Value *LHS, const Value *RHS) { | ||||||
3210 | return | ||||||
3211 | cast<ArrayType>(LHS->getType())->getNumElements() | ||||||
3212 | < | ||||||
3213 | cast<ArrayType>(RHS->getType())->getNumElements(); | ||||||
3214 | } | ||||||
3215 | |||||||
3216 | Instruction *InstCombinerImpl::visitLandingPadInst(LandingPadInst &LI) { | ||||||
3217 | // The logic here should be correct for any real-world personality function. | ||||||
3218 | // However if that turns out not to be true, the offending logic can always | ||||||
3219 | // be conditioned on the personality function, like the catch-all logic is. | ||||||
3220 | EHPersonality Personality = | ||||||
3221 | classifyEHPersonality(LI.getParent()->getParent()->getPersonalityFn()); | ||||||
3222 | |||||||
3223 | // Simplify the list of clauses, eg by removing repeated catch clauses | ||||||
3224 | // (these are often created by inlining). | ||||||
3225 | bool MakeNewInstruction = false; // If true, recreate using the following: | ||||||
3226 | SmallVector<Constant *, 16> NewClauses; // - Clauses for the new instruction; | ||||||
3227 | bool CleanupFlag = LI.isCleanup(); // - The new instruction is a cleanup. | ||||||
3228 | |||||||
3229 | SmallPtrSet<Value *, 16> AlreadyCaught; // Typeinfos known caught already. | ||||||
3230 | for (unsigned i = 0, e = LI.getNumClauses(); i != e; ++i) { | ||||||
3231 | bool isLastClause = i + 1 == e; | ||||||
3232 | if (LI.isCatch(i)) { | ||||||
3233 | // A catch clause. | ||||||
3234 | Constant *CatchClause = LI.getClause(i); | ||||||
3235 | Constant *TypeInfo = CatchClause->stripPointerCasts(); | ||||||
3236 | |||||||
3237 | // If we already saw this clause, there is no point in having a second | ||||||
3238 | // copy of it. | ||||||
3239 | if (AlreadyCaught.insert(TypeInfo).second) { | ||||||
3240 | // This catch clause was not already seen. | ||||||
3241 | NewClauses.push_back(CatchClause); | ||||||
3242 | } else { | ||||||
3243 | // Repeated catch clause - drop the redundant copy. | ||||||
3244 | MakeNewInstruction = true; | ||||||
3245 | } | ||||||
3246 | |||||||
3247 | // If this is a catch-all then there is no point in keeping any following | ||||||
3248 | // clauses or marking the landingpad as having a cleanup. | ||||||
3249 | if (isCatchAll(Personality, TypeInfo)) { | ||||||
3250 | if (!isLastClause) | ||||||
3251 | MakeNewInstruction = true; | ||||||
3252 | CleanupFlag = false; | ||||||
3253 | break; | ||||||
3254 | } | ||||||
3255 | } else { | ||||||
3256 | // A filter clause. If any of the filter elements were already caught | ||||||
3257 | // then they can be dropped from the filter. It is tempting to try to | ||||||
3258 | // exploit the filter further by saying that any typeinfo that does not | ||||||
3259 | // occur in the filter can't be caught later (and thus can be dropped). | ||||||
3260 | // However this would be wrong, since typeinfos can match without being | ||||||
3261 | // equal (for example if one represents a C++ class, and the other some | ||||||
3262 | // class derived from it). | ||||||
3263 | assert(LI.isFilter(i) && "Unsupported landingpad clause!")(static_cast <bool> (LI.isFilter(i) && "Unsupported landingpad clause!" ) ? void (0) : __assert_fail ("LI.isFilter(i) && \"Unsupported landingpad clause!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3263, __extension__ __PRETTY_FUNCTION__)); | ||||||
3264 | Constant *FilterClause = LI.getClause(i); | ||||||
3265 | ArrayType *FilterType = cast<ArrayType>(FilterClause->getType()); | ||||||
3266 | unsigned NumTypeInfos = FilterType->getNumElements(); | ||||||
3267 | |||||||
3268 | // An empty filter catches everything, so there is no point in keeping any | ||||||
3269 | // following clauses or marking the landingpad as having a cleanup. By | ||||||
3270 | // dealing with this case here the following code is made a bit simpler. | ||||||
3271 | if (!NumTypeInfos) { | ||||||
3272 | NewClauses.push_back(FilterClause); | ||||||
3273 | if (!isLastClause) | ||||||
3274 | MakeNewInstruction = true; | ||||||
3275 | CleanupFlag = false; | ||||||
3276 | break; | ||||||
3277 | } | ||||||
3278 | |||||||
3279 | bool MakeNewFilter = false; // If true, make a new filter. | ||||||
3280 | SmallVector<Constant *, 16> NewFilterElts; // New elements. | ||||||
3281 | if (isa<ConstantAggregateZero>(FilterClause)) { | ||||||
3282 | // Not an empty filter - it contains at least one null typeinfo. | ||||||
3283 | assert(NumTypeInfos > 0 && "Should have handled empty filter already!")(static_cast <bool> (NumTypeInfos > 0 && "Should have handled empty filter already!" ) ? void (0) : __assert_fail ("NumTypeInfos > 0 && \"Should have handled empty filter already!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3283, __extension__ __PRETTY_FUNCTION__)); | ||||||
3284 | Constant *TypeInfo = | ||||||
3285 | Constant::getNullValue(FilterType->getElementType()); | ||||||
3286 | // If this typeinfo is a catch-all then the filter can never match. | ||||||
3287 | if (isCatchAll(Personality, TypeInfo)) { | ||||||
3288 | // Throw the filter away. | ||||||
3289 | MakeNewInstruction = true; | ||||||
3290 | continue; | ||||||
3291 | } | ||||||
3292 | |||||||
3293 | // There is no point in having multiple copies of this typeinfo, so | ||||||
3294 | // discard all but the first copy if there is more than one. | ||||||
3295 | NewFilterElts.push_back(TypeInfo); | ||||||
3296 | if (NumTypeInfos > 1) | ||||||
3297 | MakeNewFilter = true; | ||||||
3298 | } else { | ||||||
3299 | ConstantArray *Filter = cast<ConstantArray>(FilterClause); | ||||||
3300 | SmallPtrSet<Value *, 16> SeenInFilter; // For uniquing the elements. | ||||||
3301 | NewFilterElts.reserve(NumTypeInfos); | ||||||
3302 | |||||||
3303 | // Remove any filter elements that were already caught or that already | ||||||
3304 | // occurred in the filter. While there, see if any of the elements are | ||||||
3305 | // catch-alls. If so, the filter can be discarded. | ||||||
3306 | bool SawCatchAll = false; | ||||||
3307 | for (unsigned j = 0; j != NumTypeInfos; ++j) { | ||||||
3308 | Constant *Elt = Filter->getOperand(j); | ||||||
3309 | Constant *TypeInfo = Elt->stripPointerCasts(); | ||||||
3310 | if (isCatchAll(Personality, TypeInfo)) { | ||||||
3311 | // This element is a catch-all. Bail out, noting this fact. | ||||||
3312 | SawCatchAll = true; | ||||||
3313 | break; | ||||||
3314 | } | ||||||
3315 | |||||||
3316 | // Even if we've seen a type in a catch clause, we don't want to | ||||||
3317 | // remove it from the filter. An unexpected type handler may be | ||||||
3318 | // set up for a call site which throws an exception of the same | ||||||
3319 | // type caught. In order for the exception thrown by the unexpected | ||||||
3320 | // handler to propagate correctly, the filter must be correctly | ||||||
3321 | // described for the call site. | ||||||
3322 | // | ||||||
3323 | // Example: | ||||||
3324 | // | ||||||
3325 | // void unexpected() { throw 1;} | ||||||
3326 | // void foo() throw (int) { | ||||||
3327 | // std::set_unexpected(unexpected); | ||||||
3328 | // try { | ||||||
3329 | // throw 2.0; | ||||||
3330 | // } catch (int i) {} | ||||||
3331 | // } | ||||||
3332 | |||||||
3333 | // There is no point in having multiple copies of the same typeinfo in | ||||||
3334 | // a filter, so only add it if we didn't already. | ||||||
3335 | if (SeenInFilter.insert(TypeInfo).second) | ||||||
3336 | NewFilterElts.push_back(cast<Constant>(Elt)); | ||||||
3337 | } | ||||||
3338 | // A filter containing a catch-all cannot match anything by definition. | ||||||
3339 | if (SawCatchAll) { | ||||||
3340 | // Throw the filter away. | ||||||
3341 | MakeNewInstruction = true; | ||||||
3342 | continue; | ||||||
3343 | } | ||||||
3344 | |||||||
3345 | // If we dropped something from the filter, make a new one. | ||||||
3346 | if (NewFilterElts.size() < NumTypeInfos) | ||||||
3347 | MakeNewFilter = true; | ||||||
3348 | } | ||||||
3349 | if (MakeNewFilter) { | ||||||
3350 | FilterType = ArrayType::get(FilterType->getElementType(), | ||||||
3351 | NewFilterElts.size()); | ||||||
3352 | FilterClause = ConstantArray::get(FilterType, NewFilterElts); | ||||||
3353 | MakeNewInstruction = true; | ||||||
3354 | } | ||||||
3355 | |||||||
3356 | NewClauses.push_back(FilterClause); | ||||||
3357 | |||||||
3358 | // If the new filter is empty then it will catch everything so there is | ||||||
3359 | // no point in keeping any following clauses or marking the landingpad | ||||||
3360 | // as having a cleanup. The case of the original filter being empty was | ||||||
3361 | // already handled above. | ||||||
3362 | if (MakeNewFilter && !NewFilterElts.size()) { | ||||||
3363 | assert(MakeNewInstruction && "New filter but not a new instruction!")(static_cast <bool> (MakeNewInstruction && "New filter but not a new instruction!" ) ? void (0) : __assert_fail ("MakeNewInstruction && \"New filter but not a new instruction!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3363, __extension__ __PRETTY_FUNCTION__)); | ||||||
3364 | CleanupFlag = false; | ||||||
3365 | break; | ||||||
3366 | } | ||||||
3367 | } | ||||||
3368 | } | ||||||
3369 | |||||||
3370 | // If several filters occur in a row then reorder them so that the shortest | ||||||
3371 | // filters come first (those with the smallest number of elements). This is | ||||||
3372 | // advantageous because shorter filters are more likely to match, speeding up | ||||||
3373 | // unwinding, but mostly because it increases the effectiveness of the other | ||||||
3374 | // filter optimizations below. | ||||||
3375 | for (unsigned i = 0, e = NewClauses.size(); i + 1 < e; ) { | ||||||
3376 | unsigned j; | ||||||
3377 | // Find the maximal 'j' s.t. the range [i, j) consists entirely of filters. | ||||||
3378 | for (j = i; j != e; ++j) | ||||||
3379 | if (!isa<ArrayType>(NewClauses[j]->getType())) | ||||||
3380 | break; | ||||||
3381 | |||||||
3382 | // Check whether the filters are already sorted by length. We need to know | ||||||
3383 | // if sorting them is actually going to do anything so that we only make a | ||||||
3384 | // new landingpad instruction if it does. | ||||||
3385 | for (unsigned k = i; k + 1 < j; ++k) | ||||||
3386 | if (shorter_filter(NewClauses[k+1], NewClauses[k])) { | ||||||
3387 | // Not sorted, so sort the filters now. Doing an unstable sort would be | ||||||
3388 | // correct too but reordering filters pointlessly might confuse users. | ||||||
3389 | std::stable_sort(NewClauses.begin() + i, NewClauses.begin() + j, | ||||||
3390 | shorter_filter); | ||||||
3391 | MakeNewInstruction = true; | ||||||
3392 | break; | ||||||
3393 | } | ||||||
3394 | |||||||
3395 | // Look for the next batch of filters. | ||||||
3396 | i = j + 1; | ||||||
3397 | } | ||||||
3398 | |||||||
3399 | // If typeinfos matched if and only if equal, then the elements of a filter L | ||||||
3400 | // that occurs later than a filter F could be replaced by the intersection of | ||||||
3401 | // the elements of F and L. In reality two typeinfos can match without being | ||||||
3402 | // equal (for example if one represents a C++ class, and the other some class | ||||||
3403 | // derived from it) so it would be wrong to perform this transform in general. | ||||||
3404 | // However the transform is correct and useful if F is a subset of L. In that | ||||||
3405 | // case L can be replaced by F, and thus removed altogether since repeating a | ||||||
3406 | // filter is pointless. So here we look at all pairs of filters F and L where | ||||||
3407 | // L follows F in the list of clauses, and remove L if every element of F is | ||||||
3408 | // an element of L. This can occur when inlining C++ functions with exception | ||||||
3409 | // specifications. | ||||||
3410 | for (unsigned i = 0; i + 1 < NewClauses.size(); ++i) { | ||||||
3411 | // Examine each filter in turn. | ||||||
3412 | Value *Filter = NewClauses[i]; | ||||||
3413 | ArrayType *FTy = dyn_cast<ArrayType>(Filter->getType()); | ||||||
3414 | if (!FTy) | ||||||
3415 | // Not a filter - skip it. | ||||||
3416 | continue; | ||||||
3417 | unsigned FElts = FTy->getNumElements(); | ||||||
3418 | // Examine each filter following this one. Doing this backwards means that | ||||||
3419 | // we don't have to worry about filters disappearing under us when removed. | ||||||
3420 | for (unsigned j = NewClauses.size() - 1; j != i; --j) { | ||||||
3421 | Value *LFilter = NewClauses[j]; | ||||||
3422 | ArrayType *LTy = dyn_cast<ArrayType>(LFilter->getType()); | ||||||
3423 | if (!LTy) | ||||||
3424 | // Not a filter - skip it. | ||||||
3425 | continue; | ||||||
3426 | // If Filter is a subset of LFilter, i.e. every element of Filter is also | ||||||
3427 | // an element of LFilter, then discard LFilter. | ||||||
3428 | SmallVectorImpl<Constant *>::iterator J = NewClauses.begin() + j; | ||||||
3429 | // If Filter is empty then it is a subset of LFilter. | ||||||
3430 | if (!FElts) { | ||||||
3431 | // Discard LFilter. | ||||||
3432 | NewClauses.erase(J); | ||||||
3433 | MakeNewInstruction = true; | ||||||
3434 | // Move on to the next filter. | ||||||
3435 | continue; | ||||||
3436 | } | ||||||
3437 | unsigned LElts = LTy->getNumElements(); | ||||||
3438 | // If Filter is longer than LFilter then it cannot be a subset of it. | ||||||
3439 | if (FElts > LElts) | ||||||
3440 | // Move on to the next filter. | ||||||
3441 | continue; | ||||||
3442 | // At this point we know that LFilter has at least one element. | ||||||
3443 | if (isa<ConstantAggregateZero>(LFilter)) { // LFilter only contains zeros. | ||||||
3444 | // Filter is a subset of LFilter iff Filter contains only zeros (as we | ||||||
3445 | // already know that Filter is not longer than LFilter). | ||||||
3446 | if (isa<ConstantAggregateZero>(Filter)) { | ||||||
3447 | assert(FElts <= LElts && "Should have handled this case earlier!")(static_cast <bool> (FElts <= LElts && "Should have handled this case earlier!" ) ? void (0) : __assert_fail ("FElts <= LElts && \"Should have handled this case earlier!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3447, __extension__ __PRETTY_FUNCTION__)); | ||||||
3448 | // Discard LFilter. | ||||||
3449 | NewClauses.erase(J); | ||||||
3450 | MakeNewInstruction = true; | ||||||
3451 | } | ||||||
3452 | // Move on to the next filter. | ||||||
3453 | continue; | ||||||
3454 | } | ||||||
3455 | ConstantArray *LArray = cast<ConstantArray>(LFilter); | ||||||
3456 | if (isa<ConstantAggregateZero>(Filter)) { // Filter only contains zeros. | ||||||
3457 | // Since Filter is non-empty and contains only zeros, it is a subset of | ||||||
3458 | // LFilter iff LFilter contains a zero. | ||||||
3459 | assert(FElts > 0 && "Should have eliminated the empty filter earlier!")(static_cast <bool> (FElts > 0 && "Should have eliminated the empty filter earlier!" ) ? void (0) : __assert_fail ("FElts > 0 && \"Should have eliminated the empty filter earlier!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3459, __extension__ __PRETTY_FUNCTION__)); | ||||||
3460 | for (unsigned l = 0; l != LElts; ++l) | ||||||
3461 | if (LArray->getOperand(l)->isNullValue()) { | ||||||
3462 | // LFilter contains a zero - discard it. | ||||||
3463 | NewClauses.erase(J); | ||||||
3464 | MakeNewInstruction = true; | ||||||
3465 | break; | ||||||
3466 | } | ||||||
3467 | // Move on to the next filter. | ||||||
3468 | continue; | ||||||
3469 | } | ||||||
3470 | // At this point we know that both filters are ConstantArrays. Loop over | ||||||
3471 | // operands to see whether every element of Filter is also an element of | ||||||
3472 | // LFilter. Since filters tend to be short this is probably faster than | ||||||
3473 | // using a method that scales nicely. | ||||||
3474 | ConstantArray *FArray = cast<ConstantArray>(Filter); | ||||||
3475 | bool AllFound = true; | ||||||
3476 | for (unsigned f = 0; f != FElts; ++f) { | ||||||
3477 | Value *FTypeInfo = FArray->getOperand(f)->stripPointerCasts(); | ||||||
3478 | AllFound = false; | ||||||
3479 | for (unsigned l = 0; l != LElts; ++l) { | ||||||
3480 | Value *LTypeInfo = LArray->getOperand(l)->stripPointerCasts(); | ||||||
3481 | if (LTypeInfo == FTypeInfo) { | ||||||
3482 | AllFound = true; | ||||||
3483 | break; | ||||||
3484 | } | ||||||
3485 | } | ||||||
3486 | if (!AllFound) | ||||||
3487 | break; | ||||||
3488 | } | ||||||
3489 | if (AllFound) { | ||||||
3490 | // Discard LFilter. | ||||||
3491 | NewClauses.erase(J); | ||||||
3492 | MakeNewInstruction = true; | ||||||
3493 | } | ||||||
3494 | // Move on to the next filter. | ||||||
3495 | } | ||||||
3496 | } | ||||||
3497 | |||||||
3498 | // If we changed any of the clauses, replace the old landingpad instruction | ||||||
3499 | // with a new one. | ||||||
3500 | if (MakeNewInstruction) { | ||||||
3501 | LandingPadInst *NLI = LandingPadInst::Create(LI.getType(), | ||||||
3502 | NewClauses.size()); | ||||||
3503 | for (unsigned i = 0, e = NewClauses.size(); i != e; ++i) | ||||||
3504 | NLI->addClause(NewClauses[i]); | ||||||
3505 | // A landing pad with no clauses must have the cleanup flag set. It is | ||||||
3506 | // theoretically possible, though highly unlikely, that we eliminated all | ||||||
3507 | // clauses. If so, force the cleanup flag to true. | ||||||
3508 | if (NewClauses.empty()) | ||||||
3509 | CleanupFlag = true; | ||||||
3510 | NLI->setCleanup(CleanupFlag); | ||||||
3511 | return NLI; | ||||||
3512 | } | ||||||
3513 | |||||||
3514 | // Even if none of the clauses changed, we may nonetheless have understood | ||||||
3515 | // that the cleanup flag is pointless. Clear it if so. | ||||||
3516 | if (LI.isCleanup() != CleanupFlag) { | ||||||
3517 | assert(!CleanupFlag && "Adding a cleanup, not removing one?!")(static_cast <bool> (!CleanupFlag && "Adding a cleanup, not removing one?!" ) ? void (0) : __assert_fail ("!CleanupFlag && \"Adding a cleanup, not removing one?!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3517, __extension__ __PRETTY_FUNCTION__)); | ||||||
3518 | LI.setCleanup(CleanupFlag); | ||||||
3519 | return &LI; | ||||||
3520 | } | ||||||
3521 | |||||||
3522 | return nullptr; | ||||||
3523 | } | ||||||
3524 | |||||||
3525 | Value * | ||||||
3526 | InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating(FreezeInst &OrigFI) { | ||||||
3527 | // Try to push freeze through instructions that propagate but don't produce | ||||||
3528 | // poison as far as possible. If an operand of freeze follows three | ||||||
3529 | // conditions 1) one-use, 2) does not produce poison, and 3) has all but one | ||||||
3530 | // guaranteed-non-poison operands then push the freeze through to the one | ||||||
3531 | // operand that is not guaranteed non-poison. The actual transform is as | ||||||
3532 | // follows. | ||||||
3533 | // Op1 = ... ; Op1 can be posion | ||||||
3534 | // Op0 = Inst(Op1, NonPoisonOps...) ; Op0 has only one use and only have | ||||||
3535 | // ; single guaranteed-non-poison operands | ||||||
3536 | // ... = Freeze(Op0) | ||||||
3537 | // => | ||||||
3538 | // Op1 = ... | ||||||
3539 | // Op1.fr = Freeze(Op1) | ||||||
3540 | // ... = Inst(Op1.fr, NonPoisonOps...) | ||||||
3541 | auto *OrigOp = OrigFI.getOperand(0); | ||||||
3542 | auto *OrigOpInst = dyn_cast<Instruction>(OrigOp); | ||||||
3543 | |||||||
3544 | // While we could change the other users of OrigOp to use freeze(OrigOp), that | ||||||
3545 | // potentially reduces their optimization potential, so let's only do this iff | ||||||
3546 | // the OrigOp is only used by the freeze. | ||||||
3547 | if (!OrigOpInst || !OrigOpInst->hasOneUse() || isa<PHINode>(OrigOp) || | ||||||
3548 | canCreateUndefOrPoison(dyn_cast<Operator>(OrigOp))) | ||||||
3549 | return nullptr; | ||||||
3550 | |||||||
3551 | // If operand is guaranteed not to be poison, there is no need to add freeze | ||||||
3552 | // to the operand. So we first find the operand that is not guaranteed to be | ||||||
3553 | // poison. | ||||||
3554 | Use *MaybePoisonOperand = nullptr; | ||||||
3555 | for (Use &U : OrigOpInst->operands()) { | ||||||
3556 | if (isGuaranteedNotToBeUndefOrPoison(U.get())) | ||||||
3557 | continue; | ||||||
3558 | if (!MaybePoisonOperand) | ||||||
3559 | MaybePoisonOperand = &U; | ||||||
3560 | else | ||||||
3561 | return nullptr; | ||||||
3562 | } | ||||||
3563 | |||||||
3564 | // If all operands are guaranteed to be non-poison, we can drop freeze. | ||||||
3565 | if (!MaybePoisonOperand) | ||||||
3566 | return OrigOp; | ||||||
3567 | |||||||
3568 | auto *FrozenMaybePoisonOperand = new FreezeInst( | ||||||
3569 | MaybePoisonOperand->get(), MaybePoisonOperand->get()->getName() + ".fr"); | ||||||
3570 | |||||||
3571 | replaceUse(*MaybePoisonOperand, FrozenMaybePoisonOperand); | ||||||
3572 | FrozenMaybePoisonOperand->insertBefore(OrigOpInst); | ||||||
3573 | return OrigOp; | ||||||
3574 | } | ||||||
3575 | |||||||
3576 | bool InstCombinerImpl::freezeDominatedUses(FreezeInst &FI) { | ||||||
3577 | Value *Op = FI.getOperand(0); | ||||||
3578 | |||||||
3579 | if (isa<Constant>(Op)) | ||||||
3580 | return false; | ||||||
3581 | |||||||
3582 | bool Changed = false; | ||||||
3583 | Op->replaceUsesWithIf(&FI, [&](Use &U) -> bool { | ||||||
3584 | bool Dominates = DT.dominates(&FI, U); | ||||||
3585 | Changed |= Dominates; | ||||||
3586 | return Dominates; | ||||||
3587 | }); | ||||||
3588 | |||||||
3589 | return Changed; | ||||||
3590 | } | ||||||
3591 | |||||||
3592 | Instruction *InstCombinerImpl::visitFreeze(FreezeInst &I) { | ||||||
3593 | Value *Op0 = I.getOperand(0); | ||||||
3594 | |||||||
3595 | if (Value *V = SimplifyFreezeInst(Op0, SQ.getWithInstruction(&I))) | ||||||
| |||||||
3596 | return replaceInstUsesWith(I, V); | ||||||
3597 | |||||||
3598 | // freeze (phi const, x) --> phi const, (freeze x) | ||||||
3599 | if (auto *PN = dyn_cast<PHINode>(Op0)) { | ||||||
3600 | if (Instruction *NV = foldOpIntoPhi(I, PN)) | ||||||
3601 | return NV; | ||||||
3602 | } | ||||||
3603 | |||||||
3604 | if (Value *NI
| ||||||
3605 | return replaceInstUsesWith(I, NI); | ||||||
3606 | |||||||
3607 | if (match(Op0, m_Undef())) { | ||||||
3608 | // If I is freeze(undef), see its uses and fold it to the best constant. | ||||||
3609 | // - or: pick -1 | ||||||
3610 | // - select's condition: pick the value that leads to choosing a constant | ||||||
3611 | // - other ops: pick 0 | ||||||
3612 | Constant *BestValue = nullptr; | ||||||
3613 | Constant *NullValue = Constant::getNullValue(I.getType()); | ||||||
3614 | for (const auto *U : I.users()) { | ||||||
3615 | Constant *C = NullValue; | ||||||
3616 | |||||||
3617 | if (match(U, m_Or(m_Value(), m_Value()))) | ||||||
3618 | C = Constant::getAllOnesValue(I.getType()); | ||||||
3619 | else if (const auto *SI = dyn_cast<SelectInst>(U)) { | ||||||
3620 | if (SI->getCondition() == &I) { | ||||||
3621 | APInt CondVal(1, isa<Constant>(SI->getFalseValue()) ? 0 : 1); | ||||||
3622 | C = Constant::getIntegerValue(I.getType(), CondVal); | ||||||
3623 | } | ||||||
3624 | } | ||||||
3625 | |||||||
3626 | if (!BestValue) | ||||||
3627 | BestValue = C; | ||||||
3628 | else if (BestValue != C) | ||||||
3629 | BestValue = NullValue; | ||||||
3630 | } | ||||||
3631 | |||||||
3632 | return replaceInstUsesWith(I, BestValue); | ||||||
3633 | } | ||||||
3634 | |||||||
3635 | // Replace all dominated uses of Op to freeze(Op). | ||||||
3636 | if (freezeDominatedUses(I)) | ||||||
3637 | return &I; | ||||||
3638 | |||||||
3639 | return nullptr; | ||||||
3640 | } | ||||||
3641 | |||||||
3642 | /// Try to move the specified instruction from its current block into the | ||||||
3643 | /// beginning of DestBlock, which can only happen if it's safe to move the | ||||||
3644 | /// instruction past all of the instructions between it and the end of its | ||||||
3645 | /// block. | ||||||
3646 | static bool TryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) { | ||||||
3647 | assert(I->getUniqueUndroppableUser() && "Invariants didn't hold!")(static_cast <bool> (I->getUniqueUndroppableUser() && "Invariants didn't hold!") ? void (0) : __assert_fail ("I->getUniqueUndroppableUser() && \"Invariants didn't hold!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3647, __extension__ __PRETTY_FUNCTION__)); | ||||||
3648 | BasicBlock *SrcBlock = I->getParent(); | ||||||
3649 | |||||||
3650 | // Cannot move control-flow-involving, volatile loads, vaarg, etc. | ||||||
3651 | if (isa<PHINode>(I) || I->isEHPad() || I->mayHaveSideEffects() || | ||||||
3652 | I->isTerminator()) | ||||||
3653 | return false; | ||||||
3654 | |||||||
3655 | // Do not sink static or dynamic alloca instructions. Static allocas must | ||||||
3656 | // remain in the entry block, and dynamic allocas must not be sunk in between | ||||||
3657 | // a stacksave / stackrestore pair, which would incorrectly shorten its | ||||||
3658 | // lifetime. | ||||||
3659 | if (isa<AllocaInst>(I)) | ||||||
3660 | return false; | ||||||
3661 | |||||||
3662 | // Do not sink into catchswitch blocks. | ||||||
3663 | if (isa<CatchSwitchInst>(DestBlock->getTerminator())) | ||||||
3664 | return false; | ||||||
3665 | |||||||
3666 | // Do not sink convergent call instructions. | ||||||
3667 | if (auto *CI = dyn_cast<CallInst>(I)) { | ||||||
3668 | if (CI->isConvergent()) | ||||||
3669 | return false; | ||||||
3670 | } | ||||||
3671 | // We can only sink load instructions if there is nothing between the load and | ||||||
3672 | // the end of block that could change the value. | ||||||
3673 | if (I->mayReadFromMemory()) { | ||||||
3674 | // We don't want to do any sophisticated alias analysis, so we only check | ||||||
3675 | // the instructions after I in I's parent block if we try to sink to its | ||||||
3676 | // successor block. | ||||||
3677 | if (DestBlock->getUniquePredecessor() != I->getParent()) | ||||||
3678 | return false; | ||||||
3679 | for (BasicBlock::iterator Scan = I->getIterator(), | ||||||
3680 | E = I->getParent()->end(); | ||||||
3681 | Scan != E; ++Scan) | ||||||
3682 | if (Scan->mayWriteToMemory()) | ||||||
3683 | return false; | ||||||
3684 | } | ||||||
3685 | |||||||
3686 | I->dropDroppableUses([DestBlock](const Use *U) { | ||||||
3687 | if (auto *I = dyn_cast<Instruction>(U->getUser())) | ||||||
3688 | return I->getParent() != DestBlock; | ||||||
3689 | return true; | ||||||
3690 | }); | ||||||
3691 | /// FIXME: We could remove droppable uses that are not dominated by | ||||||
3692 | /// the new position. | ||||||
3693 | |||||||
3694 | BasicBlock::iterator InsertPos = DestBlock->getFirstInsertionPt(); | ||||||
3695 | I->moveBefore(&*InsertPos); | ||||||
3696 | ++NumSunkInst; | ||||||
3697 | |||||||
3698 | // Also sink all related debug uses from the source basic block. Otherwise we | ||||||
3699 | // get debug use before the def. Attempt to salvage debug uses first, to | ||||||
3700 | // maximise the range variables have location for. If we cannot salvage, then | ||||||
3701 | // mark the location undef: we know it was supposed to receive a new location | ||||||
3702 | // here, but that computation has been sunk. | ||||||
3703 | SmallVector<DbgVariableIntrinsic *, 2> DbgUsers; | ||||||
3704 | findDbgUsers(DbgUsers, I); | ||||||
3705 | // Process the sinking DbgUsers in reverse order, as we only want to clone the | ||||||
3706 | // last appearing debug intrinsic for each given variable. | ||||||
3707 | SmallVector<DbgVariableIntrinsic *, 2> DbgUsersToSink; | ||||||
3708 | for (DbgVariableIntrinsic *DVI : DbgUsers) | ||||||
3709 | if (DVI->getParent() == SrcBlock) | ||||||
3710 | DbgUsersToSink.push_back(DVI); | ||||||
3711 | llvm::sort(DbgUsersToSink, | ||||||
3712 | [](auto *A, auto *B) { return B->comesBefore(A); }); | ||||||
3713 | |||||||
3714 | SmallVector<DbgVariableIntrinsic *, 2> DIIClones; | ||||||
3715 | SmallSet<DebugVariable, 4> SunkVariables; | ||||||
3716 | for (auto User : DbgUsersToSink) { | ||||||
3717 | // A dbg.declare instruction should not be cloned, since there can only be | ||||||
3718 | // one per variable fragment. It should be left in the original place | ||||||
3719 | // because the sunk instruction is not an alloca (otherwise we could not be | ||||||
3720 | // here). | ||||||
3721 | if (isa<DbgDeclareInst>(User)) | ||||||
3722 | continue; | ||||||
3723 | |||||||
3724 | DebugVariable DbgUserVariable = | ||||||
3725 | DebugVariable(User->getVariable(), User->getExpression(), | ||||||
3726 | User->getDebugLoc()->getInlinedAt()); | ||||||
3727 | |||||||
3728 | if (!SunkVariables.insert(DbgUserVariable).second) | ||||||
3729 | continue; | ||||||
3730 | |||||||
3731 | DIIClones.emplace_back(cast<DbgVariableIntrinsic>(User->clone())); | ||||||
3732 | if (isa<DbgDeclareInst>(User) && isa<CastInst>(I)) | ||||||
3733 | DIIClones.back()->replaceVariableLocationOp(I, I->getOperand(0)); | ||||||
3734 | LLVM_DEBUG(dbgs() << "CLONE: " << *DIIClones.back() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "CLONE: " << *DIIClones .back() << '\n'; } } while (false); | ||||||
3735 | } | ||||||
3736 | |||||||
3737 | // Perform salvaging without the clones, then sink the clones. | ||||||
3738 | if (!DIIClones.empty()) { | ||||||
3739 | salvageDebugInfoForDbgValues(*I, DbgUsers); | ||||||
3740 | // The clones are in reverse order of original appearance, reverse again to | ||||||
3741 | // maintain the original order. | ||||||
3742 | for (auto &DIIClone : llvm::reverse(DIIClones)) { | ||||||
3743 | DIIClone->insertBefore(&*InsertPos); | ||||||
3744 | LLVM_DEBUG(dbgs() << "SINK: " << *DIIClone << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "SINK: " << *DIIClone << '\n'; } } while (false); | ||||||
3745 | } | ||||||
3746 | } | ||||||
3747 | |||||||
3748 | return true; | ||||||
3749 | } | ||||||
3750 | |||||||
3751 | bool InstCombinerImpl::run() { | ||||||
3752 | while (!Worklist.isEmpty()) { | ||||||
3753 | // Walk deferred instructions in reverse order, and push them to the | ||||||
3754 | // worklist, which means they'll end up popped from the worklist in-order. | ||||||
3755 | while (Instruction *I = Worklist.popDeferred()) { | ||||||
3756 | // Check to see if we can DCE the instruction. We do this already here to | ||||||
3757 | // reduce the number of uses and thus allow other folds to trigger. | ||||||
3758 | // Note that eraseInstFromFunction() may push additional instructions on | ||||||
3759 | // the deferred worklist, so this will DCE whole instruction chains. | ||||||
3760 | if (isInstructionTriviallyDead(I, &TLI)) { | ||||||
3761 | eraseInstFromFunction(*I); | ||||||
3762 | ++NumDeadInst; | ||||||
3763 | continue; | ||||||
3764 | } | ||||||
3765 | |||||||
3766 | Worklist.push(I); | ||||||
3767 | } | ||||||
3768 | |||||||
3769 | Instruction *I = Worklist.removeOne(); | ||||||
3770 | if (I == nullptr) continue; // skip null values. | ||||||
3771 | |||||||
3772 | // Check to see if we can DCE the instruction. | ||||||
3773 | if (isInstructionTriviallyDead(I, &TLI)) { | ||||||
3774 | eraseInstFromFunction(*I); | ||||||
3775 | ++NumDeadInst; | ||||||
3776 | continue; | ||||||
3777 | } | ||||||
3778 | |||||||
3779 | if (!DebugCounter::shouldExecute(VisitCounter)) | ||||||
3780 | continue; | ||||||
3781 | |||||||
3782 | // Instruction isn't dead, see if we can constant propagate it. | ||||||
3783 | if (!I->use_empty() && | ||||||
3784 | (I->getNumOperands() == 0 || isa<Constant>(I->getOperand(0)))) { | ||||||
3785 | if (Constant *C = ConstantFoldInstruction(I, DL, &TLI)) { | ||||||
3786 | LLVM_DEBUG(dbgs() << "IC: ConstFold to: " << *C << " from: " << *Ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: ConstFold to: " << *C << " from: " << *I << '\n'; } } while ( false) | ||||||
3787 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: ConstFold to: " << *C << " from: " << *I << '\n'; } } while ( false); | ||||||
3788 | |||||||
3789 | // Add operands to the worklist. | ||||||
3790 | replaceInstUsesWith(*I, C); | ||||||
3791 | ++NumConstProp; | ||||||
3792 | if (isInstructionTriviallyDead(I, &TLI)) | ||||||
3793 | eraseInstFromFunction(*I); | ||||||
3794 | MadeIRChange = true; | ||||||
3795 | continue; | ||||||
3796 | } | ||||||
3797 | } | ||||||
3798 | |||||||
3799 | // See if we can trivially sink this instruction to its user if we can | ||||||
3800 | // prove that the successor is not executed more frequently than our block. | ||||||
3801 | // Return the UserBlock if successful. | ||||||
3802 | auto getOptionalSinkBlockForInst = | ||||||
3803 | [this](Instruction *I) -> Optional<BasicBlock *> { | ||||||
3804 | if (!EnableCodeSinking) | ||||||
3805 | return None; | ||||||
3806 | auto *UserInst = cast_or_null<Instruction>(I->getUniqueUndroppableUser()); | ||||||
3807 | if (!UserInst) | ||||||
3808 | return None; | ||||||
3809 | |||||||
3810 | BasicBlock *BB = I->getParent(); | ||||||
3811 | BasicBlock *UserParent = nullptr; | ||||||
3812 | |||||||
3813 | // Special handling for Phi nodes - get the block the use occurs in. | ||||||
3814 | if (PHINode *PN = dyn_cast<PHINode>(UserInst)) { | ||||||
3815 | for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) { | ||||||
3816 | if (PN->getIncomingValue(i) == I) { | ||||||
3817 | // Bail out if we have uses in different blocks. We don't do any | ||||||
3818 | // sophisticated analysis (i.e finding NearestCommonDominator of these | ||||||
3819 | // use blocks). | ||||||
3820 | if (UserParent && UserParent != PN->getIncomingBlock(i)) | ||||||
3821 | return None; | ||||||
3822 | UserParent = PN->getIncomingBlock(i); | ||||||
3823 | } | ||||||
3824 | } | ||||||
3825 | assert(UserParent && "expected to find user block!")(static_cast <bool> (UserParent && "expected to find user block!" ) ? void (0) : __assert_fail ("UserParent && \"expected to find user block!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3825, __extension__ __PRETTY_FUNCTION__)); | ||||||
3826 | } else | ||||||
3827 | UserParent = UserInst->getParent(); | ||||||
3828 | |||||||
3829 | // Try sinking to another block. If that block is unreachable, then do | ||||||
3830 | // not bother. SimplifyCFG should handle it. | ||||||
3831 | if (UserParent == BB || !DT.isReachableFromEntry(UserParent)) | ||||||
3832 | return None; | ||||||
3833 | |||||||
3834 | auto *Term = UserParent->getTerminator(); | ||||||
3835 | // See if the user is one of our successors that has only one | ||||||
3836 | // predecessor, so that we don't have to split the critical edge. | ||||||
3837 | // Another option where we can sink is a block that ends with a | ||||||
3838 | // terminator that does not pass control to other block (such as | ||||||
3839 | // return or unreachable). In this case: | ||||||
3840 | // - I dominates the User (by SSA form); | ||||||
3841 | // - the User will be executed at most once. | ||||||
3842 | // So sinking I down to User is always profitable or neutral. | ||||||
3843 | if (UserParent->getUniquePredecessor() == BB || | ||||||
3844 | (isa<ReturnInst>(Term) || isa<UnreachableInst>(Term))) { | ||||||
3845 | assert(DT.dominates(BB, UserParent) && "Dominance relation broken?")(static_cast <bool> (DT.dominates(BB, UserParent) && "Dominance relation broken?") ? void (0) : __assert_fail ("DT.dominates(BB, UserParent) && \"Dominance relation broken?\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3845, __extension__ __PRETTY_FUNCTION__)); | ||||||
3846 | return UserParent; | ||||||
3847 | } | ||||||
3848 | return None; | ||||||
3849 | }; | ||||||
3850 | |||||||
3851 | auto OptBB = getOptionalSinkBlockForInst(I); | ||||||
3852 | if (OptBB) { | ||||||
3853 | auto *UserParent = *OptBB; | ||||||
3854 | // Okay, the CFG is simple enough, try to sink this instruction. | ||||||
3855 | if (TryToSinkInstruction(I, UserParent)) { | ||||||
3856 | LLVM_DEBUG(dbgs() << "IC: Sink: " << *I << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: Sink: " << *I << '\n'; } } while (false); | ||||||
3857 | MadeIRChange = true; | ||||||
3858 | // We'll add uses of the sunk instruction below, but since | ||||||
3859 | // sinking can expose opportunities for it's *operands* add | ||||||
3860 | // them to the worklist | ||||||
3861 | for (Use &U : I->operands()) | ||||||
3862 | if (Instruction *OpI = dyn_cast<Instruction>(U.get())) | ||||||
3863 | Worklist.push(OpI); | ||||||
3864 | } | ||||||
3865 | } | ||||||
3866 | |||||||
3867 | // Now that we have an instruction, try combining it to simplify it. | ||||||
3868 | Builder.SetInsertPoint(I); | ||||||
3869 | Builder.CollectMetadataToCopy( | ||||||
3870 | I, {LLVMContext::MD_dbg, LLVMContext::MD_annotation}); | ||||||
3871 | |||||||
3872 | #ifndef NDEBUG | ||||||
3873 | std::string OrigI; | ||||||
3874 | #endif | ||||||
3875 | LLVM_DEBUG(raw_string_ostream SS(OrigI); I->print(SS); OrigI = SS.str();)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { raw_string_ostream SS(OrigI); I->print( SS); OrigI = SS.str();; } } while (false); | ||||||
3876 | LLVM_DEBUG(dbgs() << "IC: Visiting: " << OrigI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: Visiting: " << OrigI << '\n'; } } while (false); | ||||||
3877 | |||||||
3878 | if (Instruction *Result = visit(*I)) { | ||||||
3879 | ++NumCombined; | ||||||
3880 | // Should we replace the old instruction with a new one? | ||||||
3881 | if (Result != I) { | ||||||
3882 | LLVM_DEBUG(dbgs() << "IC: Old = " << *I << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: Old = " << *I << '\n' << " New = " << *Result << '\n'; } } while (false) | ||||||
3883 | << " New = " << *Result << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: Old = " << *I << '\n' << " New = " << *Result << '\n'; } } while (false); | ||||||
3884 | |||||||
3885 | Result->copyMetadata(*I, | ||||||
3886 | {LLVMContext::MD_dbg, LLVMContext::MD_annotation}); | ||||||
3887 | // Everything uses the new instruction now. | ||||||
3888 | I->replaceAllUsesWith(Result); | ||||||
3889 | |||||||
3890 | // Move the name to the new instruction first. | ||||||
3891 | Result->takeName(I); | ||||||
3892 | |||||||
3893 | // Insert the new instruction into the basic block... | ||||||
3894 | BasicBlock *InstParent = I->getParent(); | ||||||
3895 | BasicBlock::iterator InsertPos = I->getIterator(); | ||||||
3896 | |||||||
3897 | // Are we replace a PHI with something that isn't a PHI, or vice versa? | ||||||
3898 | if (isa<PHINode>(Result) != isa<PHINode>(I)) { | ||||||
3899 | // We need to fix up the insertion point. | ||||||
3900 | if (isa<PHINode>(I)) // PHI -> Non-PHI | ||||||
3901 | InsertPos = InstParent->getFirstInsertionPt(); | ||||||
3902 | else // Non-PHI -> PHI | ||||||
3903 | InsertPos = InstParent->getFirstNonPHI()->getIterator(); | ||||||
3904 | } | ||||||
3905 | |||||||
3906 | InstParent->getInstList().insert(InsertPos, Result); | ||||||
3907 | |||||||
3908 | // Push the new instruction and any users onto the worklist. | ||||||
3909 | Worklist.pushUsersToWorkList(*Result); | ||||||
3910 | Worklist.push(Result); | ||||||
3911 | |||||||
3912 | eraseInstFromFunction(*I); | ||||||
3913 | } else { | ||||||
3914 | LLVM_DEBUG(dbgs() << "IC: Mod = " << OrigI << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: Mod = " << OrigI << '\n' << " New = " << *I << '\n' ; } } while (false) | ||||||
3915 | << " New = " << *I << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: Mod = " << OrigI << '\n' << " New = " << *I << '\n' ; } } while (false); | ||||||
3916 | |||||||
3917 | // If the instruction was modified, it's possible that it is now dead. | ||||||
3918 | // if so, remove it. | ||||||
3919 | if (isInstructionTriviallyDead(I, &TLI)) { | ||||||
3920 | eraseInstFromFunction(*I); | ||||||
3921 | } else { | ||||||
3922 | Worklist.pushUsersToWorkList(*I); | ||||||
3923 | Worklist.push(I); | ||||||
3924 | } | ||||||
3925 | } | ||||||
3926 | MadeIRChange = true; | ||||||
3927 | } | ||||||
3928 | } | ||||||
3929 | |||||||
3930 | Worklist.zap(); | ||||||
3931 | return MadeIRChange; | ||||||
3932 | } | ||||||
3933 | |||||||
3934 | // Track the scopes used by !alias.scope and !noalias. In a function, a | ||||||
3935 | // @llvm.experimental.noalias.scope.decl is only useful if that scope is used | ||||||
3936 | // by both sets. If not, the declaration of the scope can be safely omitted. | ||||||
3937 | // The MDNode of the scope can be omitted as well for the instructions that are | ||||||
3938 | // part of this function. We do not do that at this point, as this might become | ||||||
3939 | // too time consuming to do. | ||||||
3940 | class AliasScopeTracker { | ||||||
3941 | SmallPtrSet<const MDNode *, 8> UsedAliasScopesAndLists; | ||||||
3942 | SmallPtrSet<const MDNode *, 8> UsedNoAliasScopesAndLists; | ||||||
3943 | |||||||
3944 | public: | ||||||
3945 | void analyse(Instruction *I) { | ||||||
3946 | // This seems to be faster than checking 'mayReadOrWriteMemory()'. | ||||||
3947 | if (!I->hasMetadataOtherThanDebugLoc()) | ||||||
3948 | return; | ||||||
3949 | |||||||
3950 | auto Track = [](Metadata *ScopeList, auto &Container) { | ||||||
3951 | const auto *MDScopeList = dyn_cast_or_null<MDNode>(ScopeList); | ||||||
3952 | if (!MDScopeList || !Container.insert(MDScopeList).second) | ||||||
3953 | return; | ||||||
3954 | for (auto &MDOperand : MDScopeList->operands()) | ||||||
3955 | if (auto *MDScope = dyn_cast<MDNode>(MDOperand)) | ||||||
3956 | Container.insert(MDScope); | ||||||
3957 | }; | ||||||
3958 | |||||||
3959 | Track(I->getMetadata(LLVMContext::MD_alias_scope), UsedAliasScopesAndLists); | ||||||
3960 | Track(I->getMetadata(LLVMContext::MD_noalias), UsedNoAliasScopesAndLists); | ||||||
3961 | } | ||||||
3962 | |||||||
3963 | bool isNoAliasScopeDeclDead(Instruction *Inst) { | ||||||
3964 | NoAliasScopeDeclInst *Decl = dyn_cast<NoAliasScopeDeclInst>(Inst); | ||||||
3965 | if (!Decl) | ||||||
3966 | return false; | ||||||
3967 | |||||||
3968 | assert(Decl->use_empty() &&(static_cast <bool> (Decl->use_empty() && "llvm.experimental.noalias.scope.decl in use ?" ) ? void (0) : __assert_fail ("Decl->use_empty() && \"llvm.experimental.noalias.scope.decl in use ?\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3969, __extension__ __PRETTY_FUNCTION__)) | ||||||
3969 | "llvm.experimental.noalias.scope.decl in use ?")(static_cast <bool> (Decl->use_empty() && "llvm.experimental.noalias.scope.decl in use ?" ) ? void (0) : __assert_fail ("Decl->use_empty() && \"llvm.experimental.noalias.scope.decl in use ?\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3969, __extension__ __PRETTY_FUNCTION__)); | ||||||
3970 | const MDNode *MDSL = Decl->getScopeList(); | ||||||
3971 | assert(MDSL->getNumOperands() == 1 &&(static_cast <bool> (MDSL->getNumOperands() == 1 && "llvm.experimental.noalias.scope should refer to a single scope" ) ? void (0) : __assert_fail ("MDSL->getNumOperands() == 1 && \"llvm.experimental.noalias.scope should refer to a single scope\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3972, __extension__ __PRETTY_FUNCTION__)) | ||||||
3972 | "llvm.experimental.noalias.scope should refer to a single scope")(static_cast <bool> (MDSL->getNumOperands() == 1 && "llvm.experimental.noalias.scope should refer to a single scope" ) ? void (0) : __assert_fail ("MDSL->getNumOperands() == 1 && \"llvm.experimental.noalias.scope should refer to a single scope\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp" , 3972, __extension__ __PRETTY_FUNCTION__)); | ||||||
3973 | auto &MDOperand = MDSL->getOperand(0); | ||||||
3974 | if (auto *MD = dyn_cast<MDNode>(MDOperand)) | ||||||
3975 | return !UsedAliasScopesAndLists.contains(MD) || | ||||||
3976 | !UsedNoAliasScopesAndLists.contains(MD); | ||||||
3977 | |||||||
3978 | // Not an MDNode ? throw away. | ||||||
3979 | return true; | ||||||
3980 | } | ||||||
3981 | }; | ||||||
3982 | |||||||
3983 | /// Populate the IC worklist from a function, by walking it in depth-first | ||||||
3984 | /// order and adding all reachable code to the worklist. | ||||||
3985 | /// | ||||||
3986 | /// This has a couple of tricks to make the code faster and more powerful. In | ||||||
3987 | /// particular, we constant fold and DCE instructions as we go, to avoid adding | ||||||
3988 | /// them to the worklist (this significantly speeds up instcombine on code where | ||||||
3989 | /// many instructions are dead or constant). Additionally, if we find a branch | ||||||
3990 | /// whose condition is a known constant, we only visit the reachable successors. | ||||||
3991 | static bool prepareICWorklistFromFunction(Function &F, const DataLayout &DL, | ||||||
3992 | const TargetLibraryInfo *TLI, | ||||||
3993 | InstructionWorklist &ICWorklist) { | ||||||
3994 | bool MadeIRChange = false; | ||||||
3995 | SmallPtrSet<BasicBlock *, 32> Visited; | ||||||
3996 | SmallVector<BasicBlock*, 256> Worklist; | ||||||
3997 | Worklist.push_back(&F.front()); | ||||||
3998 | |||||||
3999 | SmallVector<Instruction *, 128> InstrsForInstructionWorklist; | ||||||
4000 | DenseMap<Constant *, Constant *> FoldedConstants; | ||||||
4001 | AliasScopeTracker SeenAliasScopes; | ||||||
4002 | |||||||
4003 | do { | ||||||
4004 | BasicBlock *BB = Worklist.pop_back_val(); | ||||||
4005 | |||||||
4006 | // We have now visited this block! If we've already been here, ignore it. | ||||||
4007 | if (!Visited.insert(BB).second) | ||||||
4008 | continue; | ||||||
4009 | |||||||
4010 | for (Instruction &Inst : llvm::make_early_inc_range(*BB)) { | ||||||
4011 | // ConstantProp instruction if trivially constant. | ||||||
4012 | if (!Inst.use_empty() && | ||||||
4013 | (Inst.getNumOperands() == 0 || isa<Constant>(Inst.getOperand(0)))) | ||||||
4014 | if (Constant *C = ConstantFoldInstruction(&Inst, DL, TLI)) { | ||||||
4015 | LLVM_DEBUG(dbgs() << "IC: ConstFold to: " << *C << " from: " << Instdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: ConstFold to: " << *C << " from: " << Inst << '\n'; } } while (false) | ||||||
4016 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: ConstFold to: " << *C << " from: " << Inst << '\n'; } } while (false); | ||||||
4017 | Inst.replaceAllUsesWith(C); | ||||||
4018 | ++NumConstProp; | ||||||
4019 | if (isInstructionTriviallyDead(&Inst, TLI)) | ||||||
4020 | Inst.eraseFromParent(); | ||||||
4021 | MadeIRChange = true; | ||||||
4022 | continue; | ||||||
4023 | } | ||||||
4024 | |||||||
4025 | // See if we can constant fold its operands. | ||||||
4026 | for (Use &U : Inst.operands()) { | ||||||
4027 | if (!isa<ConstantVector>(U) && !isa<ConstantExpr>(U)) | ||||||
4028 | continue; | ||||||
4029 | |||||||
4030 | auto *C = cast<Constant>(U); | ||||||
4031 | Constant *&FoldRes = FoldedConstants[C]; | ||||||
4032 | if (!FoldRes) | ||||||
4033 | FoldRes = ConstantFoldConstant(C, DL, TLI); | ||||||
4034 | |||||||
4035 | if (FoldRes != C) { | ||||||
4036 | LLVM_DEBUG(dbgs() << "IC: ConstFold operand of: " << Instdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: ConstFold operand of: " << Inst << "\n Old = " << *C << "\n New = " << *FoldRes << '\n'; } } while (false) | ||||||
4037 | << "\n Old = " << *Cdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: ConstFold operand of: " << Inst << "\n Old = " << *C << "\n New = " << *FoldRes << '\n'; } } while (false) | ||||||
4038 | << "\n New = " << *FoldRes << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: ConstFold operand of: " << Inst << "\n Old = " << *C << "\n New = " << *FoldRes << '\n'; } } while (false); | ||||||
4039 | U = FoldRes; | ||||||
4040 | MadeIRChange = true; | ||||||
4041 | } | ||||||
4042 | } | ||||||
4043 | |||||||
4044 | // Skip processing debug and pseudo intrinsics in InstCombine. Processing | ||||||
4045 | // these call instructions consumes non-trivial amount of time and | ||||||
4046 | // provides no value for the optimization. | ||||||
4047 | if (!Inst.isDebugOrPseudoInst()) { | ||||||
4048 | InstrsForInstructionWorklist.push_back(&Inst); | ||||||
4049 | SeenAliasScopes.analyse(&Inst); | ||||||
4050 | } | ||||||
4051 | } | ||||||
4052 | |||||||
4053 | // Recursively visit successors. If this is a branch or switch on a | ||||||
4054 | // constant, only visit the reachable successor. | ||||||
4055 | Instruction *TI = BB->getTerminator(); | ||||||
4056 | if (BranchInst *BI = dyn_cast<BranchInst>(TI)) { | ||||||
4057 | if (BI->isConditional() && isa<ConstantInt>(BI->getCondition())) { | ||||||
4058 | bool CondVal = cast<ConstantInt>(BI->getCondition())->getZExtValue(); | ||||||
4059 | BasicBlock *ReachableBB = BI->getSuccessor(!CondVal); | ||||||
4060 | Worklist.push_back(ReachableBB); | ||||||
4061 | continue; | ||||||
4062 | } | ||||||
4063 | } else if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) { | ||||||
4064 | if (ConstantInt *Cond = dyn_cast<ConstantInt>(SI->getCondition())) { | ||||||
4065 | Worklist.push_back(SI->findCaseValue(Cond)->getCaseSuccessor()); | ||||||
4066 | continue; | ||||||
4067 | } | ||||||
4068 | } | ||||||
4069 | |||||||
4070 | append_range(Worklist, successors(TI)); | ||||||
4071 | } while (!Worklist.empty()); | ||||||
4072 | |||||||
4073 | // Remove instructions inside unreachable blocks. This prevents the | ||||||
4074 | // instcombine code from having to deal with some bad special cases, and | ||||||
4075 | // reduces use counts of instructions. | ||||||
4076 | for (BasicBlock &BB : F) { | ||||||
4077 | if (Visited.count(&BB)) | ||||||
4078 | continue; | ||||||
4079 | |||||||
4080 | unsigned NumDeadInstInBB; | ||||||
4081 | unsigned NumDeadDbgInstInBB; | ||||||
4082 | std::tie(NumDeadInstInBB, NumDeadDbgInstInBB) = | ||||||
4083 | removeAllNonTerminatorAndEHPadInstructions(&BB); | ||||||
4084 | |||||||
4085 | MadeIRChange |= NumDeadInstInBB + NumDeadDbgInstInBB > 0; | ||||||
4086 | NumDeadInst += NumDeadInstInBB; | ||||||
4087 | } | ||||||
4088 | |||||||
4089 | // Once we've found all of the instructions to add to instcombine's worklist, | ||||||
4090 | // add them in reverse order. This way instcombine will visit from the top | ||||||
4091 | // of the function down. This jives well with the way that it adds all uses | ||||||
4092 | // of instructions to the worklist after doing a transformation, thus avoiding | ||||||
4093 | // some N^2 behavior in pathological cases. | ||||||
4094 | ICWorklist.reserve(InstrsForInstructionWorklist.size()); | ||||||
4095 | for (Instruction *Inst : reverse(InstrsForInstructionWorklist)) { | ||||||
4096 | // DCE instruction if trivially dead. As we iterate in reverse program | ||||||
4097 | // order here, we will clean up whole chains of dead instructions. | ||||||
4098 | if (isInstructionTriviallyDead(Inst, TLI) || | ||||||
4099 | SeenAliasScopes.isNoAliasScopeDeclDead(Inst)) { | ||||||
4100 | ++NumDeadInst; | ||||||
4101 | LLVM_DEBUG(dbgs() << "IC: DCE: " << *Inst << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: DCE: " << *Inst << '\n'; } } while (false); | ||||||
4102 | salvageDebugInfo(*Inst); | ||||||
4103 | Inst->eraseFromParent(); | ||||||
4104 | MadeIRChange = true; | ||||||
4105 | continue; | ||||||
4106 | } | ||||||
4107 | |||||||
4108 | ICWorklist.push(Inst); | ||||||
4109 | } | ||||||
4110 | |||||||
4111 | return MadeIRChange; | ||||||
4112 | } | ||||||
4113 | |||||||
4114 | static bool combineInstructionsOverFunction( | ||||||
4115 | Function &F, InstructionWorklist &Worklist, AliasAnalysis *AA, | ||||||
4116 | AssumptionCache &AC, TargetLibraryInfo &TLI, TargetTransformInfo &TTI, | ||||||
4117 | DominatorTree &DT, OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI, | ||||||
4118 | ProfileSummaryInfo *PSI, unsigned MaxIterations, LoopInfo *LI) { | ||||||
4119 | auto &DL = F.getParent()->getDataLayout(); | ||||||
4120 | MaxIterations = std::min(MaxIterations, LimitMaxIterations.getValue()); | ||||||
4121 | |||||||
4122 | /// Builder - This is an IRBuilder that automatically inserts new | ||||||
4123 | /// instructions into the worklist when they are created. | ||||||
4124 | IRBuilder<TargetFolder, IRBuilderCallbackInserter> Builder( | ||||||
4125 | F.getContext(), TargetFolder(DL), | ||||||
4126 | IRBuilderCallbackInserter([&Worklist, &AC](Instruction *I) { | ||||||
4127 | Worklist.add(I); | ||||||
4128 | if (auto *Assume = dyn_cast<AssumeInst>(I)) | ||||||
4129 | AC.registerAssumption(Assume); | ||||||
4130 | })); | ||||||
4131 | |||||||
4132 | // Lower dbg.declare intrinsics otherwise their value may be clobbered | ||||||
4133 | // by instcombiner. | ||||||
4134 | bool MadeIRChange = false; | ||||||
4135 | if (ShouldLowerDbgDeclare) | ||||||
4136 | MadeIRChange = LowerDbgDeclare(F); | ||||||
4137 | |||||||
4138 | // Iterate while there is work to do. | ||||||
4139 | unsigned Iteration = 0; | ||||||
4140 | while (true) { | ||||||
4141 | ++NumWorklistIterations; | ||||||
4142 | ++Iteration; | ||||||
4143 | |||||||
4144 | if (Iteration > InfiniteLoopDetectionThreshold) { | ||||||
4145 | report_fatal_error( | ||||||
4146 | "Instruction Combining seems stuck in an infinite loop after " + | ||||||
4147 | Twine(InfiniteLoopDetectionThreshold) + " iterations."); | ||||||
4148 | } | ||||||
4149 | |||||||
4150 | if (Iteration > MaxIterations) { | ||||||
4151 | LLVM_DEBUG(dbgs() << "\n\n[IC] Iteration limit #" << MaxIterationsdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "\n\n[IC] Iteration limit #" << MaxIterations << " on " << F.getName() << " reached; stopping before reaching a fixpoint\n"; } } while (false) | ||||||
4152 | << " on " << F.getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "\n\n[IC] Iteration limit #" << MaxIterations << " on " << F.getName() << " reached; stopping before reaching a fixpoint\n"; } } while (false) | ||||||
4153 | << " reached; stopping before reaching a fixpoint\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "\n\n[IC] Iteration limit #" << MaxIterations << " on " << F.getName() << " reached; stopping before reaching a fixpoint\n"; } } while (false); | ||||||
4154 | break; | ||||||
4155 | } | ||||||
4156 | |||||||
4157 | LLVM_DEBUG(dbgs() << "\n\nINSTCOMBINE ITERATION #" << Iteration << " on "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "\n\nINSTCOMBINE ITERATION #" << Iteration << " on " << F.getName() << "\n"; } } while (false) | ||||||
4158 | << F.getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "\n\nINSTCOMBINE ITERATION #" << Iteration << " on " << F.getName() << "\n"; } } while (false); | ||||||
4159 | |||||||
4160 | MadeIRChange |= prepareICWorklistFromFunction(F, DL, &TLI, Worklist); | ||||||
4161 | |||||||
4162 | InstCombinerImpl IC(Worklist, Builder, F.hasMinSize(), AA, AC, TLI, TTI, DT, | ||||||
4163 | ORE, BFI, PSI, DL, LI); | ||||||
4164 | IC.MaxArraySizeForCombine = MaxArraySize; | ||||||
4165 | |||||||
4166 | if (!IC.run()) | ||||||
4167 | break; | ||||||
4168 | |||||||
4169 | MadeIRChange = true; | ||||||
4170 | } | ||||||
4171 | |||||||
4172 | return MadeIRChange; | ||||||
4173 | } | ||||||
4174 | |||||||
4175 | InstCombinePass::InstCombinePass() : MaxIterations(LimitMaxIterations) {} | ||||||
4176 | |||||||
4177 | InstCombinePass::InstCombinePass(unsigned MaxIterations) | ||||||
4178 | : MaxIterations(MaxIterations) {} | ||||||
4179 | |||||||
4180 | PreservedAnalyses InstCombinePass::run(Function &F, | ||||||
4181 | FunctionAnalysisManager &AM) { | ||||||
4182 | auto &AC = AM.getResult<AssumptionAnalysis>(F); | ||||||
4183 | auto &DT = AM.getResult<DominatorTreeAnalysis>(F); | ||||||
4184 | auto &TLI = AM.getResult<TargetLibraryAnalysis>(F); | ||||||
4185 | auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F); | ||||||
4186 | auto &TTI = AM.getResult<TargetIRAnalysis>(F); | ||||||
4187 | |||||||
4188 | auto *LI = AM.getCachedResult<LoopAnalysis>(F); | ||||||
4189 | |||||||
4190 | auto *AA = &AM.getResult<AAManager>(F); | ||||||
4191 | auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F); | ||||||
4192 | ProfileSummaryInfo *PSI = | ||||||
4193 | MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent()); | ||||||
4194 | auto *BFI = (PSI && PSI->hasProfileSummary()) ? | ||||||
4195 | &AM.getResult<BlockFrequencyAnalysis>(F) : nullptr; | ||||||
4196 | |||||||
4197 | if (!combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE, | ||||||
4198 | BFI, PSI, MaxIterations, LI)) | ||||||
4199 | // No changes, all analyses are preserved. | ||||||
4200 | return PreservedAnalyses::all(); | ||||||
4201 | |||||||
4202 | // Mark all the analyses that instcombine updates as preserved. | ||||||
4203 | PreservedAnalyses PA; | ||||||
4204 | PA.preserveSet<CFGAnalyses>(); | ||||||
4205 | return PA; | ||||||
4206 | } | ||||||
4207 | |||||||
4208 | void InstructionCombiningPass::getAnalysisUsage(AnalysisUsage &AU) const { | ||||||
4209 | AU.setPreservesCFG(); | ||||||
4210 | AU.addRequired<AAResultsWrapperPass>(); | ||||||
4211 | AU.addRequired<AssumptionCacheTracker>(); | ||||||
4212 | AU.addRequired<TargetLibraryInfoWrapperPass>(); | ||||||
4213 | AU.addRequired<TargetTransformInfoWrapperPass>(); | ||||||
4214 | AU.addRequired<DominatorTreeWrapperPass>(); | ||||||
4215 | AU.addRequired<OptimizationRemarkEmitterWrapperPass>(); | ||||||
4216 | AU.addPreserved<DominatorTreeWrapperPass>(); | ||||||
4217 | AU.addPreserved<AAResultsWrapperPass>(); | ||||||
4218 | AU.addPreserved<BasicAAWrapperPass>(); | ||||||
4219 | AU.addPreserved<GlobalsAAWrapperPass>(); | ||||||
4220 | AU.addRequired<ProfileSummaryInfoWrapperPass>(); | ||||||
4221 | LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU); | ||||||
4222 | } | ||||||
4223 | |||||||
4224 | bool InstructionCombiningPass::runOnFunction(Function &F) { | ||||||
4225 | if (skipFunction(F)) | ||||||
4226 | return false; | ||||||
4227 | |||||||
4228 | // Required analyses. | ||||||
4229 | auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); | ||||||
4230 | auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F); | ||||||
4231 | auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); | ||||||
4232 | auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); | ||||||
4233 | auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree(); | ||||||
4234 | auto &ORE = getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE(); | ||||||
4235 | |||||||
4236 | // Optional analyses. | ||||||
4237 | auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>(); | ||||||
4238 | auto *LI = LIWP ? &LIWP->getLoopInfo() : nullptr; | ||||||
4239 | ProfileSummaryInfo *PSI = | ||||||
4240 | &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); | ||||||
4241 | BlockFrequencyInfo *BFI = | ||||||
4242 | (PSI && PSI->hasProfileSummary()) ? | ||||||
4243 | &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI() : | ||||||
4244 | nullptr; | ||||||
4245 | |||||||
4246 | return combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE, | ||||||
4247 | BFI, PSI, MaxIterations, LI); | ||||||
4248 | } | ||||||
4249 | |||||||
4250 | char InstructionCombiningPass::ID = 0; | ||||||
4251 | |||||||
4252 | InstructionCombiningPass::InstructionCombiningPass() | ||||||
4253 | : FunctionPass(ID), MaxIterations(InstCombineDefaultMaxIterations) { | ||||||
4254 | initializeInstructionCombiningPassPass(*PassRegistry::getPassRegistry()); | ||||||
4255 | } | ||||||
4256 | |||||||
4257 | InstructionCombiningPass::InstructionCombiningPass(unsigned MaxIterations) | ||||||
4258 | : FunctionPass(ID), MaxIterations(MaxIterations) { | ||||||
4259 | initializeInstructionCombiningPassPass(*PassRegistry::getPassRegistry()); | ||||||
4260 | } | ||||||
4261 | |||||||
4262 | INITIALIZE_PASS_BEGIN(InstructionCombiningPass, "instcombine",static void *initializeInstructionCombiningPassPassOnce(PassRegistry &Registry) { | ||||||
4263 | "Combine redundant instructions", false, false)static void *initializeInstructionCombiningPassPassOnce(PassRegistry &Registry) { | ||||||
4264 | INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry); | ||||||
4265 | INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry); | ||||||
4266 | INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry); | ||||||
4267 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry); | ||||||
4268 | INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry); | ||||||
4269 | INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)initializeGlobalsAAWrapperPassPass(Registry); | ||||||
4270 | INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)initializeOptimizationRemarkEmitterWrapperPassPass(Registry); | ||||||
4271 | INITIALIZE_PASS_DEPENDENCY(LazyBlockFrequencyInfoPass)initializeLazyBlockFrequencyInfoPassPass(Registry); | ||||||
4272 | INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)initializeProfileSummaryInfoWrapperPassPass(Registry); | ||||||
4273 | INITIALIZE_PASS_END(InstructionCombiningPass, "instcombine",PassInfo *PI = new PassInfo( "Combine redundant instructions" , "instcombine", &InstructionCombiningPass::ID, PassInfo:: NormalCtor_t(callDefaultCtor<InstructionCombiningPass>) , false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeInstructionCombiningPassPassFlag ; void llvm::initializeInstructionCombiningPassPass(PassRegistry &Registry) { llvm::call_once(InitializeInstructionCombiningPassPassFlag , initializeInstructionCombiningPassPassOnce, std::ref(Registry )); } | ||||||
4274 | "Combine redundant instructions", false, false)PassInfo *PI = new PassInfo( "Combine redundant instructions" , "instcombine", &InstructionCombiningPass::ID, PassInfo:: NormalCtor_t(callDefaultCtor<InstructionCombiningPass>) , false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeInstructionCombiningPassPassFlag ; void llvm::initializeInstructionCombiningPassPass(PassRegistry &Registry) { llvm::call_once(InitializeInstructionCombiningPassPassFlag , initializeInstructionCombiningPassPassOnce, std::ref(Registry )); } | ||||||
4275 | |||||||
4276 | // Initialization Routines | ||||||
4277 | void llvm::initializeInstCombine(PassRegistry &Registry) { | ||||||
4278 | initializeInstructionCombiningPassPass(Registry); | ||||||
4279 | } | ||||||
4280 | |||||||
4281 | void LLVMInitializeInstCombine(LLVMPassRegistryRef R) { | ||||||
4282 | initializeInstructionCombiningPassPass(*unwrap(R)); | ||||||
4283 | } | ||||||
4284 | |||||||
4285 | FunctionPass *llvm::createInstructionCombiningPass() { | ||||||
4286 | return new InstructionCombiningPass(); | ||||||
4287 | } | ||||||
4288 | |||||||
4289 | FunctionPass *llvm::createInstructionCombiningPass(unsigned MaxIterations) { | ||||||
4290 | return new InstructionCombiningPass(MaxIterations); | ||||||
4291 | } | ||||||
4292 | |||||||
4293 | void LLVMAddInstructionCombiningPass(LLVMPassManagerRef PM) { | ||||||
4294 | unwrap(PM)->add(createInstructionCombiningPass()); | ||||||
4295 | } |
1 | //===- InstCombineInternal.h - InstCombine pass internals -------*- C++ -*-===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | /// \file | |||
10 | /// | |||
11 | /// This file provides internal interfaces used to implement the InstCombine. | |||
12 | // | |||
13 | //===----------------------------------------------------------------------===// | |||
14 | ||||
15 | #ifndef LLVM_LIB_TRANSFORMS_INSTCOMBINE_INSTCOMBINEINTERNAL_H | |||
16 | #define LLVM_LIB_TRANSFORMS_INSTCOMBINE_INSTCOMBINEINTERNAL_H | |||
17 | ||||
18 | #include "llvm/ADT/Statistic.h" | |||
19 | #include "llvm/Analysis/InstructionSimplify.h" | |||
20 | #include "llvm/Analysis/TargetFolder.h" | |||
21 | #include "llvm/Analysis/ValueTracking.h" | |||
22 | #include "llvm/IR/IRBuilder.h" | |||
23 | #include "llvm/IR/InstVisitor.h" | |||
24 | #include "llvm/IR/PatternMatch.h" | |||
25 | #include "llvm/IR/Value.h" | |||
26 | #include "llvm/Support/Debug.h" | |||
27 | #include "llvm/Support/KnownBits.h" | |||
28 | #include "llvm/Transforms/InstCombine/InstCombiner.h" | |||
29 | #include "llvm/Transforms/Utils/Local.h" | |||
30 | #include <cassert> | |||
31 | ||||
32 | #define DEBUG_TYPE"instcombine" "instcombine" | |||
33 | #include "llvm/Transforms/Utils/InstructionWorklist.h" | |||
34 | ||||
35 | using namespace llvm::PatternMatch; | |||
36 | ||||
37 | // As a default, let's assume that we want to be aggressive, | |||
38 | // and attempt to traverse with no limits in attempt to sink negation. | |||
39 | static constexpr unsigned NegatorDefaultMaxDepth = ~0U; | |||
40 | ||||
41 | // Let's guesstimate that most often we will end up visiting/producing | |||
42 | // fairly small number of new instructions. | |||
43 | static constexpr unsigned NegatorMaxNodesSSO = 16; | |||
44 | ||||
45 | namespace llvm { | |||
46 | ||||
47 | class AAResults; | |||
48 | class APInt; | |||
49 | class AssumptionCache; | |||
50 | class BlockFrequencyInfo; | |||
51 | class DataLayout; | |||
52 | class DominatorTree; | |||
53 | class GEPOperator; | |||
54 | class GlobalVariable; | |||
55 | class LoopInfo; | |||
56 | class OptimizationRemarkEmitter; | |||
57 | class ProfileSummaryInfo; | |||
58 | class TargetLibraryInfo; | |||
59 | class User; | |||
60 | ||||
61 | class LLVM_LIBRARY_VISIBILITY__attribute__ ((visibility("hidden"))) InstCombinerImpl final | |||
62 | : public InstCombiner, | |||
63 | public InstVisitor<InstCombinerImpl, Instruction *> { | |||
64 | public: | |||
65 | InstCombinerImpl(InstructionWorklist &Worklist, BuilderTy &Builder, | |||
66 | bool MinimizeSize, AAResults *AA, AssumptionCache &AC, | |||
67 | TargetLibraryInfo &TLI, TargetTransformInfo &TTI, | |||
68 | DominatorTree &DT, OptimizationRemarkEmitter &ORE, | |||
69 | BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, | |||
70 | const DataLayout &DL, LoopInfo *LI) | |||
71 | : InstCombiner(Worklist, Builder, MinimizeSize, AA, AC, TLI, TTI, DT, ORE, | |||
72 | BFI, PSI, DL, LI) {} | |||
73 | ||||
74 | virtual ~InstCombinerImpl() {} | |||
75 | ||||
76 | /// Run the combiner over the entire worklist until it is empty. | |||
77 | /// | |||
78 | /// \returns true if the IR is changed. | |||
79 | bool run(); | |||
80 | ||||
81 | // Visitation implementation - Implement instruction combining for different | |||
82 | // instruction types. The semantics are as follows: | |||
83 | // Return Value: | |||
84 | // null - No change was made | |||
85 | // I - Change was made, I is still valid, I may be dead though | |||
86 | // otherwise - Change was made, replace I with returned instruction | |||
87 | // | |||
88 | Instruction *visitFNeg(UnaryOperator &I); | |||
89 | Instruction *visitAdd(BinaryOperator &I); | |||
90 | Instruction *visitFAdd(BinaryOperator &I); | |||
91 | Value *OptimizePointerDifference( | |||
92 | Value *LHS, Value *RHS, Type *Ty, bool isNUW); | |||
93 | Instruction *visitSub(BinaryOperator &I); | |||
94 | Instruction *visitFSub(BinaryOperator &I); | |||
95 | Instruction *visitMul(BinaryOperator &I); | |||
96 | Instruction *visitFMul(BinaryOperator &I); | |||
97 | Instruction *visitURem(BinaryOperator &I); | |||
98 | Instruction *visitSRem(BinaryOperator &I); | |||
99 | Instruction *visitFRem(BinaryOperator &I); | |||
100 | bool simplifyDivRemOfSelectWithZeroOp(BinaryOperator &I); | |||
101 | Instruction *commonIRemTransforms(BinaryOperator &I); | |||
102 | Instruction *commonIDivTransforms(BinaryOperator &I); | |||
103 | Instruction *visitUDiv(BinaryOperator &I); | |||
104 | Instruction *visitSDiv(BinaryOperator &I); | |||
105 | Instruction *visitFDiv(BinaryOperator &I); | |||
106 | Value *simplifyRangeCheck(ICmpInst *Cmp0, ICmpInst *Cmp1, bool Inverted); | |||
107 | Instruction *visitAnd(BinaryOperator &I); | |||
108 | Instruction *visitOr(BinaryOperator &I); | |||
109 | bool sinkNotIntoOtherHandOfAndOrOr(BinaryOperator &I); | |||
110 | Instruction *visitXor(BinaryOperator &I); | |||
111 | Instruction *visitShl(BinaryOperator &I); | |||
112 | Value *reassociateShiftAmtsOfTwoSameDirectionShifts( | |||
113 | BinaryOperator *Sh0, const SimplifyQuery &SQ, | |||
114 | bool AnalyzeForSignBitExtraction = false); | |||
115 | Instruction *canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract( | |||
116 | BinaryOperator &I); | |||
117 | Instruction *foldVariableSignZeroExtensionOfVariableHighBitExtract( | |||
118 | BinaryOperator &OldAShr); | |||
119 | Instruction *visitAShr(BinaryOperator &I); | |||
120 | Instruction *visitLShr(BinaryOperator &I); | |||
121 | Instruction *commonShiftTransforms(BinaryOperator &I); | |||
122 | Instruction *visitFCmpInst(FCmpInst &I); | |||
123 | CmpInst *canonicalizeICmpPredicate(CmpInst &I); | |||
124 | Instruction *visitICmpInst(ICmpInst &I); | |||
125 | Instruction *FoldShiftByConstant(Value *Op0, Constant *Op1, | |||
126 | BinaryOperator &I); | |||
127 | Instruction *commonCastTransforms(CastInst &CI); | |||
128 | Instruction *commonPointerCastTransforms(CastInst &CI); | |||
129 | Instruction *visitTrunc(TruncInst &CI); | |||
130 | Instruction *visitZExt(ZExtInst &CI); | |||
131 | Instruction *visitSExt(SExtInst &CI); | |||
132 | Instruction *visitFPTrunc(FPTruncInst &CI); | |||
133 | Instruction *visitFPExt(CastInst &CI); | |||
134 | Instruction *visitFPToUI(FPToUIInst &FI); | |||
135 | Instruction *visitFPToSI(FPToSIInst &FI); | |||
136 | Instruction *visitUIToFP(CastInst &CI); | |||
137 | Instruction *visitSIToFP(CastInst &CI); | |||
138 | Instruction *visitPtrToInt(PtrToIntInst &CI); | |||
139 | Instruction *visitIntToPtr(IntToPtrInst &CI); | |||
140 | Instruction *visitBitCast(BitCastInst &CI); | |||
141 | Instruction *visitAddrSpaceCast(AddrSpaceCastInst &CI); | |||
142 | Instruction *foldItoFPtoI(CastInst &FI); | |||
143 | Instruction *visitSelectInst(SelectInst &SI); | |||
144 | Instruction *visitCallInst(CallInst &CI); | |||
145 | Instruction *visitInvokeInst(InvokeInst &II); | |||
146 | Instruction *visitCallBrInst(CallBrInst &CBI); | |||
147 | ||||
148 | Instruction *SliceUpIllegalIntegerPHI(PHINode &PN); | |||
149 | Instruction *visitPHINode(PHINode &PN); | |||
150 | Instruction *visitGetElementPtrInst(GetElementPtrInst &GEP); | |||
151 | Instruction *visitAllocaInst(AllocaInst &AI); | |||
152 | Instruction *visitAllocSite(Instruction &FI); | |||
153 | Instruction *visitFree(CallInst &FI); | |||
154 | Instruction *visitLoadInst(LoadInst &LI); | |||
155 | Instruction *visitStoreInst(StoreInst &SI); | |||
156 | Instruction *visitAtomicRMWInst(AtomicRMWInst &SI); | |||
157 | Instruction *visitUnconditionalBranchInst(BranchInst &BI); | |||
158 | Instruction *visitBranchInst(BranchInst &BI); | |||
159 | Instruction *visitFenceInst(FenceInst &FI); | |||
160 | Instruction *visitSwitchInst(SwitchInst &SI); | |||
161 | Instruction *visitReturnInst(ReturnInst &RI); | |||
162 | Instruction *visitUnreachableInst(UnreachableInst &I); | |||
163 | Instruction * | |||
164 | foldAggregateConstructionIntoAggregateReuse(InsertValueInst &OrigIVI); | |||
165 | Instruction *visitInsertValueInst(InsertValueInst &IV); | |||
166 | Instruction *visitInsertElementInst(InsertElementInst &IE); | |||
167 | Instruction *visitExtractElementInst(ExtractElementInst &EI); | |||
168 | Instruction *visitShuffleVectorInst(ShuffleVectorInst &SVI); | |||
169 | Instruction *visitExtractValueInst(ExtractValueInst &EV); | |||
170 | Instruction *visitLandingPadInst(LandingPadInst &LI); | |||
171 | Instruction *visitVAEndInst(VAEndInst &I); | |||
172 | Value *pushFreezeToPreventPoisonFromPropagating(FreezeInst &FI); | |||
173 | bool freezeDominatedUses(FreezeInst &FI); | |||
174 | Instruction *visitFreeze(FreezeInst &I); | |||
175 | ||||
176 | /// Specify what to return for unhandled instructions. | |||
177 | Instruction *visitInstruction(Instruction &I) { return nullptr; } | |||
178 | ||||
179 | /// True when DB dominates all uses of DI except UI. | |||
180 | /// UI must be in the same block as DI. | |||
181 | /// The routine checks that the DI parent and DB are different. | |||
182 | bool dominatesAllUses(const Instruction *DI, const Instruction *UI, | |||
183 | const BasicBlock *DB) const; | |||
184 | ||||
185 | /// Try to replace select with select operand SIOpd in SI-ICmp sequence. | |||
186 | bool replacedSelectWithOperand(SelectInst *SI, const ICmpInst *Icmp, | |||
187 | const unsigned SIOpd); | |||
188 | ||||
189 | LoadInst *combineLoadToNewType(LoadInst &LI, Type *NewTy, | |||
190 | const Twine &Suffix = ""); | |||
191 | ||||
192 | private: | |||
193 | void annotateAnyAllocSite(CallBase &Call, const TargetLibraryInfo *TLI); | |||
194 | bool isDesirableIntType(unsigned BitWidth) const; | |||
195 | bool shouldChangeType(unsigned FromBitWidth, unsigned ToBitWidth) const; | |||
196 | bool shouldChangeType(Type *From, Type *To) const; | |||
197 | Value *dyn_castNegVal(Value *V) const; | |||
198 | Type *FindElementAtOffset(PointerType *PtrTy, int64_t Offset, | |||
199 | SmallVectorImpl<Value *> &NewIndices); | |||
200 | ||||
201 | /// Classify whether a cast is worth optimizing. | |||
202 | /// | |||
203 | /// This is a helper to decide whether the simplification of | |||
204 | /// logic(cast(A), cast(B)) to cast(logic(A, B)) should be performed. | |||
205 | /// | |||
206 | /// \param CI The cast we are interested in. | |||
207 | /// | |||
208 | /// \return true if this cast actually results in any code being generated and | |||
209 | /// if it cannot already be eliminated by some other transformation. | |||
210 | bool shouldOptimizeCast(CastInst *CI); | |||
211 | ||||
212 | /// Try to optimize a sequence of instructions checking if an operation | |||
213 | /// on LHS and RHS overflows. | |||
214 | /// | |||
215 | /// If this overflow check is done via one of the overflow check intrinsics, | |||
216 | /// then CtxI has to be the call instruction calling that intrinsic. If this | |||
217 | /// overflow check is done by arithmetic followed by a compare, then CtxI has | |||
218 | /// to be the arithmetic instruction. | |||
219 | /// | |||
220 | /// If a simplification is possible, stores the simplified result of the | |||
221 | /// operation in OperationResult and result of the overflow check in | |||
222 | /// OverflowResult, and return true. If no simplification is possible, | |||
223 | /// returns false. | |||
224 | bool OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp, bool IsSigned, | |||
225 | Value *LHS, Value *RHS, | |||
226 | Instruction &CtxI, Value *&OperationResult, | |||
227 | Constant *&OverflowResult); | |||
228 | ||||
229 | Instruction *visitCallBase(CallBase &Call); | |||
230 | Instruction *tryOptimizeCall(CallInst *CI); | |||
231 | bool transformConstExprCastCall(CallBase &Call); | |||
232 | Instruction *transformCallThroughTrampoline(CallBase &Call, | |||
233 | IntrinsicInst &Tramp); | |||
234 | ||||
235 | Value *simplifyMaskedLoad(IntrinsicInst &II); | |||
236 | Instruction *simplifyMaskedStore(IntrinsicInst &II); | |||
237 | Instruction *simplifyMaskedGather(IntrinsicInst &II); | |||
238 | Instruction *simplifyMaskedScatter(IntrinsicInst &II); | |||
239 | ||||
240 | /// Transform (zext icmp) to bitwise / integer operations in order to | |||
241 | /// eliminate it. | |||
242 | /// | |||
243 | /// \param ICI The icmp of the (zext icmp) pair we are interested in. | |||
244 | /// \parem CI The zext of the (zext icmp) pair we are interested in. | |||
245 | /// | |||
246 | /// \return null if the transformation cannot be performed. If the | |||
247 | /// transformation can be performed the new instruction that replaces the | |||
248 | /// (zext icmp) pair will be returned. | |||
249 | Instruction *transformZExtICmp(ICmpInst *ICI, ZExtInst &CI); | |||
250 | ||||
251 | Instruction *transformSExtICmp(ICmpInst *ICI, Instruction &CI); | |||
252 | ||||
253 | bool willNotOverflowSignedAdd(const Value *LHS, const Value *RHS, | |||
254 | const Instruction &CxtI) const { | |||
255 | return computeOverflowForSignedAdd(LHS, RHS, &CxtI) == | |||
256 | OverflowResult::NeverOverflows; | |||
257 | } | |||
258 | ||||
259 | bool willNotOverflowUnsignedAdd(const Value *LHS, const Value *RHS, | |||
260 | const Instruction &CxtI) const { | |||
261 | return computeOverflowForUnsignedAdd(LHS, RHS, &CxtI) == | |||
262 | OverflowResult::NeverOverflows; | |||
263 | } | |||
264 | ||||
265 | bool willNotOverflowAdd(const Value *LHS, const Value *RHS, | |||
266 | const Instruction &CxtI, bool IsSigned) const { | |||
267 | return IsSigned ? willNotOverflowSignedAdd(LHS, RHS, CxtI) | |||
268 | : willNotOverflowUnsignedAdd(LHS, RHS, CxtI); | |||
269 | } | |||
270 | ||||
271 | bool willNotOverflowSignedSub(const Value *LHS, const Value *RHS, | |||
272 | const Instruction &CxtI) const { | |||
273 | return computeOverflowForSignedSub(LHS, RHS, &CxtI) == | |||
274 | OverflowResult::NeverOverflows; | |||
275 | } | |||
276 | ||||
277 | bool willNotOverflowUnsignedSub(const Value *LHS, const Value *RHS, | |||
278 | const Instruction &CxtI) const { | |||
279 | return computeOverflowForUnsignedSub(LHS, RHS, &CxtI) == | |||
280 | OverflowResult::NeverOverflows; | |||
281 | } | |||
282 | ||||
283 | bool willNotOverflowSub(const Value *LHS, const Value *RHS, | |||
284 | const Instruction &CxtI, bool IsSigned) const { | |||
285 | return IsSigned ? willNotOverflowSignedSub(LHS, RHS, CxtI) | |||
286 | : willNotOverflowUnsignedSub(LHS, RHS, CxtI); | |||
287 | } | |||
288 | ||||
289 | bool willNotOverflowSignedMul(const Value *LHS, const Value *RHS, | |||
290 | const Instruction &CxtI) const { | |||
291 | return computeOverflowForSignedMul(LHS, RHS, &CxtI) == | |||
292 | OverflowResult::NeverOverflows; | |||
293 | } | |||
294 | ||||
295 | bool willNotOverflowUnsignedMul(const Value *LHS, const Value *RHS, | |||
296 | const Instruction &CxtI) const { | |||
297 | return computeOverflowForUnsignedMul(LHS, RHS, &CxtI) == | |||
298 | OverflowResult::NeverOverflows; | |||
299 | } | |||
300 | ||||
301 | bool willNotOverflowMul(const Value *LHS, const Value *RHS, | |||
302 | const Instruction &CxtI, bool IsSigned) const { | |||
303 | return IsSigned ? willNotOverflowSignedMul(LHS, RHS, CxtI) | |||
304 | : willNotOverflowUnsignedMul(LHS, RHS, CxtI); | |||
305 | } | |||
306 | ||||
307 | bool willNotOverflow(BinaryOperator::BinaryOps Opcode, const Value *LHS, | |||
308 | const Value *RHS, const Instruction &CxtI, | |||
309 | bool IsSigned) const { | |||
310 | switch (Opcode) { | |||
311 | case Instruction::Add: return willNotOverflowAdd(LHS, RHS, CxtI, IsSigned); | |||
312 | case Instruction::Sub: return willNotOverflowSub(LHS, RHS, CxtI, IsSigned); | |||
313 | case Instruction::Mul: return willNotOverflowMul(LHS, RHS, CxtI, IsSigned); | |||
314 | default: llvm_unreachable("Unexpected opcode for overflow query")::llvm::llvm_unreachable_internal("Unexpected opcode for overflow query" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstCombineInternal.h" , 314); | |||
315 | } | |||
316 | } | |||
317 | ||||
318 | Value *EmitGEPOffset(User *GEP); | |||
319 | Instruction *scalarizePHI(ExtractElementInst &EI, PHINode *PN); | |||
320 | Instruction *foldBitcastExtElt(ExtractElementInst &ExtElt); | |||
321 | Instruction *foldCastedBitwiseLogic(BinaryOperator &I); | |||
322 | Instruction *narrowBinOp(TruncInst &Trunc); | |||
323 | Instruction *narrowMaskedBinOp(BinaryOperator &And); | |||
324 | Instruction *narrowMathIfNoOverflow(BinaryOperator &I); | |||
325 | Instruction *narrowFunnelShift(TruncInst &Trunc); | |||
326 | Instruction *optimizeBitCastFromPhi(CastInst &CI, PHINode *PN); | |||
327 | Instruction *matchSAddSubSat(Instruction &MinMax1); | |||
328 | Instruction *foldNot(BinaryOperator &I); | |||
329 | ||||
330 | void freelyInvertAllUsersOf(Value *V); | |||
331 | ||||
332 | /// Determine if a pair of casts can be replaced by a single cast. | |||
333 | /// | |||
334 | /// \param CI1 The first of a pair of casts. | |||
335 | /// \param CI2 The second of a pair of casts. | |||
336 | /// | |||
337 | /// \return 0 if the cast pair cannot be eliminated, otherwise returns an | |||
338 | /// Instruction::CastOps value for a cast that can replace the pair, casting | |||
339 | /// CI1->getSrcTy() to CI2->getDstTy(). | |||
340 | /// | |||
341 | /// \see CastInst::isEliminableCastPair | |||
342 | Instruction::CastOps isEliminableCastPair(const CastInst *CI1, | |||
343 | const CastInst *CI2); | |||
344 | Value *simplifyIntToPtrRoundTripCast(Value *Val); | |||
345 | ||||
346 | Value *foldAndOfICmps(ICmpInst *LHS, ICmpInst *RHS, BinaryOperator &And); | |||
347 | Value *foldOrOfICmps(ICmpInst *LHS, ICmpInst *RHS, BinaryOperator &Or); | |||
348 | Value *foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS, BinaryOperator &Xor); | |||
349 | ||||
350 | Value *foldEqOfParts(ICmpInst *Cmp0, ICmpInst *Cmp1, bool IsAnd); | |||
351 | ||||
352 | /// Optimize (fcmp)&(fcmp) or (fcmp)|(fcmp). | |||
353 | /// NOTE: Unlike most of instcombine, this returns a Value which should | |||
354 | /// already be inserted into the function. | |||
355 | Value *foldLogicOfFCmps(FCmpInst *LHS, FCmpInst *RHS, bool IsAnd); | |||
356 | ||||
357 | Value *foldAndOrOfICmpsOfAndWithPow2(ICmpInst *LHS, ICmpInst *RHS, | |||
358 | Instruction *CxtI, bool IsAnd, | |||
359 | bool IsLogical = false); | |||
360 | Value *matchSelectFromAndOr(Value *A, Value *B, Value *C, Value *D); | |||
361 | Value *getSelectCondition(Value *A, Value *B); | |||
362 | ||||
363 | Instruction *foldIntrinsicWithOverflowCommon(IntrinsicInst *II); | |||
364 | Instruction *foldFPSignBitOps(BinaryOperator &I); | |||
365 | ||||
366 | // Optimize one of these forms: | |||
367 | // and i1 Op, SI / select i1 Op, i1 SI, i1 false (if IsAnd = true) | |||
368 | // or i1 Op, SI / select i1 Op, i1 true, i1 SI (if IsAnd = false) | |||
369 | // into simplier select instruction using isImpliedCondition. | |||
370 | Instruction *foldAndOrOfSelectUsingImpliedCond(Value *Op, SelectInst &SI, | |||
371 | bool IsAnd); | |||
372 | ||||
373 | public: | |||
374 | /// Inserts an instruction \p New before instruction \p Old | |||
375 | /// | |||
376 | /// Also adds the new instruction to the worklist and returns \p New so that | |||
377 | /// it is suitable for use as the return from the visitation patterns. | |||
378 | Instruction *InsertNewInstBefore(Instruction *New, Instruction &Old) { | |||
379 | assert(New && !New->getParent() &&(static_cast <bool> (New && !New->getParent( ) && "New instruction already inserted into a basic block!" ) ? void (0) : __assert_fail ("New && !New->getParent() && \"New instruction already inserted into a basic block!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstCombineInternal.h" , 380, __extension__ __PRETTY_FUNCTION__)) | |||
380 | "New instruction already inserted into a basic block!")(static_cast <bool> (New && !New->getParent( ) && "New instruction already inserted into a basic block!" ) ? void (0) : __assert_fail ("New && !New->getParent() && \"New instruction already inserted into a basic block!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstCombineInternal.h" , 380, __extension__ __PRETTY_FUNCTION__)); | |||
381 | BasicBlock *BB = Old.getParent(); | |||
382 | BB->getInstList().insert(Old.getIterator(), New); // Insert inst | |||
383 | Worklist.add(New); | |||
384 | return New; | |||
385 | } | |||
386 | ||||
387 | /// Same as InsertNewInstBefore, but also sets the debug loc. | |||
388 | Instruction *InsertNewInstWith(Instruction *New, Instruction &Old) { | |||
389 | New->setDebugLoc(Old.getDebugLoc()); | |||
390 | return InsertNewInstBefore(New, Old); | |||
391 | } | |||
392 | ||||
393 | /// A combiner-aware RAUW-like routine. | |||
394 | /// | |||
395 | /// This method is to be used when an instruction is found to be dead, | |||
396 | /// replaceable with another preexisting expression. Here we add all uses of | |||
397 | /// I to the worklist, replace all uses of I with the new value, then return | |||
398 | /// I, so that the inst combiner will know that I was modified. | |||
399 | Instruction *replaceInstUsesWith(Instruction &I, Value *V) { | |||
400 | // If there are no uses to replace, then we return nullptr to indicate that | |||
401 | // no changes were made to the program. | |||
402 | if (I.use_empty()) return nullptr; | |||
403 | ||||
404 | Worklist.pushUsersToWorkList(I); // Add all modified instrs to worklist. | |||
405 | ||||
406 | // If we are replacing the instruction with itself, this must be in a | |||
407 | // segment of unreachable code, so just clobber the instruction. | |||
408 | if (&I == V) | |||
409 | V = UndefValue::get(I.getType()); | |||
410 | ||||
411 | LLVM_DEBUG(dbgs() << "IC: Replacing " << I << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: Replacing " << I << "\n" << " with " << *V << '\n' ; } } while (false) | |||
| ||||
412 | << " with " << *V << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: Replacing " << I << "\n" << " with " << *V << '\n' ; } } while (false); | |||
413 | ||||
414 | I.replaceAllUsesWith(V); | |||
415 | MadeIRChange = true; | |||
416 | return &I; | |||
417 | } | |||
418 | ||||
419 | /// Replace operand of instruction and add old operand to the worklist. | |||
420 | Instruction *replaceOperand(Instruction &I, unsigned OpNum, Value *V) { | |||
421 | Worklist.addValue(I.getOperand(OpNum)); | |||
422 | I.setOperand(OpNum, V); | |||
423 | return &I; | |||
424 | } | |||
425 | ||||
426 | /// Replace use and add the previously used value to the worklist. | |||
427 | void replaceUse(Use &U, Value *NewValue) { | |||
428 | Worklist.addValue(U); | |||
429 | U = NewValue; | |||
430 | } | |||
431 | ||||
432 | /// Create and insert the idiom we use to indicate a block is unreachable | |||
433 | /// without having to rewrite the CFG from within InstCombine. | |||
434 | void CreateNonTerminatorUnreachable(Instruction *InsertAt) { | |||
435 | auto &Ctx = InsertAt->getContext(); | |||
436 | new StoreInst(ConstantInt::getTrue(Ctx), | |||
437 | UndefValue::get(Type::getInt1PtrTy(Ctx)), | |||
438 | InsertAt); | |||
439 | } | |||
440 | ||||
441 | ||||
442 | /// Combiner aware instruction erasure. | |||
443 | /// | |||
444 | /// When dealing with an instruction that has side effects or produces a void | |||
445 | /// value, we can't rely on DCE to delete the instruction. Instead, visit | |||
446 | /// methods should return the value returned by this function. | |||
447 | Instruction *eraseInstFromFunction(Instruction &I) override { | |||
448 | LLVM_DEBUG(dbgs() << "IC: ERASE " << I << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("instcombine")) { dbgs() << "IC: ERASE " << I << '\n'; } } while (false); | |||
449 | assert(I.use_empty() && "Cannot erase instruction that is used!")(static_cast <bool> (I.use_empty() && "Cannot erase instruction that is used!" ) ? void (0) : __assert_fail ("I.use_empty() && \"Cannot erase instruction that is used!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/lib/Transforms/InstCombine/InstCombineInternal.h" , 449, __extension__ __PRETTY_FUNCTION__)); | |||
450 | salvageDebugInfo(I); | |||
451 | ||||
452 | // Make sure that we reprocess all operands now that we reduced their | |||
453 | // use counts. | |||
454 | for (Use &Operand : I.operands()) | |||
455 | if (auto *Inst = dyn_cast<Instruction>(Operand)) | |||
456 | Worklist.add(Inst); | |||
457 | ||||
458 | Worklist.remove(&I); | |||
459 | I.eraseFromParent(); | |||
460 | MadeIRChange = true; | |||
461 | return nullptr; // Don't do anything with FI | |||
462 | } | |||
463 | ||||
464 | void computeKnownBits(const Value *V, KnownBits &Known, | |||
465 | unsigned Depth, const Instruction *CxtI) const { | |||
466 | llvm::computeKnownBits(V, Known, DL, Depth, &AC, CxtI, &DT); | |||
467 | } | |||
468 | ||||
469 | KnownBits computeKnownBits(const Value *V, unsigned Depth, | |||
470 | const Instruction *CxtI) const { | |||
471 | return llvm::computeKnownBits(V, DL, Depth, &AC, CxtI, &DT); | |||
472 | } | |||
473 | ||||
474 | bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero = false, | |||
475 | unsigned Depth = 0, | |||
476 | const Instruction *CxtI = nullptr) { | |||
477 | return llvm::isKnownToBeAPowerOfTwo(V, DL, OrZero, Depth, &AC, CxtI, &DT); | |||
478 | } | |||
479 | ||||
480 | bool MaskedValueIsZero(const Value *V, const APInt &Mask, unsigned Depth = 0, | |||
481 | const Instruction *CxtI = nullptr) const { | |||
482 | return llvm::MaskedValueIsZero(V, Mask, DL, Depth, &AC, CxtI, &DT); | |||
483 | } | |||
484 | ||||
485 | unsigned ComputeNumSignBits(const Value *Op, unsigned Depth = 0, | |||
486 | const Instruction *CxtI = nullptr) const { | |||
487 | return llvm::ComputeNumSignBits(Op, DL, Depth, &AC, CxtI, &DT); | |||
488 | } | |||
489 | ||||
490 | OverflowResult computeOverflowForUnsignedMul(const Value *LHS, | |||
491 | const Value *RHS, | |||
492 | const Instruction *CxtI) const { | |||
493 | return llvm::computeOverflowForUnsignedMul(LHS, RHS, DL, &AC, CxtI, &DT); | |||
494 | } | |||
495 | ||||
496 | OverflowResult computeOverflowForSignedMul(const Value *LHS, | |||
497 | const Value *RHS, | |||
498 | const Instruction *CxtI) const { | |||
499 | return llvm::computeOverflowForSignedMul(LHS, RHS, DL, &AC, CxtI, &DT); | |||
500 | } | |||
501 | ||||
502 | OverflowResult computeOverflowForUnsignedAdd(const Value *LHS, | |||
503 | const Value *RHS, | |||
504 | const Instruction *CxtI) const { | |||
505 | return llvm::computeOverflowForUnsignedAdd(LHS, RHS, DL, &AC, CxtI, &DT); | |||
506 | } | |||
507 | ||||
508 | OverflowResult computeOverflowForSignedAdd(const Value *LHS, | |||
509 | const Value *RHS, | |||
510 | const Instruction *CxtI) const { | |||
511 | return llvm::computeOverflowForSignedAdd(LHS, RHS, DL, &AC, CxtI, &DT); | |||
512 | } | |||
513 | ||||
514 | OverflowResult computeOverflowForUnsignedSub(const Value *LHS, | |||
515 | const Value *RHS, | |||
516 | const Instruction *CxtI) const { | |||
517 | return llvm::computeOverflowForUnsignedSub(LHS, RHS, DL, &AC, CxtI, &DT); | |||
518 | } | |||
519 | ||||
520 | OverflowResult computeOverflowForSignedSub(const Value *LHS, const Value *RHS, | |||
521 | const Instruction *CxtI) const { | |||
522 | return llvm::computeOverflowForSignedSub(LHS, RHS, DL, &AC, CxtI, &DT); | |||
523 | } | |||
524 | ||||
525 | OverflowResult computeOverflow( | |||
526 | Instruction::BinaryOps BinaryOp, bool IsSigned, | |||
527 | Value *LHS, Value *RHS, Instruction *CxtI) const; | |||
528 | ||||
529 | /// Performs a few simplifications for operators which are associative | |||
530 | /// or commutative. | |||
531 | bool SimplifyAssociativeOrCommutative(BinaryOperator &I); | |||
532 | ||||
533 | /// Tries to simplify binary operations which some other binary | |||
534 | /// operation distributes over. | |||
535 | /// | |||
536 | /// It does this by either by factorizing out common terms (eg "(A*B)+(A*C)" | |||
537 | /// -> "A*(B+C)") or expanding out if this results in simplifications (eg: "A | |||
538 | /// & (B | C) -> (A&B) | (A&C)" if this is a win). Returns the simplified | |||
539 | /// value, or null if it didn't simplify. | |||
540 | Value *SimplifyUsingDistributiveLaws(BinaryOperator &I); | |||
541 | ||||
542 | /// Tries to simplify add operations using the definition of remainder. | |||
543 | /// | |||
544 | /// The definition of remainder is X % C = X - (X / C ) * C. The add | |||
545 | /// expression X % C0 + (( X / C0 ) % C1) * C0 can be simplified to | |||
546 | /// X % (C0 * C1) | |||
547 | Value *SimplifyAddWithRemainder(BinaryOperator &I); | |||
548 | ||||
549 | // Binary Op helper for select operations where the expression can be | |||
550 | // efficiently reorganized. | |||
551 | Value *SimplifySelectsFeedingBinaryOp(BinaryOperator &I, Value *LHS, | |||
552 | Value *RHS); | |||
553 | ||||
554 | /// This tries to simplify binary operations by factorizing out common terms | |||
555 | /// (e. g. "(A*B)+(A*C)" -> "A*(B+C)"). | |||
556 | Value *tryFactorization(BinaryOperator &, Instruction::BinaryOps, Value *, | |||
557 | Value *, Value *, Value *); | |||
558 | ||||
559 | /// Match a select chain which produces one of three values based on whether | |||
560 | /// the LHS is less than, equal to, or greater than RHS respectively. | |||
561 | /// Return true if we matched a three way compare idiom. The LHS, RHS, Less, | |||
562 | /// Equal and Greater values are saved in the matching process and returned to | |||
563 | /// the caller. | |||
564 | bool matchThreeWayIntCompare(SelectInst *SI, Value *&LHS, Value *&RHS, | |||
565 | ConstantInt *&Less, ConstantInt *&Equal, | |||
566 | ConstantInt *&Greater); | |||
567 | ||||
568 | /// Attempts to replace V with a simpler value based on the demanded | |||
569 | /// bits. | |||
570 | Value *SimplifyDemandedUseBits(Value *V, APInt DemandedMask, KnownBits &Known, | |||
571 | unsigned Depth, Instruction *CxtI); | |||
572 | bool SimplifyDemandedBits(Instruction *I, unsigned Op, | |||
573 | const APInt &DemandedMask, KnownBits &Known, | |||
574 | unsigned Depth = 0) override; | |||
575 | ||||
576 | /// Helper routine of SimplifyDemandedUseBits. It computes KnownZero/KnownOne | |||
577 | /// bits. It also tries to handle simplifications that can be done based on | |||
578 | /// DemandedMask, but without modifying the Instruction. | |||
579 | Value *SimplifyMultipleUseDemandedBits(Instruction *I, | |||
580 | const APInt &DemandedMask, | |||
581 | KnownBits &Known, | |||
582 | unsigned Depth, Instruction *CxtI); | |||
583 | ||||
584 | /// Helper routine of SimplifyDemandedUseBits. It tries to simplify demanded | |||
585 | /// bit for "r1 = shr x, c1; r2 = shl r1, c2" instruction sequence. | |||
586 | Value *simplifyShrShlDemandedBits( | |||
587 | Instruction *Shr, const APInt &ShrOp1, Instruction *Shl, | |||
588 | const APInt &ShlOp1, const APInt &DemandedMask, KnownBits &Known); | |||
589 | ||||
590 | /// Tries to simplify operands to an integer instruction based on its | |||
591 | /// demanded bits. | |||
592 | bool SimplifyDemandedInstructionBits(Instruction &Inst); | |||
593 | ||||
594 | virtual Value * | |||
595 | SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &UndefElts, | |||
596 | unsigned Depth = 0, | |||
597 | bool AllowMultipleUsers = false) override; | |||
598 | ||||
599 | /// Canonicalize the position of binops relative to shufflevector. | |||
600 | Instruction *foldVectorBinop(BinaryOperator &Inst); | |||
601 | Instruction *foldVectorSelect(SelectInst &Sel); | |||
602 | ||||
603 | /// Given a binary operator, cast instruction, or select which has a PHI node | |||
604 | /// as operand #0, see if we can fold the instruction into the PHI (which is | |||
605 | /// only possible if all operands to the PHI are constants). | |||
606 | Instruction *foldOpIntoPhi(Instruction &I, PHINode *PN); | |||
607 | ||||
608 | /// Given an instruction with a select as one operand and a constant as the | |||
609 | /// other operand, try to fold the binary operator into the select arguments. | |||
610 | /// This also works for Cast instructions, which obviously do not have a | |||
611 | /// second operand. | |||
612 | Instruction *FoldOpIntoSelect(Instruction &Op, SelectInst *SI); | |||
613 | ||||
614 | /// This is a convenience wrapper function for the above two functions. | |||
615 | Instruction *foldBinOpIntoSelectOrPhi(BinaryOperator &I); | |||
616 | ||||
617 | Instruction *foldAddWithConstant(BinaryOperator &Add); | |||
618 | ||||
619 | /// Try to rotate an operation below a PHI node, using PHI nodes for | |||
620 | /// its operands. | |||
621 | Instruction *foldPHIArgOpIntoPHI(PHINode &PN); | |||
622 | Instruction *foldPHIArgBinOpIntoPHI(PHINode &PN); | |||
623 | Instruction *foldPHIArgInsertValueInstructionIntoPHI(PHINode &PN); | |||
624 | Instruction *foldPHIArgExtractValueInstructionIntoPHI(PHINode &PN); | |||
625 | Instruction *foldPHIArgGEPIntoPHI(PHINode &PN); | |||
626 | Instruction *foldPHIArgLoadIntoPHI(PHINode &PN); | |||
627 | Instruction *foldPHIArgZextsIntoPHI(PHINode &PN); | |||
628 | Instruction *foldPHIArgIntToPtrToPHI(PHINode &PN); | |||
629 | ||||
630 | /// If an integer typed PHI has only one use which is an IntToPtr operation, | |||
631 | /// replace the PHI with an existing pointer typed PHI if it exists. Otherwise | |||
632 | /// insert a new pointer typed PHI and replace the original one. | |||
633 | Instruction *foldIntegerTypedPHI(PHINode &PN); | |||
634 | ||||
635 | /// Helper function for FoldPHIArgXIntoPHI() to set debug location for the | |||
636 | /// folded operation. | |||
637 | void PHIArgMergedDebugLoc(Instruction *Inst, PHINode &PN); | |||
638 | ||||
639 | Instruction *foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, | |||
640 | ICmpInst::Predicate Cond, Instruction &I); | |||
641 | Instruction *foldAllocaCmp(ICmpInst &ICI, const AllocaInst *Alloca, | |||
642 | const Value *Other); | |||
643 | Instruction *foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, | |||
644 | GlobalVariable *GV, CmpInst &ICI, | |||
645 | ConstantInt *AndCst = nullptr); | |||
646 | Instruction *foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, | |||
647 | Constant *RHSC); | |||
648 | Instruction *foldICmpAddOpConst(Value *X, const APInt &C, | |||
649 | ICmpInst::Predicate Pred); | |||
650 | Instruction *foldICmpWithCastOp(ICmpInst &ICI); | |||
651 | ||||
652 | Instruction *foldICmpUsingKnownBits(ICmpInst &Cmp); | |||
653 | Instruction *foldICmpWithDominatingICmp(ICmpInst &Cmp); | |||
654 | Instruction *foldICmpWithConstant(ICmpInst &Cmp); | |||
655 | Instruction *foldICmpInstWithConstant(ICmpInst &Cmp); | |||
656 | Instruction *foldICmpInstWithConstantNotInt(ICmpInst &Cmp); | |||
657 | Instruction *foldICmpBinOp(ICmpInst &Cmp, const SimplifyQuery &SQ); | |||
658 | Instruction *foldICmpEquality(ICmpInst &Cmp); | |||
659 | Instruction *foldIRemByPowerOfTwoToBitTest(ICmpInst &I); | |||
660 | Instruction *foldSignBitTest(ICmpInst &I); | |||
661 | Instruction *foldICmpWithZero(ICmpInst &Cmp); | |||
662 | ||||
663 | Value *foldMultiplicationOverflowCheck(ICmpInst &Cmp); | |||
664 | ||||
665 | Instruction *foldICmpSelectConstant(ICmpInst &Cmp, SelectInst *Select, | |||
666 | ConstantInt *C); | |||
667 | Instruction *foldICmpTruncConstant(ICmpInst &Cmp, TruncInst *Trunc, | |||
668 | const APInt &C); | |||
669 | Instruction *foldICmpAndConstant(ICmpInst &Cmp, BinaryOperator *And, | |||
670 | const APInt &C); | |||
671 | Instruction *foldICmpXorConstant(ICmpInst &Cmp, BinaryOperator *Xor, | |||
672 | const APInt &C); | |||
673 | Instruction *foldICmpOrConstant(ICmpInst &Cmp, BinaryOperator *Or, | |||
674 | const APInt &C); | |||
675 | Instruction *foldICmpMulConstant(ICmpInst &Cmp, BinaryOperator *Mul, | |||
676 | const APInt &C); | |||
677 | Instruction *foldICmpShlConstant(ICmpInst &Cmp, BinaryOperator *Shl, | |||
678 | const APInt &C); | |||
679 | Instruction *foldICmpShrConstant(ICmpInst &Cmp, BinaryOperator *Shr, | |||
680 | const APInt &C); | |||
681 | Instruction *foldICmpSRemConstant(ICmpInst &Cmp, BinaryOperator *UDiv, | |||
682 | const APInt &C); | |||
683 | Instruction *foldICmpUDivConstant(ICmpInst &Cmp, BinaryOperator *UDiv, | |||
684 | const APInt &C); | |||
685 | Instruction *foldICmpDivConstant(ICmpInst &Cmp, BinaryOperator *Div, | |||
686 | const APInt &C); | |||
687 | Instruction *foldICmpSubConstant(ICmpInst &Cmp, BinaryOperator *Sub, | |||
688 | const APInt &C); | |||
689 | Instruction *foldICmpAddConstant(ICmpInst &Cmp, BinaryOperator *Add, | |||
690 | const APInt &C); | |||
691 | Instruction *foldICmpAndConstConst(ICmpInst &Cmp, BinaryOperator *And, | |||
692 | const APInt &C1); | |||
693 | Instruction *foldICmpAndShift(ICmpInst &Cmp, BinaryOperator *And, | |||
694 | const APInt &C1, const APInt &C2); | |||
695 | Instruction *foldICmpShrConstConst(ICmpInst &I, Value *ShAmt, const APInt &C1, | |||
696 | const APInt &C2); | |||
697 | Instruction *foldICmpShlConstConst(ICmpInst &I, Value *ShAmt, const APInt &C1, | |||
698 | const APInt &C2); | |||
699 | ||||
700 | Instruction *foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp, | |||
701 | BinaryOperator *BO, | |||
702 | const APInt &C); | |||
703 | Instruction *foldICmpIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II, | |||
704 | const APInt &C); | |||
705 | Instruction *foldICmpEqIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II, | |||
706 | const APInt &C); | |||
707 | Instruction *foldICmpBitCast(ICmpInst &Cmp); | |||
708 | ||||
709 | // Helpers of visitSelectInst(). | |||
710 | Instruction *foldSelectExtConst(SelectInst &Sel); | |||
711 | Instruction *foldSelectOpOp(SelectInst &SI, Instruction *TI, Instruction *FI); | |||
712 | Instruction *foldSelectIntoOp(SelectInst &SI, Value *, Value *); | |||
713 | Instruction *foldSPFofSPF(Instruction *Inner, SelectPatternFlavor SPF1, | |||
714 | Value *A, Value *B, Instruction &Outer, | |||
715 | SelectPatternFlavor SPF2, Value *C); | |||
716 | Instruction *foldSelectInstWithICmp(SelectInst &SI, ICmpInst *ICI); | |||
717 | Instruction *foldSelectValueEquivalence(SelectInst &SI, ICmpInst &ICI); | |||
718 | ||||
719 | Value *insertRangeTest(Value *V, const APInt &Lo, const APInt &Hi, | |||
720 | bool isSigned, bool Inside); | |||
721 | Instruction *PromoteCastOfAllocation(BitCastInst &CI, AllocaInst &AI); | |||
722 | bool mergeStoreIntoSuccessor(StoreInst &SI); | |||
723 | ||||
724 | /// Given an initial instruction, check to see if it is the root of a | |||
725 | /// bswap/bitreverse idiom. If so, return the equivalent bswap/bitreverse | |||
726 | /// intrinsic. | |||
727 | Instruction *matchBSwapOrBitReverse(Instruction &I, bool MatchBSwaps, | |||
728 | bool MatchBitReversals); | |||
729 | ||||
730 | Instruction *SimplifyAnyMemTransfer(AnyMemTransferInst *MI); | |||
731 | Instruction *SimplifyAnyMemSet(AnyMemSetInst *MI); | |||
732 | ||||
733 | Value *EvaluateInDifferentType(Value *V, Type *Ty, bool isSigned); | |||
734 | ||||
735 | /// Returns a value X such that Val = X * Scale, or null if none. | |||
736 | /// | |||
737 | /// If the multiplication is known not to overflow then NoSignedWrap is set. | |||
738 | Value *Descale(Value *Val, APInt Scale, bool &NoSignedWrap); | |||
739 | }; | |||
740 | ||||
741 | class Negator final { | |||
742 | /// Top-to-bottom, def-to-use negated instruction tree we produced. | |||
743 | SmallVector<Instruction *, NegatorMaxNodesSSO> NewInstructions; | |||
744 | ||||
745 | using BuilderTy = IRBuilder<TargetFolder, IRBuilderCallbackInserter>; | |||
746 | BuilderTy Builder; | |||
747 | ||||
748 | const DataLayout &DL; | |||
749 | AssumptionCache &AC; | |||
750 | const DominatorTree &DT; | |||
751 | ||||
752 | const bool IsTrulyNegation; | |||
753 | ||||
754 | SmallDenseMap<Value *, Value *> NegationsCache; | |||
755 | ||||
756 | Negator(LLVMContext &C, const DataLayout &DL, AssumptionCache &AC, | |||
757 | const DominatorTree &DT, bool IsTrulyNegation); | |||
758 | ||||
759 | #if LLVM_ENABLE_STATS1 | |||
760 | unsigned NumValuesVisitedInThisNegator = 0; | |||
761 | ~Negator(); | |||
762 | #endif | |||
763 | ||||
764 | using Result = std::pair<ArrayRef<Instruction *> /*NewInstructions*/, | |||
765 | Value * /*NegatedRoot*/>; | |||
766 | ||||
767 | std::array<Value *, 2> getSortedOperandsOfBinOp(Instruction *I); | |||
768 | ||||
769 | LLVM_NODISCARD[[clang::warn_unused_result]] Value *visitImpl(Value *V, unsigned Depth); | |||
770 | ||||
771 | LLVM_NODISCARD[[clang::warn_unused_result]] Value *negate(Value *V, unsigned Depth); | |||
772 | ||||
773 | /// Recurse depth-first and attempt to sink the negation. | |||
774 | /// FIXME: use worklist? | |||
775 | LLVM_NODISCARD[[clang::warn_unused_result]] Optional<Result> run(Value *Root); | |||
776 | ||||
777 | Negator(const Negator &) = delete; | |||
778 | Negator(Negator &&) = delete; | |||
779 | Negator &operator=(const Negator &) = delete; | |||
780 | Negator &operator=(Negator &&) = delete; | |||
781 | ||||
782 | public: | |||
783 | /// Attempt to negate \p Root. Retuns nullptr if negation can't be performed, | |||
784 | /// otherwise returns negated value. | |||
785 | LLVM_NODISCARD[[clang::warn_unused_result]] static Value *Negate(bool LHSIsZero, Value *Root, | |||
786 | InstCombinerImpl &IC); | |||
787 | }; | |||
788 | ||||
789 | } // end namespace llvm | |||
790 | ||||
791 | #undef DEBUG_TYPE"instcombine" | |||
792 | ||||
793 | #endif // LLVM_LIB_TRANSFORMS_INSTCOMBINE_INSTCOMBINEINTERNAL_H |
1 | //===- llvm/Value.h - Definition of the Value class -------------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file declares the Value class. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #ifndef LLVM_IR_VALUE_H |
14 | #define LLVM_IR_VALUE_H |
15 | |
16 | #include "llvm-c/Types.h" |
17 | #include "llvm/ADT/STLExtras.h" |
18 | #include "llvm/ADT/StringRef.h" |
19 | #include "llvm/ADT/iterator_range.h" |
20 | #include "llvm/IR/Use.h" |
21 | #include "llvm/Support/Alignment.h" |
22 | #include "llvm/Support/CBindingWrapping.h" |
23 | #include "llvm/Support/Casting.h" |
24 | #include <cassert> |
25 | #include <iterator> |
26 | #include <memory> |
27 | |
28 | namespace llvm { |
29 | |
30 | class APInt; |
31 | class Argument; |
32 | class BasicBlock; |
33 | class Constant; |
34 | class ConstantData; |
35 | class ConstantAggregate; |
36 | class DataLayout; |
37 | class Function; |
38 | class GlobalAlias; |
39 | class GlobalIFunc; |
40 | class GlobalIndirectSymbol; |
41 | class GlobalObject; |
42 | class GlobalValue; |
43 | class GlobalVariable; |
44 | class InlineAsm; |
45 | class Instruction; |
46 | class LLVMContext; |
47 | class MDNode; |
48 | class Module; |
49 | class ModuleSlotTracker; |
50 | class raw_ostream; |
51 | template<typename ValueTy> class StringMapEntry; |
52 | class Twine; |
53 | class Type; |
54 | class User; |
55 | |
56 | using ValueName = StringMapEntry<Value *>; |
57 | |
58 | //===----------------------------------------------------------------------===// |
59 | // Value Class |
60 | //===----------------------------------------------------------------------===// |
61 | |
62 | /// LLVM Value Representation |
63 | /// |
64 | /// This is a very important LLVM class. It is the base class of all values |
65 | /// computed by a program that may be used as operands to other values. Value is |
66 | /// the super class of other important classes such as Instruction and Function. |
67 | /// All Values have a Type. Type is not a subclass of Value. Some values can |
68 | /// have a name and they belong to some Module. Setting the name on the Value |
69 | /// automatically updates the module's symbol table. |
70 | /// |
71 | /// Every value has a "use list" that keeps track of which other Values are |
72 | /// using this Value. A Value can also have an arbitrary number of ValueHandle |
73 | /// objects that watch it and listen to RAUW and Destroy events. See |
74 | /// llvm/IR/ValueHandle.h for details. |
75 | class Value { |
76 | Type *VTy; |
77 | Use *UseList; |
78 | |
79 | friend class ValueAsMetadata; // Allow access to IsUsedByMD. |
80 | friend class ValueHandleBase; |
81 | |
82 | const unsigned char SubclassID; // Subclass identifier (for isa/dyn_cast) |
83 | unsigned char HasValueHandle : 1; // Has a ValueHandle pointing to this? |
84 | |
85 | protected: |
86 | /// Hold subclass data that can be dropped. |
87 | /// |
88 | /// This member is similar to SubclassData, however it is for holding |
89 | /// information which may be used to aid optimization, but which may be |
90 | /// cleared to zero without affecting conservative interpretation. |
91 | unsigned char SubclassOptionalData : 7; |
92 | |
93 | private: |
94 | /// Hold arbitrary subclass data. |
95 | /// |
96 | /// This member is defined by this class, but is not used for anything. |
97 | /// Subclasses can use it to hold whatever state they find useful. This |
98 | /// field is initialized to zero by the ctor. |
99 | unsigned short SubclassData; |
100 | |
101 | protected: |
102 | /// The number of operands in the subclass. |
103 | /// |
104 | /// This member is defined by this class, but not used for anything. |
105 | /// Subclasses can use it to store their number of operands, if they have |
106 | /// any. |
107 | /// |
108 | /// This is stored here to save space in User on 64-bit hosts. Since most |
109 | /// instances of Value have operands, 32-bit hosts aren't significantly |
110 | /// affected. |
111 | /// |
112 | /// Note, this should *NOT* be used directly by any class other than User. |
113 | /// User uses this value to find the Use list. |
114 | enum : unsigned { NumUserOperandsBits = 27 }; |
115 | unsigned NumUserOperands : NumUserOperandsBits; |
116 | |
117 | // Use the same type as the bitfield above so that MSVC will pack them. |
118 | unsigned IsUsedByMD : 1; |
119 | unsigned HasName : 1; |
120 | unsigned HasMetadata : 1; // Has metadata attached to this? |
121 | unsigned HasHungOffUses : 1; |
122 | unsigned HasDescriptor : 1; |
123 | |
124 | private: |
125 | template <typename UseT> // UseT == 'Use' or 'const Use' |
126 | class use_iterator_impl { |
127 | friend class Value; |
128 | |
129 | UseT *U; |
130 | |
131 | explicit use_iterator_impl(UseT *u) : U(u) {} |
132 | |
133 | public: |
134 | using iterator_category = std::forward_iterator_tag; |
135 | using value_type = UseT *; |
136 | using difference_type = std::ptrdiff_t; |
137 | using pointer = value_type *; |
138 | using reference = value_type &; |
139 | |
140 | use_iterator_impl() : U() {} |
141 | |
142 | bool operator==(const use_iterator_impl &x) const { return U == x.U; } |
143 | bool operator!=(const use_iterator_impl &x) const { return !operator==(x); } |
144 | |
145 | use_iterator_impl &operator++() { // Preincrement |
146 | assert(U && "Cannot increment end iterator!")(static_cast <bool> (U && "Cannot increment end iterator!" ) ? void (0) : __assert_fail ("U && \"Cannot increment end iterator!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/include/llvm/IR/Value.h" , 146, __extension__ __PRETTY_FUNCTION__)); |
147 | U = U->getNext(); |
148 | return *this; |
149 | } |
150 | |
151 | use_iterator_impl operator++(int) { // Postincrement |
152 | auto tmp = *this; |
153 | ++*this; |
154 | return tmp; |
155 | } |
156 | |
157 | UseT &operator*() const { |
158 | assert(U && "Cannot dereference end iterator!")(static_cast <bool> (U && "Cannot dereference end iterator!" ) ? void (0) : __assert_fail ("U && \"Cannot dereference end iterator!\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/include/llvm/IR/Value.h" , 158, __extension__ __PRETTY_FUNCTION__)); |
159 | return *U; |
160 | } |
161 | |
162 | UseT *operator->() const { return &operator*(); } |
163 | |
164 | operator use_iterator_impl<const UseT>() const { |
165 | return use_iterator_impl<const UseT>(U); |
166 | } |
167 | }; |
168 | |
169 | template <typename UserTy> // UserTy == 'User' or 'const User' |
170 | class user_iterator_impl { |
171 | use_iterator_impl<Use> UI; |
172 | explicit user_iterator_impl(Use *U) : UI(U) {} |
173 | friend class Value; |
174 | |
175 | public: |
176 | using iterator_category = std::forward_iterator_tag; |
177 | using value_type = UserTy *; |
178 | using difference_type = std::ptrdiff_t; |
179 | using pointer = value_type *; |
180 | using reference = value_type &; |
181 | |
182 | user_iterator_impl() = default; |
183 | |
184 | bool operator==(const user_iterator_impl &x) const { return UI == x.UI; } |
185 | bool operator!=(const user_iterator_impl &x) const { return !operator==(x); } |
186 | |
187 | /// Returns true if this iterator is equal to user_end() on the value. |
188 | bool atEnd() const { return *this == user_iterator_impl(); } |
189 | |
190 | user_iterator_impl &operator++() { // Preincrement |
191 | ++UI; |
192 | return *this; |
193 | } |
194 | |
195 | user_iterator_impl operator++(int) { // Postincrement |
196 | auto tmp = *this; |
197 | ++*this; |
198 | return tmp; |
199 | } |
200 | |
201 | // Retrieve a pointer to the current User. |
202 | UserTy *operator*() const { |
203 | return UI->getUser(); |
204 | } |
205 | |
206 | UserTy *operator->() const { return operator*(); } |
207 | |
208 | operator user_iterator_impl<const UserTy>() const { |
209 | return user_iterator_impl<const UserTy>(*UI); |
210 | } |
211 | |
212 | Use &getUse() const { return *UI; } |
213 | }; |
214 | |
215 | protected: |
216 | Value(Type *Ty, unsigned scid); |
217 | |
218 | /// Value's destructor should be virtual by design, but that would require |
219 | /// that Value and all of its subclasses have a vtable that effectively |
220 | /// duplicates the information in the value ID. As a size optimization, the |
221 | /// destructor has been protected, and the caller should manually call |
222 | /// deleteValue. |
223 | ~Value(); // Use deleteValue() to delete a generic Value. |
224 | |
225 | public: |
226 | Value(const Value &) = delete; |
227 | Value &operator=(const Value &) = delete; |
228 | |
229 | /// Delete a pointer to a generic Value. |
230 | void deleteValue(); |
231 | |
232 | /// Support for debugging, callable in GDB: V->dump() |
233 | void dump() const; |
234 | |
235 | /// Implement operator<< on Value. |
236 | /// @{ |
237 | void print(raw_ostream &O, bool IsForDebug = false) const; |
238 | void print(raw_ostream &O, ModuleSlotTracker &MST, |
239 | bool IsForDebug = false) const; |
240 | /// @} |
241 | |
242 | /// Print the name of this Value out to the specified raw_ostream. |
243 | /// |
244 | /// This is useful when you just want to print 'int %reg126', not the |
245 | /// instruction that generated it. If you specify a Module for context, then |
246 | /// even constanst get pretty-printed; for example, the type of a null |
247 | /// pointer is printed symbolically. |
248 | /// @{ |
249 | void printAsOperand(raw_ostream &O, bool PrintType = true, |
250 | const Module *M = nullptr) const; |
251 | void printAsOperand(raw_ostream &O, bool PrintType, |
252 | ModuleSlotTracker &MST) const; |
253 | /// @} |
254 | |
255 | /// All values are typed, get the type of this value. |
256 | Type *getType() const { return VTy; } |
257 | |
258 | /// All values hold a context through their type. |
259 | LLVMContext &getContext() const; |
260 | |
261 | // All values can potentially be named. |
262 | bool hasName() const { return HasName; } |
263 | ValueName *getValueName() const; |
264 | void setValueName(ValueName *VN); |
265 | |
266 | private: |
267 | void destroyValueName(); |
268 | enum class ReplaceMetadataUses { No, Yes }; |
269 | void doRAUW(Value *New, ReplaceMetadataUses); |
270 | void setNameImpl(const Twine &Name); |
271 | |
272 | public: |
273 | /// Return a constant reference to the value's name. |
274 | /// |
275 | /// This guaranteed to return the same reference as long as the value is not |
276 | /// modified. If the value has a name, this does a hashtable lookup, so it's |
277 | /// not free. |
278 | StringRef getName() const; |
279 | |
280 | /// Change the name of the value. |
281 | /// |
282 | /// Choose a new unique name if the provided name is taken. |
283 | /// |
284 | /// \param Name The new name; or "" if the value's name should be removed. |
285 | void setName(const Twine &Name); |
286 | |
287 | /// Transfer the name from V to this value. |
288 | /// |
289 | /// After taking V's name, sets V's name to empty. |
290 | /// |
291 | /// \note It is an error to call V->takeName(V). |
292 | void takeName(Value *V); |
293 | |
294 | #ifndef NDEBUG |
295 | std::string getNameOrAsOperand() const; |
296 | #endif |
297 | |
298 | /// Change all uses of this to point to a new Value. |
299 | /// |
300 | /// Go through the uses list for this definition and make each use point to |
301 | /// "V" instead of "this". After this completes, 'this's use list is |
302 | /// guaranteed to be empty. |
303 | void replaceAllUsesWith(Value *V); |
304 | |
305 | /// Change non-metadata uses of this to point to a new Value. |
306 | /// |
307 | /// Go through the uses list for this definition and make each use point to |
308 | /// "V" instead of "this". This function skips metadata entries in the list. |
309 | void replaceNonMetadataUsesWith(Value *V); |
310 | |
311 | /// Go through the uses list for this definition and make each use point |
312 | /// to "V" if the callback ShouldReplace returns true for the given Use. |
313 | /// Unlike replaceAllUsesWith() this function does not support basic block |
314 | /// values. |
315 | void replaceUsesWithIf(Value *New, |
316 | llvm::function_ref<bool(Use &U)> ShouldReplace); |
317 | |
318 | /// replaceUsesOutsideBlock - Go through the uses list for this definition and |
319 | /// make each use point to "V" instead of "this" when the use is outside the |
320 | /// block. 'This's use list is expected to have at least one element. |
321 | /// Unlike replaceAllUsesWith() this function does not support basic block |
322 | /// values. |
323 | void replaceUsesOutsideBlock(Value *V, BasicBlock *BB); |
324 | |
325 | //---------------------------------------------------------------------- |
326 | // Methods for handling the chain of uses of this Value. |
327 | // |
328 | // Materializing a function can introduce new uses, so these methods come in |
329 | // two variants: |
330 | // The methods that start with materialized_ check the uses that are |
331 | // currently known given which functions are materialized. Be very careful |
332 | // when using them since you might not get all uses. |
333 | // The methods that don't start with materialized_ assert that modules is |
334 | // fully materialized. |
335 | void assertModuleIsMaterializedImpl() const; |
336 | // This indirection exists so we can keep assertModuleIsMaterializedImpl() |
337 | // around in release builds of Value.cpp to be linked with other code built |
338 | // in debug mode. But this avoids calling it in any of the release built code. |
339 | void assertModuleIsMaterialized() const { |
340 | #ifndef NDEBUG |
341 | assertModuleIsMaterializedImpl(); |
342 | #endif |
343 | } |
344 | |
345 | bool use_empty() const { |
346 | assertModuleIsMaterialized(); |
347 | return UseList == nullptr; |
348 | } |
349 | |
350 | bool materialized_use_empty() const { |
351 | return UseList == nullptr; |
352 | } |
353 | |
354 | using use_iterator = use_iterator_impl<Use>; |
355 | using const_use_iterator = use_iterator_impl<const Use>; |
356 | |
357 | use_iterator materialized_use_begin() { return use_iterator(UseList); } |
358 | const_use_iterator materialized_use_begin() const { |
359 | return const_use_iterator(UseList); |
360 | } |
361 | use_iterator use_begin() { |
362 | assertModuleIsMaterialized(); |
363 | return materialized_use_begin(); |
364 | } |
365 | const_use_iterator use_begin() const { |
366 | assertModuleIsMaterialized(); |
367 | return materialized_use_begin(); |
368 | } |
369 | use_iterator use_end() { return use_iterator(); } |
370 | const_use_iterator use_end() const { return const_use_iterator(); } |
371 | iterator_range<use_iterator> materialized_uses() { |
372 | return make_range(materialized_use_begin(), use_end()); |
373 | } |
374 | iterator_range<const_use_iterator> materialized_uses() const { |
375 | return make_range(materialized_use_begin(), use_end()); |
376 | } |
377 | iterator_range<use_iterator> uses() { |
378 | assertModuleIsMaterialized(); |
379 | return materialized_uses(); |
380 | } |
381 | iterator_range<const_use_iterator> uses() const { |
382 | assertModuleIsMaterialized(); |
383 | return materialized_uses(); |
384 | } |
385 | |
386 | bool user_empty() const { |
387 | assertModuleIsMaterialized(); |
388 | return UseList == nullptr; |
389 | } |
390 | |
391 | using user_iterator = user_iterator_impl<User>; |
392 | using const_user_iterator = user_iterator_impl<const User>; |
393 | |
394 | user_iterator materialized_user_begin() { return user_iterator(UseList); } |
395 | const_user_iterator materialized_user_begin() const { |
396 | return const_user_iterator(UseList); |
397 | } |
398 | user_iterator user_begin() { |
399 | assertModuleIsMaterialized(); |
400 | return materialized_user_begin(); |
401 | } |
402 | const_user_iterator user_begin() const { |
403 | assertModuleIsMaterialized(); |
404 | return materialized_user_begin(); |
405 | } |
406 | user_iterator user_end() { return user_iterator(); } |
407 | const_user_iterator user_end() const { return const_user_iterator(); } |
408 | User *user_back() { |
409 | assertModuleIsMaterialized(); |
410 | return *materialized_user_begin(); |
411 | } |
412 | const User *user_back() const { |
413 | assertModuleIsMaterialized(); |
414 | return *materialized_user_begin(); |
415 | } |
416 | iterator_range<user_iterator> materialized_users() { |
417 | return make_range(materialized_user_begin(), user_end()); |
418 | } |
419 | iterator_range<const_user_iterator> materialized_users() const { |
420 | return make_range(materialized_user_begin(), user_end()); |
421 | } |
422 | iterator_range<user_iterator> users() { |
423 | assertModuleIsMaterialized(); |
424 | return materialized_users(); |
425 | } |
426 | iterator_range<const_user_iterator> users() const { |
427 | assertModuleIsMaterialized(); |
428 | return materialized_users(); |
429 | } |
430 | |
431 | /// Return true if there is exactly one use of this value. |
432 | /// |
433 | /// This is specialized because it is a common request and does not require |
434 | /// traversing the whole use list. |
435 | bool hasOneUse() const { return hasSingleElement(uses()); } |
436 | |
437 | /// Return true if this Value has exactly N uses. |
438 | bool hasNUses(unsigned N) const; |
439 | |
440 | /// Return true if this value has N uses or more. |
441 | /// |
442 | /// This is logically equivalent to getNumUses() >= N. |
443 | bool hasNUsesOrMore(unsigned N) const; |
444 | |
445 | /// Return true if there is exactly one user of this value. |
446 | /// |
447 | /// Note that this is not the same as "has one use". If a value has one use, |
448 | /// then there certainly is a single user. But if value has several uses, |
449 | /// it is possible that all uses are in a single user, or not. |
450 | /// |
451 | /// This check is potentially costly, since it requires traversing, |
452 | /// in the worst case, the whole use list of a value. |
453 | bool hasOneUser() const; |
454 | |
455 | /// Return true if there is exactly one use of this value that cannot be |
456 | /// dropped. |
457 | Use *getSingleUndroppableUse(); |
458 | const Use *getSingleUndroppableUse() const { |
459 | return const_cast<Value *>(this)->getSingleUndroppableUse(); |
460 | } |
461 | |
462 | /// Return true if there is exactly one unique user of this value that cannot be |
463 | /// dropped (that user can have multiple uses of this value). |
464 | User *getUniqueUndroppableUser(); |
465 | const User *getUniqueUndroppableUser() const { |
466 | return const_cast<Value *>(this)->getUniqueUndroppableUser(); |
467 | } |
468 | |
469 | /// Return true if there this value. |
470 | /// |
471 | /// This is specialized because it is a common request and does not require |
472 | /// traversing the whole use list. |
473 | bool hasNUndroppableUses(unsigned N) const; |
474 | |
475 | /// Return true if this value has N uses or more. |
476 | /// |
477 | /// This is logically equivalent to getNumUses() >= N. |
478 | bool hasNUndroppableUsesOrMore(unsigned N) const; |
479 | |
480 | /// Remove every uses that can safely be removed. |
481 | /// |
482 | /// This will remove for example uses in llvm.assume. |
483 | /// This should be used when performing want to perform a tranformation but |
484 | /// some Droppable uses pervent it. |
485 | /// This function optionally takes a filter to only remove some droppable |
486 | /// uses. |
487 | void dropDroppableUses(llvm::function_ref<bool(const Use *)> ShouldDrop = |
488 | [](const Use *) { return true; }); |
489 | |
490 | /// Remove every use of this value in \p User that can safely be removed. |
491 | void dropDroppableUsesIn(User &Usr); |
492 | |
493 | /// Remove the droppable use \p U. |
494 | static void dropDroppableUse(Use &U); |
495 | |
496 | /// Check if this value is used in the specified basic block. |
497 | bool isUsedInBasicBlock(const BasicBlock *BB) const; |
498 | |
499 | /// This method computes the number of uses of this Value. |
500 | /// |
501 | /// This is a linear time operation. Use hasOneUse, hasNUses, or |
502 | /// hasNUsesOrMore to check for specific values. |
503 | unsigned getNumUses() const; |
504 | |
505 | /// This method should only be used by the Use class. |
506 | void addUse(Use &U) { U.addToList(&UseList); } |
507 | |
508 | /// Concrete subclass of this. |
509 | /// |
510 | /// An enumeration for keeping track of the concrete subclass of Value that |
511 | /// is actually instantiated. Values of this enumeration are kept in the |
512 | /// Value classes SubclassID field. They are used for concrete type |
513 | /// identification. |
514 | enum ValueTy { |
515 | #define HANDLE_VALUE(Name) Name##Val, |
516 | #include "llvm/IR/Value.def" |
517 | |
518 | // Markers: |
519 | #define HANDLE_CONSTANT_MARKER(Marker, Constant) Marker = Constant##Val, |
520 | #include "llvm/IR/Value.def" |
521 | }; |
522 | |
523 | /// Return an ID for the concrete type of this object. |
524 | /// |
525 | /// This is used to implement the classof checks. This should not be used |
526 | /// for any other purpose, as the values may change as LLVM evolves. Also, |
527 | /// note that for instructions, the Instruction's opcode is added to |
528 | /// InstructionVal. So this means three things: |
529 | /// # there is no value with code InstructionVal (no opcode==0). |
530 | /// # there are more possible values for the value type than in ValueTy enum. |
531 | /// # the InstructionVal enumerator must be the highest valued enumerator in |
532 | /// the ValueTy enum. |
533 | unsigned getValueID() const { |
534 | return SubclassID; |
535 | } |
536 | |
537 | /// Return the raw optional flags value contained in this value. |
538 | /// |
539 | /// This should only be used when testing two Values for equivalence. |
540 | unsigned getRawSubclassOptionalData() const { |
541 | return SubclassOptionalData; |
542 | } |
543 | |
544 | /// Clear the optional flags contained in this value. |
545 | void clearSubclassOptionalData() { |
546 | SubclassOptionalData = 0; |
547 | } |
548 | |
549 | /// Check the optional flags for equality. |
550 | bool hasSameSubclassOptionalData(const Value *V) const { |
551 | return SubclassOptionalData == V->SubclassOptionalData; |
552 | } |
553 | |
554 | /// Return true if there is a value handle associated with this value. |
555 | bool hasValueHandle() const { return HasValueHandle; } |
556 | |
557 | /// Return true if there is metadata referencing this value. |
558 | bool isUsedByMetadata() const { return IsUsedByMD; } |
559 | |
560 | // Return true if this value is only transitively referenced by metadata. |
561 | bool isTransitiveUsedByMetadataOnly() const; |
562 | |
563 | protected: |
564 | /// Get the current metadata attachments for the given kind, if any. |
565 | /// |
566 | /// These functions require that the value have at most a single attachment |
567 | /// of the given kind, and return \c nullptr if such an attachment is missing. |
568 | /// @{ |
569 | MDNode *getMetadata(unsigned KindID) const; |
570 | MDNode *getMetadata(StringRef Kind) const; |
571 | /// @} |
572 | |
573 | /// Appends all attachments with the given ID to \c MDs in insertion order. |
574 | /// If the Value has no attachments with the given ID, or if ID is invalid, |
575 | /// leaves MDs unchanged. |
576 | /// @{ |
577 | void getMetadata(unsigned KindID, SmallVectorImpl<MDNode *> &MDs) const; |
578 | void getMetadata(StringRef Kind, SmallVectorImpl<MDNode *> &MDs) const; |
579 | /// @} |
580 | |
581 | /// Appends all metadata attached to this value to \c MDs, sorting by |
582 | /// KindID. The first element of each pair returned is the KindID, the second |
583 | /// element is the metadata value. Attachments with the same ID appear in |
584 | /// insertion order. |
585 | void |
586 | getAllMetadata(SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const; |
587 | |
588 | /// Return true if this value has any metadata attached to it. |
589 | bool hasMetadata() const { return (bool)HasMetadata; } |
590 | |
591 | /// Return true if this value has the given type of metadata attached. |
592 | /// @{ |
593 | bool hasMetadata(unsigned KindID) const { |
594 | return getMetadata(KindID) != nullptr; |
595 | } |
596 | bool hasMetadata(StringRef Kind) const { |
597 | return getMetadata(Kind) != nullptr; |
598 | } |
599 | /// @} |
600 | |
601 | /// Set a particular kind of metadata attachment. |
602 | /// |
603 | /// Sets the given attachment to \c MD, erasing it if \c MD is \c nullptr or |
604 | /// replacing it if it already exists. |
605 | /// @{ |
606 | void setMetadata(unsigned KindID, MDNode *Node); |
607 | void setMetadata(StringRef Kind, MDNode *Node); |
608 | /// @} |
609 | |
610 | /// Add a metadata attachment. |
611 | /// @{ |
612 | void addMetadata(unsigned KindID, MDNode &MD); |
613 | void addMetadata(StringRef Kind, MDNode &MD); |
614 | /// @} |
615 | |
616 | /// Erase all metadata attachments with the given kind. |
617 | /// |
618 | /// \returns true if any metadata was removed. |
619 | bool eraseMetadata(unsigned KindID); |
620 | |
621 | /// Erase all metadata attached to this Value. |
622 | void clearMetadata(); |
623 | |
624 | public: |
625 | /// Return true if this value is a swifterror value. |
626 | /// |
627 | /// swifterror values can be either a function argument or an alloca with a |
628 | /// swifterror attribute. |
629 | bool isSwiftError() const; |
630 | |
631 | /// Strip off pointer casts, all-zero GEPs and address space casts. |
632 | /// |
633 | /// Returns the original uncasted value. If this is called on a non-pointer |
634 | /// value, it returns 'this'. |
635 | const Value *stripPointerCasts() const; |
636 | Value *stripPointerCasts() { |
637 | return const_cast<Value *>( |
638 | static_cast<const Value *>(this)->stripPointerCasts()); |
639 | } |
640 | |
641 | /// Strip off pointer casts, all-zero GEPs, address space casts, and aliases. |
642 | /// |
643 | /// Returns the original uncasted value. If this is called on a non-pointer |
644 | /// value, it returns 'this'. |
645 | const Value *stripPointerCastsAndAliases() const; |
646 | Value *stripPointerCastsAndAliases() { |
647 | return const_cast<Value *>( |
648 | static_cast<const Value *>(this)->stripPointerCastsAndAliases()); |
649 | } |
650 | |
651 | /// Strip off pointer casts, all-zero GEPs and address space casts |
652 | /// but ensures the representation of the result stays the same. |
653 | /// |
654 | /// Returns the original uncasted value with the same representation. If this |
655 | /// is called on a non-pointer value, it returns 'this'. |
656 | const Value *stripPointerCastsSameRepresentation() const; |
657 | Value *stripPointerCastsSameRepresentation() { |
658 | return const_cast<Value *>(static_cast<const Value *>(this) |
659 | ->stripPointerCastsSameRepresentation()); |
660 | } |
661 | |
662 | /// Strip off pointer casts, all-zero GEPs, single-argument phi nodes and |
663 | /// invariant group info. |
664 | /// |
665 | /// Returns the original uncasted value. If this is called on a non-pointer |
666 | /// value, it returns 'this'. This function should be used only in |
667 | /// Alias analysis. |
668 | const Value *stripPointerCastsForAliasAnalysis() const; |
669 | Value *stripPointerCastsForAliasAnalysis() { |
670 | return const_cast<Value *>(static_cast<const Value *>(this) |
671 | ->stripPointerCastsForAliasAnalysis()); |
672 | } |
673 | |
674 | /// Strip off pointer casts and all-constant inbounds GEPs. |
675 | /// |
676 | /// Returns the original pointer value. If this is called on a non-pointer |
677 | /// value, it returns 'this'. |
678 | const Value *stripInBoundsConstantOffsets() const; |
679 | Value *stripInBoundsConstantOffsets() { |
680 | return const_cast<Value *>( |
681 | static_cast<const Value *>(this)->stripInBoundsConstantOffsets()); |
682 | } |
683 | |
684 | /// Accumulate the constant offset this value has compared to a base pointer. |
685 | /// Only 'getelementptr' instructions (GEPs) are accumulated but other |
686 | /// instructions, e.g., casts, are stripped away as well. |
687 | /// The accumulated constant offset is added to \p Offset and the base |
688 | /// pointer is returned. |
689 | /// |
690 | /// The APInt \p Offset has to have a bit-width equal to the IntPtr type for |
691 | /// the address space of 'this' pointer value, e.g., use |
692 | /// DataLayout::getIndexTypeSizeInBits(Ty). |
693 | /// |
694 | /// If \p AllowNonInbounds is true, offsets in GEPs are stripped and |
695 | /// accumulated even if the GEP is not "inbounds". |
696 | /// |
697 | /// If \p ExternalAnalysis is provided it will be used to calculate a offset |
698 | /// when a operand of GEP is not constant. |
699 | /// For example, for a value \p ExternalAnalysis might try to calculate a |
700 | /// lower bound. If \p ExternalAnalysis is successful, it should return true. |
701 | /// |
702 | /// If this is called on a non-pointer value, it returns 'this' and the |
703 | /// \p Offset is not modified. |
704 | /// |
705 | /// Note that this function will never return a nullptr. It will also never |
706 | /// manipulate the \p Offset in a way that would not match the difference |
707 | /// between the underlying value and the returned one. Thus, if no constant |
708 | /// offset was found, the returned value is the underlying one and \p Offset |
709 | /// is unchanged. |
710 | const Value *stripAndAccumulateConstantOffsets( |
711 | const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, |
712 | function_ref<bool(Value &Value, APInt &Offset)> ExternalAnalysis = |
713 | nullptr) const; |
714 | Value *stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, |
715 | bool AllowNonInbounds) { |
716 | return const_cast<Value *>( |
717 | static_cast<const Value *>(this)->stripAndAccumulateConstantOffsets( |
718 | DL, Offset, AllowNonInbounds)); |
719 | } |
720 | |
721 | /// This is a wrapper around stripAndAccumulateConstantOffsets with the |
722 | /// in-bounds requirement set to false. |
723 | const Value *stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, |
724 | APInt &Offset) const { |
725 | return stripAndAccumulateConstantOffsets(DL, Offset, |
726 | /* AllowNonInbounds */ false); |
727 | } |
728 | Value *stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, |
729 | APInt &Offset) { |
730 | return stripAndAccumulateConstantOffsets(DL, Offset, |
731 | /* AllowNonInbounds */ false); |
732 | } |
733 | |
734 | /// Strip off pointer casts and inbounds GEPs. |
735 | /// |
736 | /// Returns the original pointer value. If this is called on a non-pointer |
737 | /// value, it returns 'this'. |
738 | const Value *stripInBoundsOffsets(function_ref<void(const Value *)> Func = |
739 | [](const Value *) {}) const; |
740 | inline Value *stripInBoundsOffsets(function_ref<void(const Value *)> Func = |
741 | [](const Value *) {}) { |
742 | return const_cast<Value *>( |
743 | static_cast<const Value *>(this)->stripInBoundsOffsets(Func)); |
744 | } |
745 | |
746 | /// Return true if the memory object referred to by V can by freed in the |
747 | /// scope for which the SSA value defining the allocation is statically |
748 | /// defined. E.g. deallocation after the static scope of a value does not |
749 | /// count, but a deallocation before that does. |
750 | bool canBeFreed() const; |
751 | |
752 | /// Returns the number of bytes known to be dereferenceable for the |
753 | /// pointer value. |
754 | /// |
755 | /// If CanBeNull is set by this function the pointer can either be null or be |
756 | /// dereferenceable up to the returned number of bytes. |
757 | /// |
758 | /// IF CanBeFreed is true, the pointer is known to be dereferenceable at |
759 | /// point of definition only. Caller must prove that allocation is not |
760 | /// deallocated between point of definition and use. |
761 | uint64_t getPointerDereferenceableBytes(const DataLayout &DL, |
762 | bool &CanBeNull, |
763 | bool &CanBeFreed) const; |
764 | |
765 | /// Returns an alignment of the pointer value. |
766 | /// |
767 | /// Returns an alignment which is either specified explicitly, e.g. via |
768 | /// align attribute of a function argument, or guaranteed by DataLayout. |
769 | Align getPointerAlignment(const DataLayout &DL) const; |
770 | |
771 | /// Translate PHI node to its predecessor from the given basic block. |
772 | /// |
773 | /// If this value is a PHI node with CurBB as its parent, return the value in |
774 | /// the PHI node corresponding to PredBB. If not, return ourself. This is |
775 | /// useful if you want to know the value something has in a predecessor |
776 | /// block. |
777 | const Value *DoPHITranslation(const BasicBlock *CurBB, |
778 | const BasicBlock *PredBB) const; |
779 | Value *DoPHITranslation(const BasicBlock *CurBB, const BasicBlock *PredBB) { |
780 | return const_cast<Value *>( |
781 | static_cast<const Value *>(this)->DoPHITranslation(CurBB, PredBB)); |
782 | } |
783 | |
784 | /// The maximum alignment for instructions. |
785 | /// |
786 | /// This is the greatest alignment value supported by load, store, and alloca |
787 | /// instructions, and global values. |
788 | static constexpr unsigned MaxAlignmentExponent = 30; |
789 | static constexpr unsigned MaximumAlignment = 1u << MaxAlignmentExponent; |
790 | |
791 | /// Mutate the type of this Value to be of the specified type. |
792 | /// |
793 | /// Note that this is an extremely dangerous operation which can create |
794 | /// completely invalid IR very easily. It is strongly recommended that you |
795 | /// recreate IR objects with the right types instead of mutating them in |
796 | /// place. |
797 | void mutateType(Type *Ty) { |
798 | VTy = Ty; |
799 | } |
800 | |
801 | /// Sort the use-list. |
802 | /// |
803 | /// Sorts the Value's use-list by Cmp using a stable mergesort. Cmp is |
804 | /// expected to compare two \a Use references. |
805 | template <class Compare> void sortUseList(Compare Cmp); |
806 | |
807 | /// Reverse the use-list. |
808 | void reverseUseList(); |
809 | |
810 | private: |
811 | /// Merge two lists together. |
812 | /// |
813 | /// Merges \c L and \c R using \c Cmp. To enable stable sorts, always pushes |
814 | /// "equal" items from L before items from R. |
815 | /// |
816 | /// \return the first element in the list. |
817 | /// |
818 | /// \note Completely ignores \a Use::Prev (doesn't read, doesn't update). |
819 | template <class Compare> |
820 | static Use *mergeUseLists(Use *L, Use *R, Compare Cmp) { |
821 | Use *Merged; |
822 | Use **Next = &Merged; |
823 | |
824 | while (true) { |
825 | if (!L) { |
826 | *Next = R; |
827 | break; |
828 | } |
829 | if (!R) { |
830 | *Next = L; |
831 | break; |
832 | } |
833 | if (Cmp(*R, *L)) { |
834 | *Next = R; |
835 | Next = &R->Next; |
836 | R = R->Next; |
837 | } else { |
838 | *Next = L; |
839 | Next = &L->Next; |
840 | L = L->Next; |
841 | } |
842 | } |
843 | |
844 | return Merged; |
845 | } |
846 | |
847 | protected: |
848 | unsigned short getSubclassDataFromValue() const { return SubclassData; } |
849 | void setValueSubclassData(unsigned short D) { SubclassData = D; } |
850 | }; |
851 | |
852 | struct ValueDeleter { void operator()(Value *V) { V->deleteValue(); } }; |
853 | |
854 | /// Use this instead of std::unique_ptr<Value> or std::unique_ptr<Instruction>. |
855 | /// Those don't work because Value and Instruction's destructors are protected, |
856 | /// aren't virtual, and won't destroy the complete object. |
857 | using unique_value = std::unique_ptr<Value, ValueDeleter>; |
858 | |
859 | inline raw_ostream &operator<<(raw_ostream &OS, const Value &V) { |
860 | V.print(OS); |
861 | return OS; |
862 | } |
863 | |
864 | void Use::set(Value *V) { |
865 | if (Val) removeFromList(); |
866 | Val = V; |
867 | if (V) V->addUse(*this); |
868 | } |
869 | |
870 | Value *Use::operator=(Value *RHS) { |
871 | set(RHS); |
872 | return RHS; |
873 | } |
874 | |
875 | const Use &Use::operator=(const Use &RHS) { |
876 | set(RHS.Val); |
877 | return *this; |
878 | } |
879 | |
880 | template <class Compare> void Value::sortUseList(Compare Cmp) { |
881 | if (!UseList || !UseList->Next) |
882 | // No need to sort 0 or 1 uses. |
883 | return; |
884 | |
885 | // Note: this function completely ignores Prev pointers until the end when |
886 | // they're fixed en masse. |
887 | |
888 | // Create a binomial vector of sorted lists, visiting uses one at a time and |
889 | // merging lists as necessary. |
890 | const unsigned MaxSlots = 32; |
891 | Use *Slots[MaxSlots]; |
892 | |
893 | // Collect the first use, turning it into a single-item list. |
894 | Use *Next = UseList->Next; |
895 | UseList->Next = nullptr; |
896 | unsigned NumSlots = 1; |
897 | Slots[0] = UseList; |
898 | |
899 | // Collect all but the last use. |
900 | while (Next->Next) { |
901 | Use *Current = Next; |
902 | Next = Current->Next; |
903 | |
904 | // Turn Current into a single-item list. |
905 | Current->Next = nullptr; |
906 | |
907 | // Save Current in the first available slot, merging on collisions. |
908 | unsigned I; |
909 | for (I = 0; I < NumSlots; ++I) { |
910 | if (!Slots[I]) |
911 | break; |
912 | |
913 | // Merge two lists, doubling the size of Current and emptying slot I. |
914 | // |
915 | // Since the uses in Slots[I] originally preceded those in Current, send |
916 | // Slots[I] in as the left parameter to maintain a stable sort. |
917 | Current = mergeUseLists(Slots[I], Current, Cmp); |
918 | Slots[I] = nullptr; |
919 | } |
920 | // Check if this is a new slot. |
921 | if (I == NumSlots) { |
922 | ++NumSlots; |
923 | assert(NumSlots <= MaxSlots && "Use list bigger than 2^32")(static_cast <bool> (NumSlots <= MaxSlots && "Use list bigger than 2^32") ? void (0) : __assert_fail ("NumSlots <= MaxSlots && \"Use list bigger than 2^32\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/include/llvm/IR/Value.h" , 923, __extension__ __PRETTY_FUNCTION__)); |
924 | } |
925 | |
926 | // Found an open slot. |
927 | Slots[I] = Current; |
928 | } |
929 | |
930 | // Merge all the lists together. |
931 | assert(Next && "Expected one more Use")(static_cast <bool> (Next && "Expected one more Use" ) ? void (0) : __assert_fail ("Next && \"Expected one more Use\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/include/llvm/IR/Value.h" , 931, __extension__ __PRETTY_FUNCTION__)); |
932 | assert(!Next->Next && "Expected only one Use")(static_cast <bool> (!Next->Next && "Expected only one Use" ) ? void (0) : __assert_fail ("!Next->Next && \"Expected only one Use\"" , "/build/llvm-toolchain-snapshot-14~++20211006100657+62d67d9e7c9c/llvm/include/llvm/IR/Value.h" , 932, __extension__ __PRETTY_FUNCTION__)); |
933 | UseList = Next; |
934 | for (unsigned I = 0; I < NumSlots; ++I) |
935 | if (Slots[I]) |
936 | // Since the uses in Slots[I] originally preceded those in UseList, send |
937 | // Slots[I] in as the left parameter to maintain a stable sort. |
938 | UseList = mergeUseLists(Slots[I], UseList, Cmp); |
939 | |
940 | // Fix the Prev pointers. |
941 | for (Use *I = UseList, **Prev = &UseList; I; I = I->Next) { |
942 | I->Prev = Prev; |
943 | Prev = &I->Next; |
944 | } |
945 | } |
946 | |
947 | // isa - Provide some specializations of isa so that we don't have to include |
948 | // the subtype header files to test to see if the value is a subclass... |
949 | // |
950 | template <> struct isa_impl<Constant, Value> { |
951 | static inline bool doit(const Value &Val) { |
952 | static_assert(Value::ConstantFirstVal == 0, "Val.getValueID() >= Value::ConstantFirstVal"); |
953 | return Val.getValueID() <= Value::ConstantLastVal; |
954 | } |
955 | }; |
956 | |
957 | template <> struct isa_impl<ConstantData, Value> { |
958 | static inline bool doit(const Value &Val) { |
959 | return Val.getValueID() >= Value::ConstantDataFirstVal && |
960 | Val.getValueID() <= Value::ConstantDataLastVal; |
961 | } |
962 | }; |
963 | |
964 | template <> struct isa_impl<ConstantAggregate, Value> { |
965 | static inline bool doit(const Value &Val) { |
966 | return Val.getValueID() >= Value::ConstantAggregateFirstVal && |
967 | Val.getValueID() <= Value::ConstantAggregateLastVal; |
968 | } |
969 | }; |
970 | |
971 | template <> struct isa_impl<Argument, Value> { |
972 | static inline bool doit (const Value &Val) { |
973 | return Val.getValueID() == Value::ArgumentVal; |
974 | } |
975 | }; |
976 | |
977 | template <> struct isa_impl<InlineAsm, Value> { |
978 | static inline bool doit(const Value &Val) { |
979 | return Val.getValueID() == Value::InlineAsmVal; |
980 | } |
981 | }; |
982 | |
983 | template <> struct isa_impl<Instruction, Value> { |
984 | static inline bool doit(const Value &Val) { |
985 | return Val.getValueID() >= Value::InstructionVal; |
986 | } |
987 | }; |
988 | |
989 | template <> struct isa_impl<BasicBlock, Value> { |
990 | static inline bool doit(const Value &Val) { |
991 | return Val.getValueID() == Value::BasicBlockVal; |
992 | } |
993 | }; |
994 | |
995 | template <> struct isa_impl<Function, Value> { |
996 | static inline bool doit(const Value &Val) { |
997 | return Val.getValueID() == Value::FunctionVal; |
998 | } |
999 | }; |
1000 | |
1001 | template <> struct isa_impl<GlobalVariable, Value> { |
1002 | static inline bool doit(const Value &Val) { |
1003 | return Val.getValueID() == Value::GlobalVariableVal; |
1004 | } |
1005 | }; |
1006 | |
1007 | template <> struct isa_impl<GlobalAlias, Value> { |
1008 | static inline bool doit(const Value &Val) { |
1009 | return Val.getValueID() == Value::GlobalAliasVal; |
1010 | } |
1011 | }; |
1012 | |
1013 | template <> struct isa_impl<GlobalIFunc, Value> { |
1014 | static inline bool doit(const Value &Val) { |
1015 | return Val.getValueID() == Value::GlobalIFuncVal; |
1016 | } |
1017 | }; |
1018 | |
1019 | template <> struct isa_impl<GlobalIndirectSymbol, Value> { |
1020 | static inline bool doit(const Value &Val) { |
1021 | return isa<GlobalAlias>(Val) || isa<GlobalIFunc>(Val); |
1022 | } |
1023 | }; |
1024 | |
1025 | template <> struct isa_impl<GlobalValue, Value> { |
1026 | static inline bool doit(const Value &Val) { |
1027 | return isa<GlobalObject>(Val) || isa<GlobalIndirectSymbol>(Val); |
1028 | } |
1029 | }; |
1030 | |
1031 | template <> struct isa_impl<GlobalObject, Value> { |
1032 | static inline bool doit(const Value &Val) { |
1033 | return isa<GlobalVariable>(Val) || isa<Function>(Val); |
1034 | } |
1035 | }; |
1036 | |
1037 | // Create wrappers for C Binding types (see CBindingWrapping.h). |
1038 | DEFINE_ISA_CONVERSION_FUNCTIONS(Value, LLVMValueRef)inline Value *unwrap(LLVMValueRef P) { return reinterpret_cast <Value*>(P); } inline LLVMValueRef wrap(const Value *P) { return reinterpret_cast<LLVMValueRef>(const_cast< Value*>(P)); } template<typename T> inline T *unwrap (LLVMValueRef P) { return cast<T>(unwrap(P)); } |
1039 | |
1040 | // Specialized opaque value conversions. |
1041 | inline Value **unwrap(LLVMValueRef *Vals) { |
1042 | return reinterpret_cast<Value**>(Vals); |
1043 | } |
1044 | |
1045 | template<typename T> |
1046 | inline T **unwrap(LLVMValueRef *Vals, unsigned Length) { |
1047 | #ifndef NDEBUG |
1048 | for (LLVMValueRef *I = Vals, *E = Vals + Length; I != E; ++I) |
1049 | unwrap<T>(*I); // For side effect of calling assert on invalid usage. |
1050 | #endif |
1051 | (void)Length; |
1052 | return reinterpret_cast<T**>(Vals); |
1053 | } |
1054 | |
1055 | inline LLVMValueRef *wrap(const Value **Vals) { |
1056 | return reinterpret_cast<LLVMValueRef*>(const_cast<Value**>(Vals)); |
1057 | } |
1058 | |
1059 | } // end namespace llvm |
1060 | |
1061 | #endif // LLVM_IR_VALUE_H |