File: | build/source/llvm/lib/Analysis/InstructionSimplify.cpp |
Warning: | line 6535, column 9 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- InstructionSimplify.cpp - Fold instruction operands ----------------===// | ||||
2 | // | ||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||
6 | // | ||||
7 | //===----------------------------------------------------------------------===// | ||||
8 | // | ||||
9 | // This file implements routines for folding instructions into simpler forms | ||||
10 | // that do not require creating new instructions. This does constant folding | ||||
11 | // ("add i32 1, 1" -> "2") but can also handle non-constant operands, either | ||||
12 | // returning a constant ("and i32 %x, 0" -> "0") or an already existing value | ||||
13 | // ("and i32 %x, %x" -> "%x"). All operands are assumed to have already been | ||||
14 | // simplified: This is usually true and assuming it simplifies the logic (if | ||||
15 | // they have not been simplified then results are correct but maybe suboptimal). | ||||
16 | // | ||||
17 | //===----------------------------------------------------------------------===// | ||||
18 | |||||
19 | #include "llvm/Analysis/InstructionSimplify.h" | ||||
20 | |||||
21 | #include "llvm/ADT/STLExtras.h" | ||||
22 | #include "llvm/ADT/SetVector.h" | ||||
23 | #include "llvm/ADT/Statistic.h" | ||||
24 | #include "llvm/Analysis/AliasAnalysis.h" | ||||
25 | #include "llvm/Analysis/AssumptionCache.h" | ||||
26 | #include "llvm/Analysis/CaptureTracking.h" | ||||
27 | #include "llvm/Analysis/CmpInstAnalysis.h" | ||||
28 | #include "llvm/Analysis/ConstantFolding.h" | ||||
29 | #include "llvm/Analysis/InstSimplifyFolder.h" | ||||
30 | #include "llvm/Analysis/LoopAnalysisManager.h" | ||||
31 | #include "llvm/Analysis/MemoryBuiltins.h" | ||||
32 | #include "llvm/Analysis/OverflowInstAnalysis.h" | ||||
33 | #include "llvm/Analysis/ValueTracking.h" | ||||
34 | #include "llvm/Analysis/VectorUtils.h" | ||||
35 | #include "llvm/IR/ConstantRange.h" | ||||
36 | #include "llvm/IR/DataLayout.h" | ||||
37 | #include "llvm/IR/Dominators.h" | ||||
38 | #include "llvm/IR/InstrTypes.h" | ||||
39 | #include "llvm/IR/Instructions.h" | ||||
40 | #include "llvm/IR/Operator.h" | ||||
41 | #include "llvm/IR/PatternMatch.h" | ||||
42 | #include "llvm/Support/KnownBits.h" | ||||
43 | #include <algorithm> | ||||
44 | #include <optional> | ||||
45 | using namespace llvm; | ||||
46 | using namespace llvm::PatternMatch; | ||||
47 | |||||
48 | #define DEBUG_TYPE"instsimplify" "instsimplify" | ||||
49 | |||||
50 | enum { RecursionLimit = 3 }; | ||||
51 | |||||
52 | STATISTIC(NumExpand, "Number of expansions")static llvm::Statistic NumExpand = {"instsimplify", "NumExpand" , "Number of expansions"}; | ||||
53 | STATISTIC(NumReassoc, "Number of reassociations")static llvm::Statistic NumReassoc = {"instsimplify", "NumReassoc" , "Number of reassociations"}; | ||||
54 | |||||
55 | static Value *simplifyAndInst(Value *, Value *, const SimplifyQuery &, | ||||
56 | unsigned); | ||||
57 | static Value *simplifyUnOp(unsigned, Value *, const SimplifyQuery &, unsigned); | ||||
58 | static Value *simplifyFPUnOp(unsigned, Value *, const FastMathFlags &, | ||||
59 | const SimplifyQuery &, unsigned); | ||||
60 | static Value *simplifyBinOp(unsigned, Value *, Value *, const SimplifyQuery &, | ||||
61 | unsigned); | ||||
62 | static Value *simplifyBinOp(unsigned, Value *, Value *, const FastMathFlags &, | ||||
63 | const SimplifyQuery &, unsigned); | ||||
64 | static Value *simplifyCmpInst(unsigned, Value *, Value *, const SimplifyQuery &, | ||||
65 | unsigned); | ||||
66 | static Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, | ||||
67 | const SimplifyQuery &Q, unsigned MaxRecurse); | ||||
68 | static Value *simplifyOrInst(Value *, Value *, const SimplifyQuery &, unsigned); | ||||
69 | static Value *simplifyXorInst(Value *, Value *, const SimplifyQuery &, | ||||
70 | unsigned); | ||||
71 | static Value *simplifyCastInst(unsigned, Value *, Type *, const SimplifyQuery &, | ||||
72 | unsigned); | ||||
73 | static Value *simplifyGEPInst(Type *, Value *, ArrayRef<Value *>, bool, | ||||
74 | const SimplifyQuery &, unsigned); | ||||
75 | static Value *simplifySelectInst(Value *, Value *, Value *, | ||||
76 | const SimplifyQuery &, unsigned); | ||||
77 | |||||
78 | static Value *foldSelectWithBinaryOp(Value *Cond, Value *TrueVal, | ||||
79 | Value *FalseVal) { | ||||
80 | BinaryOperator::BinaryOps BinOpCode; | ||||
81 | if (auto *BO = dyn_cast<BinaryOperator>(Cond)) | ||||
82 | BinOpCode = BO->getOpcode(); | ||||
83 | else | ||||
84 | return nullptr; | ||||
85 | |||||
86 | CmpInst::Predicate ExpectedPred, Pred1, Pred2; | ||||
87 | if (BinOpCode == BinaryOperator::Or) { | ||||
88 | ExpectedPred = ICmpInst::ICMP_NE; | ||||
89 | } else if (BinOpCode == BinaryOperator::And) { | ||||
90 | ExpectedPred = ICmpInst::ICMP_EQ; | ||||
91 | } else | ||||
92 | return nullptr; | ||||
93 | |||||
94 | // %A = icmp eq %TV, %FV | ||||
95 | // %B = icmp eq %X, %Y (and one of these is a select operand) | ||||
96 | // %C = and %A, %B | ||||
97 | // %D = select %C, %TV, %FV | ||||
98 | // --> | ||||
99 | // %FV | ||||
100 | |||||
101 | // %A = icmp ne %TV, %FV | ||||
102 | // %B = icmp ne %X, %Y (and one of these is a select operand) | ||||
103 | // %C = or %A, %B | ||||
104 | // %D = select %C, %TV, %FV | ||||
105 | // --> | ||||
106 | // %TV | ||||
107 | Value *X, *Y; | ||||
108 | if (!match(Cond, m_c_BinOp(m_c_ICmp(Pred1, m_Specific(TrueVal), | ||||
109 | m_Specific(FalseVal)), | ||||
110 | m_ICmp(Pred2, m_Value(X), m_Value(Y)))) || | ||||
111 | Pred1 != Pred2 || Pred1 != ExpectedPred) | ||||
112 | return nullptr; | ||||
113 | |||||
114 | if (X == TrueVal || X == FalseVal || Y == TrueVal || Y == FalseVal) | ||||
115 | return BinOpCode == BinaryOperator::Or ? TrueVal : FalseVal; | ||||
116 | |||||
117 | return nullptr; | ||||
118 | } | ||||
119 | |||||
120 | /// For a boolean type or a vector of boolean type, return false or a vector | ||||
121 | /// with every element false. | ||||
122 | static Constant *getFalse(Type *Ty) { return ConstantInt::getFalse(Ty); } | ||||
123 | |||||
124 | /// For a boolean type or a vector of boolean type, return true or a vector | ||||
125 | /// with every element true. | ||||
126 | static Constant *getTrue(Type *Ty) { return ConstantInt::getTrue(Ty); } | ||||
127 | |||||
128 | /// isSameCompare - Is V equivalent to the comparison "LHS Pred RHS"? | ||||
129 | static bool isSameCompare(Value *V, CmpInst::Predicate Pred, Value *LHS, | ||||
130 | Value *RHS) { | ||||
131 | CmpInst *Cmp = dyn_cast<CmpInst>(V); | ||||
132 | if (!Cmp) | ||||
133 | return false; | ||||
134 | CmpInst::Predicate CPred = Cmp->getPredicate(); | ||||
135 | Value *CLHS = Cmp->getOperand(0), *CRHS = Cmp->getOperand(1); | ||||
136 | if (CPred == Pred && CLHS == LHS && CRHS == RHS) | ||||
137 | return true; | ||||
138 | return CPred == CmpInst::getSwappedPredicate(Pred) && CLHS == RHS && | ||||
139 | CRHS == LHS; | ||||
140 | } | ||||
141 | |||||
142 | /// Simplify comparison with true or false branch of select: | ||||
143 | /// %sel = select i1 %cond, i32 %tv, i32 %fv | ||||
144 | /// %cmp = icmp sle i32 %sel, %rhs | ||||
145 | /// Compose new comparison by substituting %sel with either %tv or %fv | ||||
146 | /// and see if it simplifies. | ||||
147 | static Value *simplifyCmpSelCase(CmpInst::Predicate Pred, Value *LHS, | ||||
148 | Value *RHS, Value *Cond, | ||||
149 | const SimplifyQuery &Q, unsigned MaxRecurse, | ||||
150 | Constant *TrueOrFalse) { | ||||
151 | Value *SimplifiedCmp = simplifyCmpInst(Pred, LHS, RHS, Q, MaxRecurse); | ||||
152 | if (SimplifiedCmp == Cond) { | ||||
153 | // %cmp simplified to the select condition (%cond). | ||||
154 | return TrueOrFalse; | ||||
155 | } else if (!SimplifiedCmp && isSameCompare(Cond, Pred, LHS, RHS)) { | ||||
156 | // It didn't simplify. However, if composed comparison is equivalent | ||||
157 | // to the select condition (%cond) then we can replace it. | ||||
158 | return TrueOrFalse; | ||||
159 | } | ||||
160 | return SimplifiedCmp; | ||||
161 | } | ||||
162 | |||||
163 | /// Simplify comparison with true branch of select | ||||
164 | static Value *simplifyCmpSelTrueCase(CmpInst::Predicate Pred, Value *LHS, | ||||
165 | Value *RHS, Value *Cond, | ||||
166 | const SimplifyQuery &Q, | ||||
167 | unsigned MaxRecurse) { | ||||
168 | return simplifyCmpSelCase(Pred, LHS, RHS, Cond, Q, MaxRecurse, | ||||
169 | getTrue(Cond->getType())); | ||||
170 | } | ||||
171 | |||||
172 | /// Simplify comparison with false branch of select | ||||
173 | static Value *simplifyCmpSelFalseCase(CmpInst::Predicate Pred, Value *LHS, | ||||
174 | Value *RHS, Value *Cond, | ||||
175 | const SimplifyQuery &Q, | ||||
176 | unsigned MaxRecurse) { | ||||
177 | return simplifyCmpSelCase(Pred, LHS, RHS, Cond, Q, MaxRecurse, | ||||
178 | getFalse(Cond->getType())); | ||||
179 | } | ||||
180 | |||||
181 | /// We know comparison with both branches of select can be simplified, but they | ||||
182 | /// are not equal. This routine handles some logical simplifications. | ||||
183 | static Value *handleOtherCmpSelSimplifications(Value *TCmp, Value *FCmp, | ||||
184 | Value *Cond, | ||||
185 | const SimplifyQuery &Q, | ||||
186 | unsigned MaxRecurse) { | ||||
187 | // If the false value simplified to false, then the result of the compare | ||||
188 | // is equal to "Cond && TCmp". This also catches the case when the false | ||||
189 | // value simplified to false and the true value to true, returning "Cond". | ||||
190 | // Folding select to and/or isn't poison-safe in general; impliesPoison | ||||
191 | // checks whether folding it does not convert a well-defined value into | ||||
192 | // poison. | ||||
193 | if (match(FCmp, m_Zero()) && impliesPoison(TCmp, Cond)) | ||||
194 | if (Value *V = simplifyAndInst(Cond, TCmp, Q, MaxRecurse)) | ||||
195 | return V; | ||||
196 | // If the true value simplified to true, then the result of the compare | ||||
197 | // is equal to "Cond || FCmp". | ||||
198 | if (match(TCmp, m_One()) && impliesPoison(FCmp, Cond)) | ||||
199 | if (Value *V = simplifyOrInst(Cond, FCmp, Q, MaxRecurse)) | ||||
200 | return V; | ||||
201 | // Finally, if the false value simplified to true and the true value to | ||||
202 | // false, then the result of the compare is equal to "!Cond". | ||||
203 | if (match(FCmp, m_One()) && match(TCmp, m_Zero())) | ||||
204 | if (Value *V = simplifyXorInst( | ||||
205 | Cond, Constant::getAllOnesValue(Cond->getType()), Q, MaxRecurse)) | ||||
206 | return V; | ||||
207 | return nullptr; | ||||
208 | } | ||||
209 | |||||
210 | /// Does the given value dominate the specified phi node? | ||||
211 | static bool valueDominatesPHI(Value *V, PHINode *P, const DominatorTree *DT) { | ||||
212 | Instruction *I = dyn_cast<Instruction>(V); | ||||
213 | if (!I) | ||||
214 | // Arguments and constants dominate all instructions. | ||||
215 | return true; | ||||
216 | |||||
217 | // If we are processing instructions (and/or basic blocks) that have not been | ||||
218 | // fully added to a function, the parent nodes may still be null. Simply | ||||
219 | // return the conservative answer in these cases. | ||||
220 | if (!I->getParent() || !P->getParent() || !I->getFunction()) | ||||
221 | return false; | ||||
222 | |||||
223 | // If we have a DominatorTree then do a precise test. | ||||
224 | if (DT) | ||||
225 | return DT->dominates(I, P); | ||||
226 | |||||
227 | // Otherwise, if the instruction is in the entry block and is not an invoke, | ||||
228 | // then it obviously dominates all phi nodes. | ||||
229 | if (I->getParent()->isEntryBlock() && !isa<InvokeInst>(I) && | ||||
230 | !isa<CallBrInst>(I)) | ||||
231 | return true; | ||||
232 | |||||
233 | return false; | ||||
234 | } | ||||
235 | |||||
236 | /// Try to simplify a binary operator of form "V op OtherOp" where V is | ||||
237 | /// "(B0 opex B1)" by distributing 'op' across 'opex' as | ||||
238 | /// "(B0 op OtherOp) opex (B1 op OtherOp)". | ||||
239 | static Value *expandBinOp(Instruction::BinaryOps Opcode, Value *V, | ||||
240 | Value *OtherOp, Instruction::BinaryOps OpcodeToExpand, | ||||
241 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
242 | auto *B = dyn_cast<BinaryOperator>(V); | ||||
243 | if (!B || B->getOpcode() != OpcodeToExpand) | ||||
244 | return nullptr; | ||||
245 | Value *B0 = B->getOperand(0), *B1 = B->getOperand(1); | ||||
246 | Value *L = | ||||
247 | simplifyBinOp(Opcode, B0, OtherOp, Q.getWithoutUndef(), MaxRecurse); | ||||
248 | if (!L) | ||||
249 | return nullptr; | ||||
250 | Value *R = | ||||
251 | simplifyBinOp(Opcode, B1, OtherOp, Q.getWithoutUndef(), MaxRecurse); | ||||
252 | if (!R) | ||||
253 | return nullptr; | ||||
254 | |||||
255 | // Does the expanded pair of binops simplify to the existing binop? | ||||
256 | if ((L == B0 && R == B1) || | ||||
257 | (Instruction::isCommutative(OpcodeToExpand) && L == B1 && R == B0)) { | ||||
258 | ++NumExpand; | ||||
259 | return B; | ||||
260 | } | ||||
261 | |||||
262 | // Otherwise, return "L op' R" if it simplifies. | ||||
263 | Value *S = simplifyBinOp(OpcodeToExpand, L, R, Q, MaxRecurse); | ||||
264 | if (!S) | ||||
265 | return nullptr; | ||||
266 | |||||
267 | ++NumExpand; | ||||
268 | return S; | ||||
269 | } | ||||
270 | |||||
271 | /// Try to simplify binops of form "A op (B op' C)" or the commuted variant by | ||||
272 | /// distributing op over op'. | ||||
273 | static Value *expandCommutativeBinOp(Instruction::BinaryOps Opcode, Value *L, | ||||
274 | Value *R, | ||||
275 | Instruction::BinaryOps OpcodeToExpand, | ||||
276 | const SimplifyQuery &Q, | ||||
277 | unsigned MaxRecurse) { | ||||
278 | // Recursion is always used, so bail out at once if we already hit the limit. | ||||
279 | if (!MaxRecurse--) | ||||
280 | return nullptr; | ||||
281 | |||||
282 | if (Value *V = expandBinOp(Opcode, L, R, OpcodeToExpand, Q, MaxRecurse)) | ||||
283 | return V; | ||||
284 | if (Value *V = expandBinOp(Opcode, R, L, OpcodeToExpand, Q, MaxRecurse)) | ||||
285 | return V; | ||||
286 | return nullptr; | ||||
287 | } | ||||
288 | |||||
289 | /// Generic simplifications for associative binary operations. | ||||
290 | /// Returns the simpler value, or null if none was found. | ||||
291 | static Value *simplifyAssociativeBinOp(Instruction::BinaryOps Opcode, | ||||
292 | Value *LHS, Value *RHS, | ||||
293 | const SimplifyQuery &Q, | ||||
294 | unsigned MaxRecurse) { | ||||
295 | assert(Instruction::isAssociative(Opcode) && "Not an associative operation!")(static_cast <bool> (Instruction::isAssociative(Opcode) && "Not an associative operation!") ? void (0) : __assert_fail ("Instruction::isAssociative(Opcode) && \"Not an associative operation!\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 295, __extension__ __PRETTY_FUNCTION__)); | ||||
296 | |||||
297 | // Recursion is always used, so bail out at once if we already hit the limit. | ||||
298 | if (!MaxRecurse--) | ||||
299 | return nullptr; | ||||
300 | |||||
301 | BinaryOperator *Op0 = dyn_cast<BinaryOperator>(LHS); | ||||
302 | BinaryOperator *Op1 = dyn_cast<BinaryOperator>(RHS); | ||||
303 | |||||
304 | // Transform: "(A op B) op C" ==> "A op (B op C)" if it simplifies completely. | ||||
305 | if (Op0 && Op0->getOpcode() == Opcode) { | ||||
306 | Value *A = Op0->getOperand(0); | ||||
307 | Value *B = Op0->getOperand(1); | ||||
308 | Value *C = RHS; | ||||
309 | |||||
310 | // Does "B op C" simplify? | ||||
311 | if (Value *V = simplifyBinOp(Opcode, B, C, Q, MaxRecurse)) { | ||||
312 | // It does! Return "A op V" if it simplifies or is already available. | ||||
313 | // If V equals B then "A op V" is just the LHS. | ||||
314 | if (V == B) | ||||
315 | return LHS; | ||||
316 | // Otherwise return "A op V" if it simplifies. | ||||
317 | if (Value *W = simplifyBinOp(Opcode, A, V, Q, MaxRecurse)) { | ||||
318 | ++NumReassoc; | ||||
319 | return W; | ||||
320 | } | ||||
321 | } | ||||
322 | } | ||||
323 | |||||
324 | // Transform: "A op (B op C)" ==> "(A op B) op C" if it simplifies completely. | ||||
325 | if (Op1 && Op1->getOpcode() == Opcode) { | ||||
326 | Value *A = LHS; | ||||
327 | Value *B = Op1->getOperand(0); | ||||
328 | Value *C = Op1->getOperand(1); | ||||
329 | |||||
330 | // Does "A op B" simplify? | ||||
331 | if (Value *V = simplifyBinOp(Opcode, A, B, Q, MaxRecurse)) { | ||||
332 | // It does! Return "V op C" if it simplifies or is already available. | ||||
333 | // If V equals B then "V op C" is just the RHS. | ||||
334 | if (V == B) | ||||
335 | return RHS; | ||||
336 | // Otherwise return "V op C" if it simplifies. | ||||
337 | if (Value *W = simplifyBinOp(Opcode, V, C, Q, MaxRecurse)) { | ||||
338 | ++NumReassoc; | ||||
339 | return W; | ||||
340 | } | ||||
341 | } | ||||
342 | } | ||||
343 | |||||
344 | // The remaining transforms require commutativity as well as associativity. | ||||
345 | if (!Instruction::isCommutative(Opcode)) | ||||
346 | return nullptr; | ||||
347 | |||||
348 | // Transform: "(A op B) op C" ==> "(C op A) op B" if it simplifies completely. | ||||
349 | if (Op0 && Op0->getOpcode() == Opcode) { | ||||
350 | Value *A = Op0->getOperand(0); | ||||
351 | Value *B = Op0->getOperand(1); | ||||
352 | Value *C = RHS; | ||||
353 | |||||
354 | // Does "C op A" simplify? | ||||
355 | if (Value *V = simplifyBinOp(Opcode, C, A, Q, MaxRecurse)) { | ||||
356 | // It does! Return "V op B" if it simplifies or is already available. | ||||
357 | // If V equals A then "V op B" is just the LHS. | ||||
358 | if (V == A) | ||||
359 | return LHS; | ||||
360 | // Otherwise return "V op B" if it simplifies. | ||||
361 | if (Value *W = simplifyBinOp(Opcode, V, B, Q, MaxRecurse)) { | ||||
362 | ++NumReassoc; | ||||
363 | return W; | ||||
364 | } | ||||
365 | } | ||||
366 | } | ||||
367 | |||||
368 | // Transform: "A op (B op C)" ==> "B op (C op A)" if it simplifies completely. | ||||
369 | if (Op1 && Op1->getOpcode() == Opcode) { | ||||
370 | Value *A = LHS; | ||||
371 | Value *B = Op1->getOperand(0); | ||||
372 | Value *C = Op1->getOperand(1); | ||||
373 | |||||
374 | // Does "C op A" simplify? | ||||
375 | if (Value *V = simplifyBinOp(Opcode, C, A, Q, MaxRecurse)) { | ||||
376 | // It does! Return "B op V" if it simplifies or is already available. | ||||
377 | // If V equals C then "B op V" is just the RHS. | ||||
378 | if (V == C) | ||||
379 | return RHS; | ||||
380 | // Otherwise return "B op V" if it simplifies. | ||||
381 | if (Value *W = simplifyBinOp(Opcode, B, V, Q, MaxRecurse)) { | ||||
382 | ++NumReassoc; | ||||
383 | return W; | ||||
384 | } | ||||
385 | } | ||||
386 | } | ||||
387 | |||||
388 | return nullptr; | ||||
389 | } | ||||
390 | |||||
391 | /// In the case of a binary operation with a select instruction as an operand, | ||||
392 | /// try to simplify the binop by seeing whether evaluating it on both branches | ||||
393 | /// of the select results in the same value. Returns the common value if so, | ||||
394 | /// otherwise returns null. | ||||
395 | static Value *threadBinOpOverSelect(Instruction::BinaryOps Opcode, Value *LHS, | ||||
396 | Value *RHS, const SimplifyQuery &Q, | ||||
397 | unsigned MaxRecurse) { | ||||
398 | // Recursion is always used, so bail out at once if we already hit the limit. | ||||
399 | if (!MaxRecurse--) | ||||
400 | return nullptr; | ||||
401 | |||||
402 | SelectInst *SI; | ||||
403 | if (isa<SelectInst>(LHS)) { | ||||
404 | SI = cast<SelectInst>(LHS); | ||||
405 | } else { | ||||
406 | assert(isa<SelectInst>(RHS) && "No select instruction operand!")(static_cast <bool> (isa<SelectInst>(RHS) && "No select instruction operand!") ? void (0) : __assert_fail ("isa<SelectInst>(RHS) && \"No select instruction operand!\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 406, __extension__ __PRETTY_FUNCTION__)); | ||||
407 | SI = cast<SelectInst>(RHS); | ||||
408 | } | ||||
409 | |||||
410 | // Evaluate the BinOp on the true and false branches of the select. | ||||
411 | Value *TV; | ||||
412 | Value *FV; | ||||
413 | if (SI == LHS) { | ||||
414 | TV = simplifyBinOp(Opcode, SI->getTrueValue(), RHS, Q, MaxRecurse); | ||||
415 | FV = simplifyBinOp(Opcode, SI->getFalseValue(), RHS, Q, MaxRecurse); | ||||
416 | } else { | ||||
417 | TV = simplifyBinOp(Opcode, LHS, SI->getTrueValue(), Q, MaxRecurse); | ||||
418 | FV = simplifyBinOp(Opcode, LHS, SI->getFalseValue(), Q, MaxRecurse); | ||||
419 | } | ||||
420 | |||||
421 | // If they simplified to the same value, then return the common value. | ||||
422 | // If they both failed to simplify then return null. | ||||
423 | if (TV == FV) | ||||
424 | return TV; | ||||
425 | |||||
426 | // If one branch simplified to undef, return the other one. | ||||
427 | if (TV && Q.isUndefValue(TV)) | ||||
428 | return FV; | ||||
429 | if (FV && Q.isUndefValue(FV)) | ||||
430 | return TV; | ||||
431 | |||||
432 | // If applying the operation did not change the true and false select values, | ||||
433 | // then the result of the binop is the select itself. | ||||
434 | if (TV == SI->getTrueValue() && FV == SI->getFalseValue()) | ||||
435 | return SI; | ||||
436 | |||||
437 | // If one branch simplified and the other did not, and the simplified | ||||
438 | // value is equal to the unsimplified one, return the simplified value. | ||||
439 | // For example, select (cond, X, X & Z) & Z -> X & Z. | ||||
440 | if ((FV && !TV) || (TV && !FV)) { | ||||
441 | // Check that the simplified value has the form "X op Y" where "op" is the | ||||
442 | // same as the original operation. | ||||
443 | Instruction *Simplified = dyn_cast<Instruction>(FV ? FV : TV); | ||||
444 | if (Simplified && Simplified->getOpcode() == unsigned(Opcode)) { | ||||
445 | // The value that didn't simplify is "UnsimplifiedLHS op UnsimplifiedRHS". | ||||
446 | // We already know that "op" is the same as for the simplified value. See | ||||
447 | // if the operands match too. If so, return the simplified value. | ||||
448 | Value *UnsimplifiedBranch = FV ? SI->getTrueValue() : SI->getFalseValue(); | ||||
449 | Value *UnsimplifiedLHS = SI == LHS ? UnsimplifiedBranch : LHS; | ||||
450 | Value *UnsimplifiedRHS = SI == LHS ? RHS : UnsimplifiedBranch; | ||||
451 | if (Simplified->getOperand(0) == UnsimplifiedLHS && | ||||
452 | Simplified->getOperand(1) == UnsimplifiedRHS) | ||||
453 | return Simplified; | ||||
454 | if (Simplified->isCommutative() && | ||||
455 | Simplified->getOperand(1) == UnsimplifiedLHS && | ||||
456 | Simplified->getOperand(0) == UnsimplifiedRHS) | ||||
457 | return Simplified; | ||||
458 | } | ||||
459 | } | ||||
460 | |||||
461 | return nullptr; | ||||
462 | } | ||||
463 | |||||
464 | /// In the case of a comparison with a select instruction, try to simplify the | ||||
465 | /// comparison by seeing whether both branches of the select result in the same | ||||
466 | /// value. Returns the common value if so, otherwise returns null. | ||||
467 | /// For example, if we have: | ||||
468 | /// %tmp = select i1 %cmp, i32 1, i32 2 | ||||
469 | /// %cmp1 = icmp sle i32 %tmp, 3 | ||||
470 | /// We can simplify %cmp1 to true, because both branches of select are | ||||
471 | /// less than 3. We compose new comparison by substituting %tmp with both | ||||
472 | /// branches of select and see if it can be simplified. | ||||
473 | static Value *threadCmpOverSelect(CmpInst::Predicate Pred, Value *LHS, | ||||
474 | Value *RHS, const SimplifyQuery &Q, | ||||
475 | unsigned MaxRecurse) { | ||||
476 | // Recursion is always used, so bail out at once if we already hit the limit. | ||||
477 | if (!MaxRecurse--) | ||||
478 | return nullptr; | ||||
479 | |||||
480 | // Make sure the select is on the LHS. | ||||
481 | if (!isa<SelectInst>(LHS)) { | ||||
482 | std::swap(LHS, RHS); | ||||
483 | Pred = CmpInst::getSwappedPredicate(Pred); | ||||
484 | } | ||||
485 | assert(isa<SelectInst>(LHS) && "Not comparing with a select instruction!")(static_cast <bool> (isa<SelectInst>(LHS) && "Not comparing with a select instruction!") ? void (0) : __assert_fail ("isa<SelectInst>(LHS) && \"Not comparing with a select instruction!\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 485, __extension__ __PRETTY_FUNCTION__)); | ||||
486 | SelectInst *SI = cast<SelectInst>(LHS); | ||||
487 | Value *Cond = SI->getCondition(); | ||||
488 | Value *TV = SI->getTrueValue(); | ||||
489 | Value *FV = SI->getFalseValue(); | ||||
490 | |||||
491 | // Now that we have "cmp select(Cond, TV, FV), RHS", analyse it. | ||||
492 | // Does "cmp TV, RHS" simplify? | ||||
493 | Value *TCmp = simplifyCmpSelTrueCase(Pred, TV, RHS, Cond, Q, MaxRecurse); | ||||
494 | if (!TCmp) | ||||
495 | return nullptr; | ||||
496 | |||||
497 | // Does "cmp FV, RHS" simplify? | ||||
498 | Value *FCmp = simplifyCmpSelFalseCase(Pred, FV, RHS, Cond, Q, MaxRecurse); | ||||
499 | if (!FCmp) | ||||
500 | return nullptr; | ||||
501 | |||||
502 | // If both sides simplified to the same value, then use it as the result of | ||||
503 | // the original comparison. | ||||
504 | if (TCmp == FCmp) | ||||
505 | return TCmp; | ||||
506 | |||||
507 | // The remaining cases only make sense if the select condition has the same | ||||
508 | // type as the result of the comparison, so bail out if this is not so. | ||||
509 | if (Cond->getType()->isVectorTy() == RHS->getType()->isVectorTy()) | ||||
510 | return handleOtherCmpSelSimplifications(TCmp, FCmp, Cond, Q, MaxRecurse); | ||||
511 | |||||
512 | return nullptr; | ||||
513 | } | ||||
514 | |||||
515 | /// In the case of a binary operation with an operand that is a PHI instruction, | ||||
516 | /// try to simplify the binop by seeing whether evaluating it on the incoming | ||||
517 | /// phi values yields the same result for every value. If so returns the common | ||||
518 | /// value, otherwise returns null. | ||||
519 | static Value *threadBinOpOverPHI(Instruction::BinaryOps Opcode, Value *LHS, | ||||
520 | Value *RHS, const SimplifyQuery &Q, | ||||
521 | unsigned MaxRecurse) { | ||||
522 | // Recursion is always used, so bail out at once if we already hit the limit. | ||||
523 | if (!MaxRecurse--) | ||||
524 | return nullptr; | ||||
525 | |||||
526 | PHINode *PI; | ||||
527 | if (isa<PHINode>(LHS)) { | ||||
528 | PI = cast<PHINode>(LHS); | ||||
529 | // Bail out if RHS and the phi may be mutually interdependent due to a loop. | ||||
530 | if (!valueDominatesPHI(RHS, PI, Q.DT)) | ||||
531 | return nullptr; | ||||
532 | } else { | ||||
533 | assert(isa<PHINode>(RHS) && "No PHI instruction operand!")(static_cast <bool> (isa<PHINode>(RHS) && "No PHI instruction operand!") ? void (0) : __assert_fail ("isa<PHINode>(RHS) && \"No PHI instruction operand!\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 533, __extension__ __PRETTY_FUNCTION__)); | ||||
534 | PI = cast<PHINode>(RHS); | ||||
535 | // Bail out if LHS and the phi may be mutually interdependent due to a loop. | ||||
536 | if (!valueDominatesPHI(LHS, PI, Q.DT)) | ||||
537 | return nullptr; | ||||
538 | } | ||||
539 | |||||
540 | // Evaluate the BinOp on the incoming phi values. | ||||
541 | Value *CommonValue = nullptr; | ||||
542 | for (Use &Incoming : PI->incoming_values()) { | ||||
543 | // If the incoming value is the phi node itself, it can safely be skipped. | ||||
544 | if (Incoming == PI) | ||||
545 | continue; | ||||
546 | Instruction *InTI = PI->getIncomingBlock(Incoming)->getTerminator(); | ||||
547 | Value *V = PI == LHS | ||||
548 | ? simplifyBinOp(Opcode, Incoming, RHS, | ||||
549 | Q.getWithInstruction(InTI), MaxRecurse) | ||||
550 | : simplifyBinOp(Opcode, LHS, Incoming, | ||||
551 | Q.getWithInstruction(InTI), MaxRecurse); | ||||
552 | // If the operation failed to simplify, or simplified to a different value | ||||
553 | // to previously, then give up. | ||||
554 | if (!V || (CommonValue && V != CommonValue)) | ||||
555 | return nullptr; | ||||
556 | CommonValue = V; | ||||
557 | } | ||||
558 | |||||
559 | return CommonValue; | ||||
560 | } | ||||
561 | |||||
562 | /// In the case of a comparison with a PHI instruction, try to simplify the | ||||
563 | /// comparison by seeing whether comparing with all of the incoming phi values | ||||
564 | /// yields the same result every time. If so returns the common result, | ||||
565 | /// otherwise returns null. | ||||
566 | static Value *threadCmpOverPHI(CmpInst::Predicate Pred, Value *LHS, Value *RHS, | ||||
567 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
568 | // Recursion is always used, so bail out at once if we already hit the limit. | ||||
569 | if (!MaxRecurse--) | ||||
570 | return nullptr; | ||||
571 | |||||
572 | // Make sure the phi is on the LHS. | ||||
573 | if (!isa<PHINode>(LHS)) { | ||||
574 | std::swap(LHS, RHS); | ||||
575 | Pred = CmpInst::getSwappedPredicate(Pred); | ||||
576 | } | ||||
577 | assert(isa<PHINode>(LHS) && "Not comparing with a phi instruction!")(static_cast <bool> (isa<PHINode>(LHS) && "Not comparing with a phi instruction!") ? void (0) : __assert_fail ("isa<PHINode>(LHS) && \"Not comparing with a phi instruction!\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 577, __extension__ __PRETTY_FUNCTION__)); | ||||
578 | PHINode *PI = cast<PHINode>(LHS); | ||||
579 | |||||
580 | // Bail out if RHS and the phi may be mutually interdependent due to a loop. | ||||
581 | if (!valueDominatesPHI(RHS, PI, Q.DT)) | ||||
582 | return nullptr; | ||||
583 | |||||
584 | // Evaluate the BinOp on the incoming phi values. | ||||
585 | Value *CommonValue = nullptr; | ||||
586 | for (unsigned u = 0, e = PI->getNumIncomingValues(); u < e; ++u) { | ||||
587 | Value *Incoming = PI->getIncomingValue(u); | ||||
588 | Instruction *InTI = PI->getIncomingBlock(u)->getTerminator(); | ||||
589 | // If the incoming value is the phi node itself, it can safely be skipped. | ||||
590 | if (Incoming == PI) | ||||
591 | continue; | ||||
592 | // Change the context instruction to the "edge" that flows into the phi. | ||||
593 | // This is important because that is where incoming is actually "evaluated" | ||||
594 | // even though it is used later somewhere else. | ||||
595 | Value *V = simplifyCmpInst(Pred, Incoming, RHS, Q.getWithInstruction(InTI), | ||||
596 | MaxRecurse); | ||||
597 | // If the operation failed to simplify, or simplified to a different value | ||||
598 | // to previously, then give up. | ||||
599 | if (!V || (CommonValue && V != CommonValue)) | ||||
600 | return nullptr; | ||||
601 | CommonValue = V; | ||||
602 | } | ||||
603 | |||||
604 | return CommonValue; | ||||
605 | } | ||||
606 | |||||
607 | static Constant *foldOrCommuteConstant(Instruction::BinaryOps Opcode, | ||||
608 | Value *&Op0, Value *&Op1, | ||||
609 | const SimplifyQuery &Q) { | ||||
610 | if (auto *CLHS = dyn_cast<Constant>(Op0)) { | ||||
611 | if (auto *CRHS = dyn_cast<Constant>(Op1)) { | ||||
612 | switch (Opcode) { | ||||
613 | default: | ||||
614 | break; | ||||
615 | case Instruction::FAdd: | ||||
616 | case Instruction::FSub: | ||||
617 | case Instruction::FMul: | ||||
618 | case Instruction::FDiv: | ||||
619 | case Instruction::FRem: | ||||
620 | if (Q.CxtI != nullptr) | ||||
621 | return ConstantFoldFPInstOperands(Opcode, CLHS, CRHS, Q.DL, Q.CxtI); | ||||
622 | } | ||||
623 | return ConstantFoldBinaryOpOperands(Opcode, CLHS, CRHS, Q.DL); | ||||
624 | } | ||||
625 | |||||
626 | // Canonicalize the constant to the RHS if this is a commutative operation. | ||||
627 | if (Instruction::isCommutative(Opcode)) | ||||
628 | std::swap(Op0, Op1); | ||||
629 | } | ||||
630 | return nullptr; | ||||
631 | } | ||||
632 | |||||
633 | /// Given operands for an Add, see if we can fold the result. | ||||
634 | /// If not, this returns null. | ||||
635 | static Value *simplifyAddInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW, | ||||
636 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
637 | if (Constant *C = foldOrCommuteConstant(Instruction::Add, Op0, Op1, Q)) | ||||
638 | return C; | ||||
639 | |||||
640 | // X + poison -> poison | ||||
641 | if (isa<PoisonValue>(Op1)) | ||||
642 | return Op1; | ||||
643 | |||||
644 | // X + undef -> undef | ||||
645 | if (Q.isUndefValue(Op1)) | ||||
646 | return Op1; | ||||
647 | |||||
648 | // X + 0 -> X | ||||
649 | if (match(Op1, m_Zero())) | ||||
650 | return Op0; | ||||
651 | |||||
652 | // If two operands are negative, return 0. | ||||
653 | if (isKnownNegation(Op0, Op1)) | ||||
654 | return Constant::getNullValue(Op0->getType()); | ||||
655 | |||||
656 | // X + (Y - X) -> Y | ||||
657 | // (Y - X) + X -> Y | ||||
658 | // Eg: X + -X -> 0 | ||||
659 | Value *Y = nullptr; | ||||
660 | if (match(Op1, m_Sub(m_Value(Y), m_Specific(Op0))) || | ||||
661 | match(Op0, m_Sub(m_Value(Y), m_Specific(Op1)))) | ||||
662 | return Y; | ||||
663 | |||||
664 | // X + ~X -> -1 since ~X = -X-1 | ||||
665 | Type *Ty = Op0->getType(); | ||||
666 | if (match(Op0, m_Not(m_Specific(Op1))) || match(Op1, m_Not(m_Specific(Op0)))) | ||||
667 | return Constant::getAllOnesValue(Ty); | ||||
668 | |||||
669 | // add nsw/nuw (xor Y, signmask), signmask --> Y | ||||
670 | // The no-wrapping add guarantees that the top bit will be set by the add. | ||||
671 | // Therefore, the xor must be clearing the already set sign bit of Y. | ||||
672 | if ((IsNSW || IsNUW) && match(Op1, m_SignMask()) && | ||||
673 | match(Op0, m_Xor(m_Value(Y), m_SignMask()))) | ||||
674 | return Y; | ||||
675 | |||||
676 | // add nuw %x, -1 -> -1, because %x can only be 0. | ||||
677 | if (IsNUW && match(Op1, m_AllOnes())) | ||||
678 | return Op1; // Which is -1. | ||||
679 | |||||
680 | /// i1 add -> xor. | ||||
681 | if (MaxRecurse && Op0->getType()->isIntOrIntVectorTy(1)) | ||||
682 | if (Value *V = simplifyXorInst(Op0, Op1, Q, MaxRecurse - 1)) | ||||
683 | return V; | ||||
684 | |||||
685 | // Try some generic simplifications for associative operations. | ||||
686 | if (Value *V = | ||||
687 | simplifyAssociativeBinOp(Instruction::Add, Op0, Op1, Q, MaxRecurse)) | ||||
688 | return V; | ||||
689 | |||||
690 | // Threading Add over selects and phi nodes is pointless, so don't bother. | ||||
691 | // Threading over the select in "A + select(cond, B, C)" means evaluating | ||||
692 | // "A+B" and "A+C" and seeing if they are equal; but they are equal if and | ||||
693 | // only if B and C are equal. If B and C are equal then (since we assume | ||||
694 | // that operands have already been simplified) "select(cond, B, C)" should | ||||
695 | // have been simplified to the common value of B and C already. Analysing | ||||
696 | // "A+B" and "A+C" thus gains nothing, but costs compile time. Similarly | ||||
697 | // for threading over phi nodes. | ||||
698 | |||||
699 | return nullptr; | ||||
700 | } | ||||
701 | |||||
702 | Value *llvm::simplifyAddInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW, | ||||
703 | const SimplifyQuery &Query) { | ||||
704 | return ::simplifyAddInst(Op0, Op1, IsNSW, IsNUW, Query, RecursionLimit); | ||||
705 | } | ||||
706 | |||||
707 | /// Compute the base pointer and cumulative constant offsets for V. | ||||
708 | /// | ||||
709 | /// This strips all constant offsets off of V, leaving it the base pointer, and | ||||
710 | /// accumulates the total constant offset applied in the returned constant. | ||||
711 | /// It returns zero if there are no constant offsets applied. | ||||
712 | /// | ||||
713 | /// This is very similar to stripAndAccumulateConstantOffsets(), except it | ||||
714 | /// normalizes the offset bitwidth to the stripped pointer type, not the | ||||
715 | /// original pointer type. | ||||
716 | static APInt stripAndComputeConstantOffsets(const DataLayout &DL, Value *&V, | ||||
717 | bool AllowNonInbounds = false) { | ||||
718 | assert(V->getType()->isPtrOrPtrVectorTy())(static_cast <bool> (V->getType()->isPtrOrPtrVectorTy ()) ? void (0) : __assert_fail ("V->getType()->isPtrOrPtrVectorTy()" , "llvm/lib/Analysis/InstructionSimplify.cpp", 718, __extension__ __PRETTY_FUNCTION__)); | ||||
719 | |||||
720 | APInt Offset = APInt::getZero(DL.getIndexTypeSizeInBits(V->getType())); | ||||
721 | V = V->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds); | ||||
722 | // As that strip may trace through `addrspacecast`, need to sext or trunc | ||||
723 | // the offset calculated. | ||||
724 | return Offset.sextOrTrunc(DL.getIndexTypeSizeInBits(V->getType())); | ||||
725 | } | ||||
726 | |||||
727 | /// Compute the constant difference between two pointer values. | ||||
728 | /// If the difference is not a constant, returns zero. | ||||
729 | static Constant *computePointerDifference(const DataLayout &DL, Value *LHS, | ||||
730 | Value *RHS) { | ||||
731 | APInt LHSOffset = stripAndComputeConstantOffsets(DL, LHS); | ||||
732 | APInt RHSOffset = stripAndComputeConstantOffsets(DL, RHS); | ||||
733 | |||||
734 | // If LHS and RHS are not related via constant offsets to the same base | ||||
735 | // value, there is nothing we can do here. | ||||
736 | if (LHS != RHS) | ||||
737 | return nullptr; | ||||
738 | |||||
739 | // Otherwise, the difference of LHS - RHS can be computed as: | ||||
740 | // LHS - RHS | ||||
741 | // = (LHSOffset + Base) - (RHSOffset + Base) | ||||
742 | // = LHSOffset - RHSOffset | ||||
743 | Constant *Res = ConstantInt::get(LHS->getContext(), LHSOffset - RHSOffset); | ||||
744 | if (auto *VecTy = dyn_cast<VectorType>(LHS->getType())) | ||||
745 | Res = ConstantVector::getSplat(VecTy->getElementCount(), Res); | ||||
746 | return Res; | ||||
747 | } | ||||
748 | |||||
749 | /// Test if there is a dominating equivalence condition for the | ||||
750 | /// two operands. If there is, try to reduce the binary operation | ||||
751 | /// between the two operands. | ||||
752 | /// Example: Op0 - Op1 --> 0 when Op0 == Op1 | ||||
753 | static Value *simplifyByDomEq(unsigned Opcode, Value *Op0, Value *Op1, | ||||
754 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
755 | // Recursive run it can not get any benefit | ||||
756 | if (MaxRecurse != RecursionLimit) | ||||
757 | return nullptr; | ||||
758 | |||||
759 | std::optional<bool> Imp = | ||||
760 | isImpliedByDomCondition(CmpInst::ICMP_EQ, Op0, Op1, Q.CxtI, Q.DL); | ||||
761 | if (Imp && *Imp) { | ||||
762 | Type *Ty = Op0->getType(); | ||||
763 | switch (Opcode) { | ||||
764 | case Instruction::Sub: | ||||
765 | case Instruction::Xor: | ||||
766 | case Instruction::URem: | ||||
767 | case Instruction::SRem: | ||||
768 | return Constant::getNullValue(Ty); | ||||
769 | |||||
770 | case Instruction::SDiv: | ||||
771 | case Instruction::UDiv: | ||||
772 | return ConstantInt::get(Ty, 1); | ||||
773 | |||||
774 | case Instruction::And: | ||||
775 | case Instruction::Or: | ||||
776 | // Could be either one - choose Op1 since that's more likely a constant. | ||||
777 | return Op1; | ||||
778 | default: | ||||
779 | break; | ||||
780 | } | ||||
781 | } | ||||
782 | return nullptr; | ||||
783 | } | ||||
784 | |||||
785 | /// Given operands for a Sub, see if we can fold the result. | ||||
786 | /// If not, this returns null. | ||||
787 | static Value *simplifySubInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW, | ||||
788 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
789 | if (Constant *C = foldOrCommuteConstant(Instruction::Sub, Op0, Op1, Q)) | ||||
790 | return C; | ||||
791 | |||||
792 | // X - poison -> poison | ||||
793 | // poison - X -> poison | ||||
794 | if (isa<PoisonValue>(Op0) || isa<PoisonValue>(Op1)) | ||||
795 | return PoisonValue::get(Op0->getType()); | ||||
796 | |||||
797 | // X - undef -> undef | ||||
798 | // undef - X -> undef | ||||
799 | if (Q.isUndefValue(Op0) || Q.isUndefValue(Op1)) | ||||
800 | return UndefValue::get(Op0->getType()); | ||||
801 | |||||
802 | // X - 0 -> X | ||||
803 | if (match(Op1, m_Zero())) | ||||
804 | return Op0; | ||||
805 | |||||
806 | // X - X -> 0 | ||||
807 | if (Op0 == Op1) | ||||
808 | return Constant::getNullValue(Op0->getType()); | ||||
809 | |||||
810 | // Is this a negation? | ||||
811 | if (match(Op0, m_Zero())) { | ||||
812 | // 0 - X -> 0 if the sub is NUW. | ||||
813 | if (IsNUW) | ||||
814 | return Constant::getNullValue(Op0->getType()); | ||||
815 | |||||
816 | KnownBits Known = computeKnownBits(Op1, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
817 | if (Known.Zero.isMaxSignedValue()) { | ||||
818 | // Op1 is either 0 or the minimum signed value. If the sub is NSW, then | ||||
819 | // Op1 must be 0 because negating the minimum signed value is undefined. | ||||
820 | if (IsNSW) | ||||
821 | return Constant::getNullValue(Op0->getType()); | ||||
822 | |||||
823 | // 0 - X -> X if X is 0 or the minimum signed value. | ||||
824 | return Op1; | ||||
825 | } | ||||
826 | } | ||||
827 | |||||
828 | // (X + Y) - Z -> X + (Y - Z) or Y + (X - Z) if everything simplifies. | ||||
829 | // For example, (X + Y) - Y -> X; (Y + X) - Y -> X | ||||
830 | Value *X = nullptr, *Y = nullptr, *Z = Op1; | ||||
831 | if (MaxRecurse && match(Op0, m_Add(m_Value(X), m_Value(Y)))) { // (X + Y) - Z | ||||
832 | // See if "V === Y - Z" simplifies. | ||||
833 | if (Value *V = simplifyBinOp(Instruction::Sub, Y, Z, Q, MaxRecurse - 1)) | ||||
834 | // It does! Now see if "X + V" simplifies. | ||||
835 | if (Value *W = simplifyBinOp(Instruction::Add, X, V, Q, MaxRecurse - 1)) { | ||||
836 | // It does, we successfully reassociated! | ||||
837 | ++NumReassoc; | ||||
838 | return W; | ||||
839 | } | ||||
840 | // See if "V === X - Z" simplifies. | ||||
841 | if (Value *V = simplifyBinOp(Instruction::Sub, X, Z, Q, MaxRecurse - 1)) | ||||
842 | // It does! Now see if "Y + V" simplifies. | ||||
843 | if (Value *W = simplifyBinOp(Instruction::Add, Y, V, Q, MaxRecurse - 1)) { | ||||
844 | // It does, we successfully reassociated! | ||||
845 | ++NumReassoc; | ||||
846 | return W; | ||||
847 | } | ||||
848 | } | ||||
849 | |||||
850 | // X - (Y + Z) -> (X - Y) - Z or (X - Z) - Y if everything simplifies. | ||||
851 | // For example, X - (X + 1) -> -1 | ||||
852 | X = Op0; | ||||
853 | if (MaxRecurse && match(Op1, m_Add(m_Value(Y), m_Value(Z)))) { // X - (Y + Z) | ||||
854 | // See if "V === X - Y" simplifies. | ||||
855 | if (Value *V = simplifyBinOp(Instruction::Sub, X, Y, Q, MaxRecurse - 1)) | ||||
856 | // It does! Now see if "V - Z" simplifies. | ||||
857 | if (Value *W = simplifyBinOp(Instruction::Sub, V, Z, Q, MaxRecurse - 1)) { | ||||
858 | // It does, we successfully reassociated! | ||||
859 | ++NumReassoc; | ||||
860 | return W; | ||||
861 | } | ||||
862 | // See if "V === X - Z" simplifies. | ||||
863 | if (Value *V = simplifyBinOp(Instruction::Sub, X, Z, Q, MaxRecurse - 1)) | ||||
864 | // It does! Now see if "V - Y" simplifies. | ||||
865 | if (Value *W = simplifyBinOp(Instruction::Sub, V, Y, Q, MaxRecurse - 1)) { | ||||
866 | // It does, we successfully reassociated! | ||||
867 | ++NumReassoc; | ||||
868 | return W; | ||||
869 | } | ||||
870 | } | ||||
871 | |||||
872 | // Z - (X - Y) -> (Z - X) + Y if everything simplifies. | ||||
873 | // For example, X - (X - Y) -> Y. | ||||
874 | Z = Op0; | ||||
875 | if (MaxRecurse && match(Op1, m_Sub(m_Value(X), m_Value(Y)))) // Z - (X - Y) | ||||
876 | // See if "V === Z - X" simplifies. | ||||
877 | if (Value *V = simplifyBinOp(Instruction::Sub, Z, X, Q, MaxRecurse - 1)) | ||||
878 | // It does! Now see if "V + Y" simplifies. | ||||
879 | if (Value *W = simplifyBinOp(Instruction::Add, V, Y, Q, MaxRecurse - 1)) { | ||||
880 | // It does, we successfully reassociated! | ||||
881 | ++NumReassoc; | ||||
882 | return W; | ||||
883 | } | ||||
884 | |||||
885 | // trunc(X) - trunc(Y) -> trunc(X - Y) if everything simplifies. | ||||
886 | if (MaxRecurse && match(Op0, m_Trunc(m_Value(X))) && | ||||
887 | match(Op1, m_Trunc(m_Value(Y)))) | ||||
888 | if (X->getType() == Y->getType()) | ||||
889 | // See if "V === X - Y" simplifies. | ||||
890 | if (Value *V = simplifyBinOp(Instruction::Sub, X, Y, Q, MaxRecurse - 1)) | ||||
891 | // It does! Now see if "trunc V" simplifies. | ||||
892 | if (Value *W = simplifyCastInst(Instruction::Trunc, V, Op0->getType(), | ||||
893 | Q, MaxRecurse - 1)) | ||||
894 | // It does, return the simplified "trunc V". | ||||
895 | return W; | ||||
896 | |||||
897 | // Variations on GEP(base, I, ...) - GEP(base, i, ...) -> GEP(null, I-i, ...). | ||||
898 | if (match(Op0, m_PtrToInt(m_Value(X))) && match(Op1, m_PtrToInt(m_Value(Y)))) | ||||
899 | if (Constant *Result = computePointerDifference(Q.DL, X, Y)) | ||||
900 | return ConstantExpr::getIntegerCast(Result, Op0->getType(), true); | ||||
901 | |||||
902 | // i1 sub -> xor. | ||||
903 | if (MaxRecurse && Op0->getType()->isIntOrIntVectorTy(1)) | ||||
904 | if (Value *V = simplifyXorInst(Op0, Op1, Q, MaxRecurse - 1)) | ||||
905 | return V; | ||||
906 | |||||
907 | // Threading Sub over selects and phi nodes is pointless, so don't bother. | ||||
908 | // Threading over the select in "A - select(cond, B, C)" means evaluating | ||||
909 | // "A-B" and "A-C" and seeing if they are equal; but they are equal if and | ||||
910 | // only if B and C are equal. If B and C are equal then (since we assume | ||||
911 | // that operands have already been simplified) "select(cond, B, C)" should | ||||
912 | // have been simplified to the common value of B and C already. Analysing | ||||
913 | // "A-B" and "A-C" thus gains nothing, but costs compile time. Similarly | ||||
914 | // for threading over phi nodes. | ||||
915 | |||||
916 | if (Value *V = simplifyByDomEq(Instruction::Sub, Op0, Op1, Q, MaxRecurse)) | ||||
917 | return V; | ||||
918 | |||||
919 | return nullptr; | ||||
920 | } | ||||
921 | |||||
922 | Value *llvm::simplifySubInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW, | ||||
923 | const SimplifyQuery &Q) { | ||||
924 | return ::simplifySubInst(Op0, Op1, IsNSW, IsNUW, Q, RecursionLimit); | ||||
925 | } | ||||
926 | |||||
927 | /// Given operands for a Mul, see if we can fold the result. | ||||
928 | /// If not, this returns null. | ||||
929 | static Value *simplifyMulInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW, | ||||
930 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
931 | if (Constant *C = foldOrCommuteConstant(Instruction::Mul, Op0, Op1, Q)) | ||||
932 | return C; | ||||
933 | |||||
934 | // X * poison -> poison | ||||
935 | if (isa<PoisonValue>(Op1)) | ||||
936 | return Op1; | ||||
937 | |||||
938 | // X * undef -> 0 | ||||
939 | // X * 0 -> 0 | ||||
940 | if (Q.isUndefValue(Op1) || match(Op1, m_Zero())) | ||||
941 | return Constant::getNullValue(Op0->getType()); | ||||
942 | |||||
943 | // X * 1 -> X | ||||
944 | if (match(Op1, m_One())) | ||||
945 | return Op0; | ||||
946 | |||||
947 | // (X / Y) * Y -> X if the division is exact. | ||||
948 | Value *X = nullptr; | ||||
949 | if (Q.IIQ.UseInstrInfo && | ||||
950 | (match(Op0, | ||||
951 | m_Exact(m_IDiv(m_Value(X), m_Specific(Op1)))) || // (X / Y) * Y | ||||
952 | match(Op1, m_Exact(m_IDiv(m_Value(X), m_Specific(Op0)))))) // Y * (X / Y) | ||||
953 | return X; | ||||
954 | |||||
955 | if (Op0->getType()->isIntOrIntVectorTy(1)) { | ||||
956 | // mul i1 nsw is a special-case because -1 * -1 is poison (+1 is not | ||||
957 | // representable). All other cases reduce to 0, so just return 0. | ||||
958 | if (IsNSW) | ||||
959 | return ConstantInt::getNullValue(Op0->getType()); | ||||
960 | |||||
961 | // Treat "mul i1" as "and i1". | ||||
962 | if (MaxRecurse) | ||||
963 | if (Value *V = simplifyAndInst(Op0, Op1, Q, MaxRecurse - 1)) | ||||
964 | return V; | ||||
965 | } | ||||
966 | |||||
967 | // Try some generic simplifications for associative operations. | ||||
968 | if (Value *V = | ||||
969 | simplifyAssociativeBinOp(Instruction::Mul, Op0, Op1, Q, MaxRecurse)) | ||||
970 | return V; | ||||
971 | |||||
972 | // Mul distributes over Add. Try some generic simplifications based on this. | ||||
973 | if (Value *V = expandCommutativeBinOp(Instruction::Mul, Op0, Op1, | ||||
974 | Instruction::Add, Q, MaxRecurse)) | ||||
975 | return V; | ||||
976 | |||||
977 | // If the operation is with the result of a select instruction, check whether | ||||
978 | // operating on either branch of the select always yields the same value. | ||||
979 | if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1)) | ||||
980 | if (Value *V = | ||||
981 | threadBinOpOverSelect(Instruction::Mul, Op0, Op1, Q, MaxRecurse)) | ||||
982 | return V; | ||||
983 | |||||
984 | // If the operation is with the result of a phi instruction, check whether | ||||
985 | // operating on all incoming values of the phi always yields the same value. | ||||
986 | if (isa<PHINode>(Op0) || isa<PHINode>(Op1)) | ||||
987 | if (Value *V = | ||||
988 | threadBinOpOverPHI(Instruction::Mul, Op0, Op1, Q, MaxRecurse)) | ||||
989 | return V; | ||||
990 | |||||
991 | return nullptr; | ||||
992 | } | ||||
993 | |||||
994 | Value *llvm::simplifyMulInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW, | ||||
995 | const SimplifyQuery &Q) { | ||||
996 | return ::simplifyMulInst(Op0, Op1, IsNSW, IsNUW, Q, RecursionLimit); | ||||
997 | } | ||||
998 | |||||
999 | /// Check for common or similar folds of integer division or integer remainder. | ||||
1000 | /// This applies to all 4 opcodes (sdiv/udiv/srem/urem). | ||||
1001 | static Value *simplifyDivRem(Instruction::BinaryOps Opcode, Value *Op0, | ||||
1002 | Value *Op1, const SimplifyQuery &Q, | ||||
1003 | unsigned MaxRecurse) { | ||||
1004 | bool IsDiv = (Opcode == Instruction::SDiv || Opcode == Instruction::UDiv); | ||||
1005 | bool IsSigned = (Opcode == Instruction::SDiv || Opcode == Instruction::SRem); | ||||
1006 | |||||
1007 | Type *Ty = Op0->getType(); | ||||
1008 | |||||
1009 | // X / undef -> poison | ||||
1010 | // X % undef -> poison | ||||
1011 | if (Q.isUndefValue(Op1) || isa<PoisonValue>(Op1)) | ||||
1012 | return PoisonValue::get(Ty); | ||||
1013 | |||||
1014 | // X / 0 -> poison | ||||
1015 | // X % 0 -> poison | ||||
1016 | // We don't need to preserve faults! | ||||
1017 | if (match(Op1, m_Zero())) | ||||
1018 | return PoisonValue::get(Ty); | ||||
1019 | |||||
1020 | // If any element of a constant divisor fixed width vector is zero or undef | ||||
1021 | // the behavior is undefined and we can fold the whole op to poison. | ||||
1022 | auto *Op1C = dyn_cast<Constant>(Op1); | ||||
1023 | auto *VTy = dyn_cast<FixedVectorType>(Ty); | ||||
1024 | if (Op1C && VTy) { | ||||
1025 | unsigned NumElts = VTy->getNumElements(); | ||||
1026 | for (unsigned i = 0; i != NumElts; ++i) { | ||||
1027 | Constant *Elt = Op1C->getAggregateElement(i); | ||||
1028 | if (Elt && (Elt->isNullValue() || Q.isUndefValue(Elt))) | ||||
1029 | return PoisonValue::get(Ty); | ||||
1030 | } | ||||
1031 | } | ||||
1032 | |||||
1033 | // poison / X -> poison | ||||
1034 | // poison % X -> poison | ||||
1035 | if (isa<PoisonValue>(Op0)) | ||||
1036 | return Op0; | ||||
1037 | |||||
1038 | // undef / X -> 0 | ||||
1039 | // undef % X -> 0 | ||||
1040 | if (Q.isUndefValue(Op0)) | ||||
1041 | return Constant::getNullValue(Ty); | ||||
1042 | |||||
1043 | // 0 / X -> 0 | ||||
1044 | // 0 % X -> 0 | ||||
1045 | if (match(Op0, m_Zero())) | ||||
1046 | return Constant::getNullValue(Op0->getType()); | ||||
1047 | |||||
1048 | // X / X -> 1 | ||||
1049 | // X % X -> 0 | ||||
1050 | if (Op0 == Op1) | ||||
1051 | return IsDiv ? ConstantInt::get(Ty, 1) : Constant::getNullValue(Ty); | ||||
1052 | |||||
1053 | // X / 1 -> X | ||||
1054 | // X % 1 -> 0 | ||||
1055 | // If this is a boolean op (single-bit element type), we can't have | ||||
1056 | // division-by-zero or remainder-by-zero, so assume the divisor is 1. | ||||
1057 | // Similarly, if we're zero-extending a boolean divisor, then assume it's a 1. | ||||
1058 | Value *X; | ||||
1059 | if (match(Op1, m_One()) || Ty->isIntOrIntVectorTy(1) || | ||||
1060 | (match(Op1, m_ZExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1))) | ||||
1061 | return IsDiv ? Op0 : Constant::getNullValue(Ty); | ||||
1062 | |||||
1063 | // If X * Y does not overflow, then: | ||||
1064 | // X * Y / Y -> X | ||||
1065 | // X * Y % Y -> 0 | ||||
1066 | if (match(Op0, m_c_Mul(m_Value(X), m_Specific(Op1)))) { | ||||
1067 | auto *Mul = cast<OverflowingBinaryOperator>(Op0); | ||||
1068 | // The multiplication can't overflow if it is defined not to, or if | ||||
1069 | // X == A / Y for some A. | ||||
1070 | if ((IsSigned && Q.IIQ.hasNoSignedWrap(Mul)) || | ||||
1071 | (!IsSigned && Q.IIQ.hasNoUnsignedWrap(Mul)) || | ||||
1072 | (IsSigned && match(X, m_SDiv(m_Value(), m_Specific(Op1)))) || | ||||
1073 | (!IsSigned && match(X, m_UDiv(m_Value(), m_Specific(Op1))))) { | ||||
1074 | return IsDiv ? X : Constant::getNullValue(Op0->getType()); | ||||
1075 | } | ||||
1076 | } | ||||
1077 | |||||
1078 | if (Value *V = simplifyByDomEq(Opcode, Op0, Op1, Q, MaxRecurse)) | ||||
1079 | return V; | ||||
1080 | |||||
1081 | return nullptr; | ||||
1082 | } | ||||
1083 | |||||
1084 | /// Given a predicate and two operands, return true if the comparison is true. | ||||
1085 | /// This is a helper for div/rem simplification where we return some other value | ||||
1086 | /// when we can prove a relationship between the operands. | ||||
1087 | static bool isICmpTrue(ICmpInst::Predicate Pred, Value *LHS, Value *RHS, | ||||
1088 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
1089 | Value *V = simplifyICmpInst(Pred, LHS, RHS, Q, MaxRecurse); | ||||
1090 | Constant *C = dyn_cast_or_null<Constant>(V); | ||||
1091 | return (C && C->isAllOnesValue()); | ||||
1092 | } | ||||
1093 | |||||
1094 | /// Return true if we can simplify X / Y to 0. Remainder can adapt that answer | ||||
1095 | /// to simplify X % Y to X. | ||||
1096 | static bool isDivZero(Value *X, Value *Y, const SimplifyQuery &Q, | ||||
1097 | unsigned MaxRecurse, bool IsSigned) { | ||||
1098 | // Recursion is always used, so bail out at once if we already hit the limit. | ||||
1099 | if (!MaxRecurse--) | ||||
1100 | return false; | ||||
1101 | |||||
1102 | if (IsSigned) { | ||||
1103 | // |X| / |Y| --> 0 | ||||
1104 | // | ||||
1105 | // We require that 1 operand is a simple constant. That could be extended to | ||||
1106 | // 2 variables if we computed the sign bit for each. | ||||
1107 | // | ||||
1108 | // Make sure that a constant is not the minimum signed value because taking | ||||
1109 | // the abs() of that is undefined. | ||||
1110 | Type *Ty = X->getType(); | ||||
1111 | const APInt *C; | ||||
1112 | if (match(X, m_APInt(C)) && !C->isMinSignedValue()) { | ||||
1113 | // Is the variable divisor magnitude always greater than the constant | ||||
1114 | // dividend magnitude? | ||||
1115 | // |Y| > |C| --> Y < -abs(C) or Y > abs(C) | ||||
1116 | Constant *PosDividendC = ConstantInt::get(Ty, C->abs()); | ||||
1117 | Constant *NegDividendC = ConstantInt::get(Ty, -C->abs()); | ||||
1118 | if (isICmpTrue(CmpInst::ICMP_SLT, Y, NegDividendC, Q, MaxRecurse) || | ||||
1119 | isICmpTrue(CmpInst::ICMP_SGT, Y, PosDividendC, Q, MaxRecurse)) | ||||
1120 | return true; | ||||
1121 | } | ||||
1122 | if (match(Y, m_APInt(C))) { | ||||
1123 | // Special-case: we can't take the abs() of a minimum signed value. If | ||||
1124 | // that's the divisor, then all we have to do is prove that the dividend | ||||
1125 | // is also not the minimum signed value. | ||||
1126 | if (C->isMinSignedValue()) | ||||
1127 | return isICmpTrue(CmpInst::ICMP_NE, X, Y, Q, MaxRecurse); | ||||
1128 | |||||
1129 | // Is the variable dividend magnitude always less than the constant | ||||
1130 | // divisor magnitude? | ||||
1131 | // |X| < |C| --> X > -abs(C) and X < abs(C) | ||||
1132 | Constant *PosDivisorC = ConstantInt::get(Ty, C->abs()); | ||||
1133 | Constant *NegDivisorC = ConstantInt::get(Ty, -C->abs()); | ||||
1134 | if (isICmpTrue(CmpInst::ICMP_SGT, X, NegDivisorC, Q, MaxRecurse) && | ||||
1135 | isICmpTrue(CmpInst::ICMP_SLT, X, PosDivisorC, Q, MaxRecurse)) | ||||
1136 | return true; | ||||
1137 | } | ||||
1138 | return false; | ||||
1139 | } | ||||
1140 | |||||
1141 | // IsSigned == false. | ||||
1142 | |||||
1143 | // Is the unsigned dividend known to be less than a constant divisor? | ||||
1144 | // TODO: Convert this (and above) to range analysis | ||||
1145 | // ("computeConstantRangeIncludingKnownBits")? | ||||
1146 | const APInt *C; | ||||
1147 | if (match(Y, m_APInt(C)) && | ||||
1148 | computeKnownBits(X, Q.DL, 0, Q.AC, Q.CxtI, Q.DT).getMaxValue().ult(*C)) | ||||
1149 | return true; | ||||
1150 | |||||
1151 | // Try again for any divisor: | ||||
1152 | // Is the dividend unsigned less than the divisor? | ||||
1153 | return isICmpTrue(ICmpInst::ICMP_ULT, X, Y, Q, MaxRecurse); | ||||
1154 | } | ||||
1155 | |||||
1156 | /// These are simplifications common to SDiv and UDiv. | ||||
1157 | static Value *simplifyDiv(Instruction::BinaryOps Opcode, Value *Op0, Value *Op1, | ||||
1158 | bool IsExact, const SimplifyQuery &Q, | ||||
1159 | unsigned MaxRecurse) { | ||||
1160 | if (Constant *C = foldOrCommuteConstant(Opcode, Op0, Op1, Q)) | ||||
1161 | return C; | ||||
1162 | |||||
1163 | if (Value *V = simplifyDivRem(Opcode, Op0, Op1, Q, MaxRecurse)) | ||||
1164 | return V; | ||||
1165 | |||||
1166 | // If this is an exact divide by a constant, then the dividend (Op0) must have | ||||
1167 | // at least as many trailing zeros as the divisor to divide evenly. If it has | ||||
1168 | // less trailing zeros, then the result must be poison. | ||||
1169 | const APInt *DivC; | ||||
1170 | if (IsExact && match(Op1, m_APInt(DivC)) && DivC->countr_zero()) { | ||||
1171 | KnownBits KnownOp0 = computeKnownBits(Op0, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
1172 | if (KnownOp0.countMaxTrailingZeros() < DivC->countr_zero()) | ||||
1173 | return PoisonValue::get(Op0->getType()); | ||||
1174 | } | ||||
1175 | |||||
1176 | bool IsSigned = Opcode == Instruction::SDiv; | ||||
1177 | |||||
1178 | // (X rem Y) / Y -> 0 | ||||
1179 | if ((IsSigned && match(Op0, m_SRem(m_Value(), m_Specific(Op1)))) || | ||||
1180 | (!IsSigned && match(Op0, m_URem(m_Value(), m_Specific(Op1))))) | ||||
1181 | return Constant::getNullValue(Op0->getType()); | ||||
1182 | |||||
1183 | // (X /u C1) /u C2 -> 0 if C1 * C2 overflow | ||||
1184 | ConstantInt *C1, *C2; | ||||
1185 | if (!IsSigned && match(Op0, m_UDiv(m_Value(), m_ConstantInt(C1))) && | ||||
1186 | match(Op1, m_ConstantInt(C2))) { | ||||
1187 | bool Overflow; | ||||
1188 | (void)C1->getValue().umul_ov(C2->getValue(), Overflow); | ||||
1189 | if (Overflow) | ||||
1190 | return Constant::getNullValue(Op0->getType()); | ||||
1191 | } | ||||
1192 | |||||
1193 | // If the operation is with the result of a select instruction, check whether | ||||
1194 | // operating on either branch of the select always yields the same value. | ||||
1195 | if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1)) | ||||
1196 | if (Value *V = threadBinOpOverSelect(Opcode, Op0, Op1, Q, MaxRecurse)) | ||||
1197 | return V; | ||||
1198 | |||||
1199 | // If the operation is with the result of a phi instruction, check whether | ||||
1200 | // operating on all incoming values of the phi always yields the same value. | ||||
1201 | if (isa<PHINode>(Op0) || isa<PHINode>(Op1)) | ||||
1202 | if (Value *V = threadBinOpOverPHI(Opcode, Op0, Op1, Q, MaxRecurse)) | ||||
1203 | return V; | ||||
1204 | |||||
1205 | if (isDivZero(Op0, Op1, Q, MaxRecurse, IsSigned)) | ||||
1206 | return Constant::getNullValue(Op0->getType()); | ||||
1207 | |||||
1208 | return nullptr; | ||||
1209 | } | ||||
1210 | |||||
1211 | /// These are simplifications common to SRem and URem. | ||||
1212 | static Value *simplifyRem(Instruction::BinaryOps Opcode, Value *Op0, Value *Op1, | ||||
1213 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
1214 | if (Constant *C = foldOrCommuteConstant(Opcode, Op0, Op1, Q)) | ||||
1215 | return C; | ||||
1216 | |||||
1217 | if (Value *V = simplifyDivRem(Opcode, Op0, Op1, Q, MaxRecurse)) | ||||
1218 | return V; | ||||
1219 | |||||
1220 | // (X % Y) % Y -> X % Y | ||||
1221 | if ((Opcode == Instruction::SRem && | ||||
1222 | match(Op0, m_SRem(m_Value(), m_Specific(Op1)))) || | ||||
1223 | (Opcode == Instruction::URem && | ||||
1224 | match(Op0, m_URem(m_Value(), m_Specific(Op1))))) | ||||
1225 | return Op0; | ||||
1226 | |||||
1227 | // (X << Y) % X -> 0 | ||||
1228 | if (Q.IIQ.UseInstrInfo && | ||||
1229 | ((Opcode == Instruction::SRem && | ||||
1230 | match(Op0, m_NSWShl(m_Specific(Op1), m_Value()))) || | ||||
1231 | (Opcode == Instruction::URem && | ||||
1232 | match(Op0, m_NUWShl(m_Specific(Op1), m_Value()))))) | ||||
1233 | return Constant::getNullValue(Op0->getType()); | ||||
1234 | |||||
1235 | // If the operation is with the result of a select instruction, check whether | ||||
1236 | // operating on either branch of the select always yields the same value. | ||||
1237 | if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1)) | ||||
1238 | if (Value *V = threadBinOpOverSelect(Opcode, Op0, Op1, Q, MaxRecurse)) | ||||
1239 | return V; | ||||
1240 | |||||
1241 | // If the operation is with the result of a phi instruction, check whether | ||||
1242 | // operating on all incoming values of the phi always yields the same value. | ||||
1243 | if (isa<PHINode>(Op0) || isa<PHINode>(Op1)) | ||||
1244 | if (Value *V = threadBinOpOverPHI(Opcode, Op0, Op1, Q, MaxRecurse)) | ||||
1245 | return V; | ||||
1246 | |||||
1247 | // If X / Y == 0, then X % Y == X. | ||||
1248 | if (isDivZero(Op0, Op1, Q, MaxRecurse, Opcode == Instruction::SRem)) | ||||
1249 | return Op0; | ||||
1250 | |||||
1251 | return nullptr; | ||||
1252 | } | ||||
1253 | |||||
1254 | /// Given operands for an SDiv, see if we can fold the result. | ||||
1255 | /// If not, this returns null. | ||||
1256 | static Value *simplifySDivInst(Value *Op0, Value *Op1, bool IsExact, | ||||
1257 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
1258 | // If two operands are negated and no signed overflow, return -1. | ||||
1259 | if (isKnownNegation(Op0, Op1, /*NeedNSW=*/true)) | ||||
1260 | return Constant::getAllOnesValue(Op0->getType()); | ||||
1261 | |||||
1262 | return simplifyDiv(Instruction::SDiv, Op0, Op1, IsExact, Q, MaxRecurse); | ||||
1263 | } | ||||
1264 | |||||
1265 | Value *llvm::simplifySDivInst(Value *Op0, Value *Op1, bool IsExact, | ||||
1266 | const SimplifyQuery &Q) { | ||||
1267 | return ::simplifySDivInst(Op0, Op1, IsExact, Q, RecursionLimit); | ||||
1268 | } | ||||
1269 | |||||
1270 | /// Given operands for a UDiv, see if we can fold the result. | ||||
1271 | /// If not, this returns null. | ||||
1272 | static Value *simplifyUDivInst(Value *Op0, Value *Op1, bool IsExact, | ||||
1273 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
1274 | return simplifyDiv(Instruction::UDiv, Op0, Op1, IsExact, Q, MaxRecurse); | ||||
1275 | } | ||||
1276 | |||||
1277 | Value *llvm::simplifyUDivInst(Value *Op0, Value *Op1, bool IsExact, | ||||
1278 | const SimplifyQuery &Q) { | ||||
1279 | return ::simplifyUDivInst(Op0, Op1, IsExact, Q, RecursionLimit); | ||||
1280 | } | ||||
1281 | |||||
1282 | /// Given operands for an SRem, see if we can fold the result. | ||||
1283 | /// If not, this returns null. | ||||
1284 | static Value *simplifySRemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, | ||||
1285 | unsigned MaxRecurse) { | ||||
1286 | // If the divisor is 0, the result is undefined, so assume the divisor is -1. | ||||
1287 | // srem Op0, (sext i1 X) --> srem Op0, -1 --> 0 | ||||
1288 | Value *X; | ||||
1289 | if (match(Op1, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) | ||||
1290 | return ConstantInt::getNullValue(Op0->getType()); | ||||
1291 | |||||
1292 | // If the two operands are negated, return 0. | ||||
1293 | if (isKnownNegation(Op0, Op1)) | ||||
1294 | return ConstantInt::getNullValue(Op0->getType()); | ||||
1295 | |||||
1296 | return simplifyRem(Instruction::SRem, Op0, Op1, Q, MaxRecurse); | ||||
1297 | } | ||||
1298 | |||||
1299 | Value *llvm::simplifySRemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { | ||||
1300 | return ::simplifySRemInst(Op0, Op1, Q, RecursionLimit); | ||||
1301 | } | ||||
1302 | |||||
1303 | /// Given operands for a URem, see if we can fold the result. | ||||
1304 | /// If not, this returns null. | ||||
1305 | static Value *simplifyURemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, | ||||
1306 | unsigned MaxRecurse) { | ||||
1307 | return simplifyRem(Instruction::URem, Op0, Op1, Q, MaxRecurse); | ||||
1308 | } | ||||
1309 | |||||
1310 | Value *llvm::simplifyURemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { | ||||
1311 | return ::simplifyURemInst(Op0, Op1, Q, RecursionLimit); | ||||
1312 | } | ||||
1313 | |||||
1314 | /// Returns true if a shift by \c Amount always yields poison. | ||||
1315 | static bool isPoisonShift(Value *Amount, const SimplifyQuery &Q) { | ||||
1316 | Constant *C = dyn_cast<Constant>(Amount); | ||||
1317 | if (!C) | ||||
1318 | return false; | ||||
1319 | |||||
1320 | // X shift by undef -> poison because it may shift by the bitwidth. | ||||
1321 | if (Q.isUndefValue(C)) | ||||
1322 | return true; | ||||
1323 | |||||
1324 | // Shifting by the bitwidth or more is poison. This covers scalars and | ||||
1325 | // fixed/scalable vectors with splat constants. | ||||
1326 | const APInt *AmountC; | ||||
1327 | if (match(C, m_APInt(AmountC)) && AmountC->uge(AmountC->getBitWidth())) | ||||
1328 | return true; | ||||
1329 | |||||
1330 | // Try harder for fixed-length vectors: | ||||
1331 | // If all lanes of a vector shift are poison, the whole shift is poison. | ||||
1332 | if (isa<ConstantVector>(C) || isa<ConstantDataVector>(C)) { | ||||
1333 | for (unsigned I = 0, | ||||
1334 | E = cast<FixedVectorType>(C->getType())->getNumElements(); | ||||
1335 | I != E; ++I) | ||||
1336 | if (!isPoisonShift(C->getAggregateElement(I), Q)) | ||||
1337 | return false; | ||||
1338 | return true; | ||||
1339 | } | ||||
1340 | |||||
1341 | return false; | ||||
1342 | } | ||||
1343 | |||||
1344 | /// Given operands for an Shl, LShr or AShr, see if we can fold the result. | ||||
1345 | /// If not, this returns null. | ||||
1346 | static Value *simplifyShift(Instruction::BinaryOps Opcode, Value *Op0, | ||||
1347 | Value *Op1, bool IsNSW, const SimplifyQuery &Q, | ||||
1348 | unsigned MaxRecurse) { | ||||
1349 | if (Constant *C = foldOrCommuteConstant(Opcode, Op0, Op1, Q)) | ||||
1350 | return C; | ||||
1351 | |||||
1352 | // poison shift by X -> poison | ||||
1353 | if (isa<PoisonValue>(Op0)) | ||||
1354 | return Op0; | ||||
1355 | |||||
1356 | // 0 shift by X -> 0 | ||||
1357 | if (match(Op0, m_Zero())) | ||||
1358 | return Constant::getNullValue(Op0->getType()); | ||||
1359 | |||||
1360 | // X shift by 0 -> X | ||||
1361 | // Shift-by-sign-extended bool must be shift-by-0 because shift-by-all-ones | ||||
1362 | // would be poison. | ||||
1363 | Value *X; | ||||
1364 | if (match(Op1, m_Zero()) || | ||||
1365 | (match(Op1, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1))) | ||||
1366 | return Op0; | ||||
1367 | |||||
1368 | // Fold undefined shifts. | ||||
1369 | if (isPoisonShift(Op1, Q)) | ||||
1370 | return PoisonValue::get(Op0->getType()); | ||||
1371 | |||||
1372 | // If the operation is with the result of a select instruction, check whether | ||||
1373 | // operating on either branch of the select always yields the same value. | ||||
1374 | if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1)) | ||||
1375 | if (Value *V = threadBinOpOverSelect(Opcode, Op0, Op1, Q, MaxRecurse)) | ||||
1376 | return V; | ||||
1377 | |||||
1378 | // If the operation is with the result of a phi instruction, check whether | ||||
1379 | // operating on all incoming values of the phi always yields the same value. | ||||
1380 | if (isa<PHINode>(Op0) || isa<PHINode>(Op1)) | ||||
1381 | if (Value *V = threadBinOpOverPHI(Opcode, Op0, Op1, Q, MaxRecurse)) | ||||
1382 | return V; | ||||
1383 | |||||
1384 | // If any bits in the shift amount make that value greater than or equal to | ||||
1385 | // the number of bits in the type, the shift is undefined. | ||||
1386 | KnownBits KnownAmt = computeKnownBits(Op1, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
1387 | if (KnownAmt.getMinValue().uge(KnownAmt.getBitWidth())) | ||||
1388 | return PoisonValue::get(Op0->getType()); | ||||
1389 | |||||
1390 | // If all valid bits in the shift amount are known zero, the first operand is | ||||
1391 | // unchanged. | ||||
1392 | unsigned NumValidShiftBits = Log2_32_Ceil(KnownAmt.getBitWidth()); | ||||
1393 | if (KnownAmt.countMinTrailingZeros() >= NumValidShiftBits) | ||||
1394 | return Op0; | ||||
1395 | |||||
1396 | // Check for nsw shl leading to a poison value. | ||||
1397 | if (IsNSW) { | ||||
1398 | assert(Opcode == Instruction::Shl && "Expected shl for nsw instruction")(static_cast <bool> (Opcode == Instruction::Shl && "Expected shl for nsw instruction") ? void (0) : __assert_fail ("Opcode == Instruction::Shl && \"Expected shl for nsw instruction\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 1398, __extension__ __PRETTY_FUNCTION__)); | ||||
1399 | KnownBits KnownVal = computeKnownBits(Op0, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
1400 | KnownBits KnownShl = KnownBits::shl(KnownVal, KnownAmt); | ||||
1401 | |||||
1402 | if (KnownVal.Zero.isSignBitSet()) | ||||
1403 | KnownShl.Zero.setSignBit(); | ||||
1404 | if (KnownVal.One.isSignBitSet()) | ||||
1405 | KnownShl.One.setSignBit(); | ||||
1406 | |||||
1407 | if (KnownShl.hasConflict()) | ||||
1408 | return PoisonValue::get(Op0->getType()); | ||||
1409 | } | ||||
1410 | |||||
1411 | return nullptr; | ||||
1412 | } | ||||
1413 | |||||
1414 | /// Given operands for an LShr or AShr, see if we can fold the result. If not, | ||||
1415 | /// this returns null. | ||||
1416 | static Value *simplifyRightShift(Instruction::BinaryOps Opcode, Value *Op0, | ||||
1417 | Value *Op1, bool IsExact, | ||||
1418 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
1419 | if (Value *V = | ||||
1420 | simplifyShift(Opcode, Op0, Op1, /*IsNSW*/ false, Q, MaxRecurse)) | ||||
1421 | return V; | ||||
1422 | |||||
1423 | // X >> X -> 0 | ||||
1424 | if (Op0 == Op1) | ||||
1425 | return Constant::getNullValue(Op0->getType()); | ||||
1426 | |||||
1427 | // undef >> X -> 0 | ||||
1428 | // undef >> X -> undef (if it's exact) | ||||
1429 | if (Q.isUndefValue(Op0)) | ||||
1430 | return IsExact ? Op0 : Constant::getNullValue(Op0->getType()); | ||||
1431 | |||||
1432 | // The low bit cannot be shifted out of an exact shift if it is set. | ||||
1433 | // TODO: Generalize by counting trailing zeros (see fold for exact division). | ||||
1434 | if (IsExact) { | ||||
1435 | KnownBits Op0Known = | ||||
1436 | computeKnownBits(Op0, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT); | ||||
1437 | if (Op0Known.One[0]) | ||||
1438 | return Op0; | ||||
1439 | } | ||||
1440 | |||||
1441 | return nullptr; | ||||
1442 | } | ||||
1443 | |||||
1444 | /// Given operands for an Shl, see if we can fold the result. | ||||
1445 | /// If not, this returns null. | ||||
1446 | static Value *simplifyShlInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW, | ||||
1447 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
1448 | if (Value *V = | ||||
1449 | simplifyShift(Instruction::Shl, Op0, Op1, IsNSW, Q, MaxRecurse)) | ||||
1450 | return V; | ||||
1451 | |||||
1452 | Type *Ty = Op0->getType(); | ||||
1453 | // undef << X -> 0 | ||||
1454 | // undef << X -> undef if (if it's NSW/NUW) | ||||
1455 | if (Q.isUndefValue(Op0)) | ||||
1456 | return IsNSW || IsNUW ? Op0 : Constant::getNullValue(Ty); | ||||
1457 | |||||
1458 | // (X >> A) << A -> X | ||||
1459 | Value *X; | ||||
1460 | if (Q.IIQ.UseInstrInfo && | ||||
1461 | match(Op0, m_Exact(m_Shr(m_Value(X), m_Specific(Op1))))) | ||||
1462 | return X; | ||||
1463 | |||||
1464 | // shl nuw i8 C, %x -> C iff C has sign bit set. | ||||
1465 | if (IsNUW && match(Op0, m_Negative())) | ||||
1466 | return Op0; | ||||
1467 | // NOTE: could use computeKnownBits() / LazyValueInfo, | ||||
1468 | // but the cost-benefit analysis suggests it isn't worth it. | ||||
1469 | |||||
1470 | // "nuw" guarantees that only zeros are shifted out, and "nsw" guarantees | ||||
1471 | // that the sign-bit does not change, so the only input that does not | ||||
1472 | // produce poison is 0, and "0 << (bitwidth-1) --> 0". | ||||
1473 | if (IsNSW && IsNUW && | ||||
1474 | match(Op1, m_SpecificInt(Ty->getScalarSizeInBits() - 1))) | ||||
1475 | return Constant::getNullValue(Ty); | ||||
1476 | |||||
1477 | return nullptr; | ||||
1478 | } | ||||
1479 | |||||
1480 | Value *llvm::simplifyShlInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW, | ||||
1481 | const SimplifyQuery &Q) { | ||||
1482 | return ::simplifyShlInst(Op0, Op1, IsNSW, IsNUW, Q, RecursionLimit); | ||||
1483 | } | ||||
1484 | |||||
1485 | /// Given operands for an LShr, see if we can fold the result. | ||||
1486 | /// If not, this returns null. | ||||
1487 | static Value *simplifyLShrInst(Value *Op0, Value *Op1, bool IsExact, | ||||
1488 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
1489 | if (Value *V = simplifyRightShift(Instruction::LShr, Op0, Op1, IsExact, Q, | ||||
1490 | MaxRecurse)) | ||||
1491 | return V; | ||||
1492 | |||||
1493 | // (X << A) >> A -> X | ||||
1494 | Value *X; | ||||
1495 | if (match(Op0, m_NUWShl(m_Value(X), m_Specific(Op1)))) | ||||
1496 | return X; | ||||
1497 | |||||
1498 | // ((X << A) | Y) >> A -> X if effective width of Y is not larger than A. | ||||
1499 | // We can return X as we do in the above case since OR alters no bits in X. | ||||
1500 | // SimplifyDemandedBits in InstCombine can do more general optimization for | ||||
1501 | // bit manipulation. This pattern aims to provide opportunities for other | ||||
1502 | // optimizers by supporting a simple but common case in InstSimplify. | ||||
1503 | Value *Y; | ||||
1504 | const APInt *ShRAmt, *ShLAmt; | ||||
1505 | if (match(Op1, m_APInt(ShRAmt)) && | ||||
1506 | match(Op0, m_c_Or(m_NUWShl(m_Value(X), m_APInt(ShLAmt)), m_Value(Y))) && | ||||
1507 | *ShRAmt == *ShLAmt) { | ||||
1508 | const KnownBits YKnown = computeKnownBits(Y, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
1509 | const unsigned EffWidthY = YKnown.countMaxActiveBits(); | ||||
1510 | if (ShRAmt->uge(EffWidthY)) | ||||
1511 | return X; | ||||
1512 | } | ||||
1513 | |||||
1514 | return nullptr; | ||||
1515 | } | ||||
1516 | |||||
1517 | Value *llvm::simplifyLShrInst(Value *Op0, Value *Op1, bool IsExact, | ||||
1518 | const SimplifyQuery &Q) { | ||||
1519 | return ::simplifyLShrInst(Op0, Op1, IsExact, Q, RecursionLimit); | ||||
1520 | } | ||||
1521 | |||||
1522 | /// Given operands for an AShr, see if we can fold the result. | ||||
1523 | /// If not, this returns null. | ||||
1524 | static Value *simplifyAShrInst(Value *Op0, Value *Op1, bool IsExact, | ||||
1525 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
1526 | if (Value *V = simplifyRightShift(Instruction::AShr, Op0, Op1, IsExact, Q, | ||||
1527 | MaxRecurse)) | ||||
1528 | return V; | ||||
1529 | |||||
1530 | // -1 >>a X --> -1 | ||||
1531 | // (-1 << X) a>> X --> -1 | ||||
1532 | // Do not return Op0 because it may contain undef elements if it's a vector. | ||||
1533 | if (match(Op0, m_AllOnes()) || | ||||
1534 | match(Op0, m_Shl(m_AllOnes(), m_Specific(Op1)))) | ||||
1535 | return Constant::getAllOnesValue(Op0->getType()); | ||||
1536 | |||||
1537 | // (X << A) >> A -> X | ||||
1538 | Value *X; | ||||
1539 | if (Q.IIQ.UseInstrInfo && match(Op0, m_NSWShl(m_Value(X), m_Specific(Op1)))) | ||||
1540 | return X; | ||||
1541 | |||||
1542 | // Arithmetic shifting an all-sign-bit value is a no-op. | ||||
1543 | unsigned NumSignBits = ComputeNumSignBits(Op0, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
1544 | if (NumSignBits == Op0->getType()->getScalarSizeInBits()) | ||||
1545 | return Op0; | ||||
1546 | |||||
1547 | return nullptr; | ||||
1548 | } | ||||
1549 | |||||
1550 | Value *llvm::simplifyAShrInst(Value *Op0, Value *Op1, bool IsExact, | ||||
1551 | const SimplifyQuery &Q) { | ||||
1552 | return ::simplifyAShrInst(Op0, Op1, IsExact, Q, RecursionLimit); | ||||
1553 | } | ||||
1554 | |||||
1555 | /// Commuted variants are assumed to be handled by calling this function again | ||||
1556 | /// with the parameters swapped. | ||||
1557 | static Value *simplifyUnsignedRangeCheck(ICmpInst *ZeroICmp, | ||||
1558 | ICmpInst *UnsignedICmp, bool IsAnd, | ||||
1559 | const SimplifyQuery &Q) { | ||||
1560 | Value *X, *Y; | ||||
1561 | |||||
1562 | ICmpInst::Predicate EqPred; | ||||
1563 | if (!match(ZeroICmp, m_ICmp(EqPred, m_Value(Y), m_Zero())) || | ||||
1564 | !ICmpInst::isEquality(EqPred)) | ||||
1565 | return nullptr; | ||||
1566 | |||||
1567 | ICmpInst::Predicate UnsignedPred; | ||||
1568 | |||||
1569 | Value *A, *B; | ||||
1570 | // Y = (A - B); | ||||
1571 | if (match(Y, m_Sub(m_Value(A), m_Value(B)))) { | ||||
1572 | if (match(UnsignedICmp, | ||||
1573 | m_c_ICmp(UnsignedPred, m_Specific(A), m_Specific(B))) && | ||||
1574 | ICmpInst::isUnsigned(UnsignedPred)) { | ||||
1575 | // A >=/<= B || (A - B) != 0 <--> true | ||||
1576 | if ((UnsignedPred == ICmpInst::ICMP_UGE || | ||||
1577 | UnsignedPred == ICmpInst::ICMP_ULE) && | ||||
1578 | EqPred == ICmpInst::ICMP_NE && !IsAnd) | ||||
1579 | return ConstantInt::getTrue(UnsignedICmp->getType()); | ||||
1580 | // A </> B && (A - B) == 0 <--> false | ||||
1581 | if ((UnsignedPred == ICmpInst::ICMP_ULT || | ||||
1582 | UnsignedPred == ICmpInst::ICMP_UGT) && | ||||
1583 | EqPred == ICmpInst::ICMP_EQ && IsAnd) | ||||
1584 | return ConstantInt::getFalse(UnsignedICmp->getType()); | ||||
1585 | |||||
1586 | // A </> B && (A - B) != 0 <--> A </> B | ||||
1587 | // A </> B || (A - B) != 0 <--> (A - B) != 0 | ||||
1588 | if (EqPred == ICmpInst::ICMP_NE && (UnsignedPred == ICmpInst::ICMP_ULT || | ||||
1589 | UnsignedPred == ICmpInst::ICMP_UGT)) | ||||
1590 | return IsAnd ? UnsignedICmp : ZeroICmp; | ||||
1591 | |||||
1592 | // A <=/>= B && (A - B) == 0 <--> (A - B) == 0 | ||||
1593 | // A <=/>= B || (A - B) == 0 <--> A <=/>= B | ||||
1594 | if (EqPred == ICmpInst::ICMP_EQ && (UnsignedPred == ICmpInst::ICMP_ULE || | ||||
1595 | UnsignedPred == ICmpInst::ICMP_UGE)) | ||||
1596 | return IsAnd ? ZeroICmp : UnsignedICmp; | ||||
1597 | } | ||||
1598 | |||||
1599 | // Given Y = (A - B) | ||||
1600 | // Y >= A && Y != 0 --> Y >= A iff B != 0 | ||||
1601 | // Y < A || Y == 0 --> Y < A iff B != 0 | ||||
1602 | if (match(UnsignedICmp, | ||||
1603 | m_c_ICmp(UnsignedPred, m_Specific(Y), m_Specific(A)))) { | ||||
1604 | if (UnsignedPred == ICmpInst::ICMP_UGE && IsAnd && | ||||
1605 | EqPred == ICmpInst::ICMP_NE && | ||||
1606 | isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) | ||||
1607 | return UnsignedICmp; | ||||
1608 | if (UnsignedPred == ICmpInst::ICMP_ULT && !IsAnd && | ||||
1609 | EqPred == ICmpInst::ICMP_EQ && | ||||
1610 | isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) | ||||
1611 | return UnsignedICmp; | ||||
1612 | } | ||||
1613 | } | ||||
1614 | |||||
1615 | if (match(UnsignedICmp, m_ICmp(UnsignedPred, m_Value(X), m_Specific(Y))) && | ||||
1616 | ICmpInst::isUnsigned(UnsignedPred)) | ||||
1617 | ; | ||||
1618 | else if (match(UnsignedICmp, | ||||
1619 | m_ICmp(UnsignedPred, m_Specific(Y), m_Value(X))) && | ||||
1620 | ICmpInst::isUnsigned(UnsignedPred)) | ||||
1621 | UnsignedPred = ICmpInst::getSwappedPredicate(UnsignedPred); | ||||
1622 | else | ||||
1623 | return nullptr; | ||||
1624 | |||||
1625 | // X > Y && Y == 0 --> Y == 0 iff X != 0 | ||||
1626 | // X > Y || Y == 0 --> X > Y iff X != 0 | ||||
1627 | if (UnsignedPred == ICmpInst::ICMP_UGT && EqPred == ICmpInst::ICMP_EQ && | ||||
1628 | isKnownNonZero(X, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) | ||||
1629 | return IsAnd ? ZeroICmp : UnsignedICmp; | ||||
1630 | |||||
1631 | // X <= Y && Y != 0 --> X <= Y iff X != 0 | ||||
1632 | // X <= Y || Y != 0 --> Y != 0 iff X != 0 | ||||
1633 | if (UnsignedPred == ICmpInst::ICMP_ULE && EqPred == ICmpInst::ICMP_NE && | ||||
1634 | isKnownNonZero(X, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) | ||||
1635 | return IsAnd ? UnsignedICmp : ZeroICmp; | ||||
1636 | |||||
1637 | // The transforms below here are expected to be handled more generally with | ||||
1638 | // simplifyAndOrOfICmpsWithLimitConst() or in InstCombine's | ||||
1639 | // foldAndOrOfICmpsWithConstEq(). If we are looking to trim optimizer overlap, | ||||
1640 | // these are candidates for removal. | ||||
1641 | |||||
1642 | // X < Y && Y != 0 --> X < Y | ||||
1643 | // X < Y || Y != 0 --> Y != 0 | ||||
1644 | if (UnsignedPred == ICmpInst::ICMP_ULT && EqPred == ICmpInst::ICMP_NE) | ||||
1645 | return IsAnd ? UnsignedICmp : ZeroICmp; | ||||
1646 | |||||
1647 | // X >= Y && Y == 0 --> Y == 0 | ||||
1648 | // X >= Y || Y == 0 --> X >= Y | ||||
1649 | if (UnsignedPred == ICmpInst::ICMP_UGE && EqPred == ICmpInst::ICMP_EQ) | ||||
1650 | return IsAnd ? ZeroICmp : UnsignedICmp; | ||||
1651 | |||||
1652 | // X < Y && Y == 0 --> false | ||||
1653 | if (UnsignedPred == ICmpInst::ICMP_ULT && EqPred == ICmpInst::ICMP_EQ && | ||||
1654 | IsAnd) | ||||
1655 | return getFalse(UnsignedICmp->getType()); | ||||
1656 | |||||
1657 | // X >= Y || Y != 0 --> true | ||||
1658 | if (UnsignedPred == ICmpInst::ICMP_UGE && EqPred == ICmpInst::ICMP_NE && | ||||
1659 | !IsAnd) | ||||
1660 | return getTrue(UnsignedICmp->getType()); | ||||
1661 | |||||
1662 | return nullptr; | ||||
1663 | } | ||||
1664 | |||||
1665 | /// Test if a pair of compares with a shared operand and 2 constants has an | ||||
1666 | /// empty set intersection, full set union, or if one compare is a superset of | ||||
1667 | /// the other. | ||||
1668 | static Value *simplifyAndOrOfICmpsWithConstants(ICmpInst *Cmp0, ICmpInst *Cmp1, | ||||
1669 | bool IsAnd) { | ||||
1670 | // Look for this pattern: {and/or} (icmp X, C0), (icmp X, C1)). | ||||
1671 | if (Cmp0->getOperand(0) != Cmp1->getOperand(0)) | ||||
1672 | return nullptr; | ||||
1673 | |||||
1674 | const APInt *C0, *C1; | ||||
1675 | if (!match(Cmp0->getOperand(1), m_APInt(C0)) || | ||||
1676 | !match(Cmp1->getOperand(1), m_APInt(C1))) | ||||
1677 | return nullptr; | ||||
1678 | |||||
1679 | auto Range0 = ConstantRange::makeExactICmpRegion(Cmp0->getPredicate(), *C0); | ||||
1680 | auto Range1 = ConstantRange::makeExactICmpRegion(Cmp1->getPredicate(), *C1); | ||||
1681 | |||||
1682 | // For and-of-compares, check if the intersection is empty: | ||||
1683 | // (icmp X, C0) && (icmp X, C1) --> empty set --> false | ||||
1684 | if (IsAnd && Range0.intersectWith(Range1).isEmptySet()) | ||||
1685 | return getFalse(Cmp0->getType()); | ||||
1686 | |||||
1687 | // For or-of-compares, check if the union is full: | ||||
1688 | // (icmp X, C0) || (icmp X, C1) --> full set --> true | ||||
1689 | if (!IsAnd && Range0.unionWith(Range1).isFullSet()) | ||||
1690 | return getTrue(Cmp0->getType()); | ||||
1691 | |||||
1692 | // Is one range a superset of the other? | ||||
1693 | // If this is and-of-compares, take the smaller set: | ||||
1694 | // (icmp sgt X, 4) && (icmp sgt X, 42) --> icmp sgt X, 42 | ||||
1695 | // If this is or-of-compares, take the larger set: | ||||
1696 | // (icmp sgt X, 4) || (icmp sgt X, 42) --> icmp sgt X, 4 | ||||
1697 | if (Range0.contains(Range1)) | ||||
1698 | return IsAnd ? Cmp1 : Cmp0; | ||||
1699 | if (Range1.contains(Range0)) | ||||
1700 | return IsAnd ? Cmp0 : Cmp1; | ||||
1701 | |||||
1702 | return nullptr; | ||||
1703 | } | ||||
1704 | |||||
1705 | static Value *simplifyAndOrOfICmpsWithZero(ICmpInst *Cmp0, ICmpInst *Cmp1, | ||||
1706 | bool IsAnd) { | ||||
1707 | ICmpInst::Predicate P0 = Cmp0->getPredicate(), P1 = Cmp1->getPredicate(); | ||||
1708 | if (!match(Cmp0->getOperand(1), m_Zero()) || | ||||
1709 | !match(Cmp1->getOperand(1), m_Zero()) || P0 != P1) | ||||
1710 | return nullptr; | ||||
1711 | |||||
1712 | if ((IsAnd && P0 != ICmpInst::ICMP_NE) || (!IsAnd && P1 != ICmpInst::ICMP_EQ)) | ||||
1713 | return nullptr; | ||||
1714 | |||||
1715 | // We have either "(X == 0 || Y == 0)" or "(X != 0 && Y != 0)". | ||||
1716 | Value *X = Cmp0->getOperand(0); | ||||
1717 | Value *Y = Cmp1->getOperand(0); | ||||
1718 | |||||
1719 | // If one of the compares is a masked version of a (not) null check, then | ||||
1720 | // that compare implies the other, so we eliminate the other. Optionally, look | ||||
1721 | // through a pointer-to-int cast to match a null check of a pointer type. | ||||
1722 | |||||
1723 | // (X == 0) || (([ptrtoint] X & ?) == 0) --> ([ptrtoint] X & ?) == 0 | ||||
1724 | // (X == 0) || ((? & [ptrtoint] X) == 0) --> (? & [ptrtoint] X) == 0 | ||||
1725 | // (X != 0) && (([ptrtoint] X & ?) != 0) --> ([ptrtoint] X & ?) != 0 | ||||
1726 | // (X != 0) && ((? & [ptrtoint] X) != 0) --> (? & [ptrtoint] X) != 0 | ||||
1727 | if (match(Y, m_c_And(m_Specific(X), m_Value())) || | ||||
1728 | match(Y, m_c_And(m_PtrToInt(m_Specific(X)), m_Value()))) | ||||
1729 | return Cmp1; | ||||
1730 | |||||
1731 | // (([ptrtoint] Y & ?) == 0) || (Y == 0) --> ([ptrtoint] Y & ?) == 0 | ||||
1732 | // ((? & [ptrtoint] Y) == 0) || (Y == 0) --> (? & [ptrtoint] Y) == 0 | ||||
1733 | // (([ptrtoint] Y & ?) != 0) && (Y != 0) --> ([ptrtoint] Y & ?) != 0 | ||||
1734 | // ((? & [ptrtoint] Y) != 0) && (Y != 0) --> (? & [ptrtoint] Y) != 0 | ||||
1735 | if (match(X, m_c_And(m_Specific(Y), m_Value())) || | ||||
1736 | match(X, m_c_And(m_PtrToInt(m_Specific(Y)), m_Value()))) | ||||
1737 | return Cmp0; | ||||
1738 | |||||
1739 | return nullptr; | ||||
1740 | } | ||||
1741 | |||||
1742 | static Value *simplifyAndOfICmpsWithAdd(ICmpInst *Op0, ICmpInst *Op1, | ||||
1743 | const InstrInfoQuery &IIQ) { | ||||
1744 | // (icmp (add V, C0), C1) & (icmp V, C0) | ||||
1745 | ICmpInst::Predicate Pred0, Pred1; | ||||
1746 | const APInt *C0, *C1; | ||||
1747 | Value *V; | ||||
1748 | if (!match(Op0, m_ICmp(Pred0, m_Add(m_Value(V), m_APInt(C0)), m_APInt(C1)))) | ||||
1749 | return nullptr; | ||||
1750 | |||||
1751 | if (!match(Op1, m_ICmp(Pred1, m_Specific(V), m_Value()))) | ||||
1752 | return nullptr; | ||||
1753 | |||||
1754 | auto *AddInst = cast<OverflowingBinaryOperator>(Op0->getOperand(0)); | ||||
1755 | if (AddInst->getOperand(1) != Op1->getOperand(1)) | ||||
1756 | return nullptr; | ||||
1757 | |||||
1758 | Type *ITy = Op0->getType(); | ||||
1759 | bool IsNSW = IIQ.hasNoSignedWrap(AddInst); | ||||
1760 | bool IsNUW = IIQ.hasNoUnsignedWrap(AddInst); | ||||
1761 | |||||
1762 | const APInt Delta = *C1 - *C0; | ||||
1763 | if (C0->isStrictlyPositive()) { | ||||
1764 | if (Delta == 2) { | ||||
1765 | if (Pred0 == ICmpInst::ICMP_ULT && Pred1 == ICmpInst::ICMP_SGT) | ||||
1766 | return getFalse(ITy); | ||||
1767 | if (Pred0 == ICmpInst::ICMP_SLT && Pred1 == ICmpInst::ICMP_SGT && IsNSW) | ||||
1768 | return getFalse(ITy); | ||||
1769 | } | ||||
1770 | if (Delta == 1) { | ||||
1771 | if (Pred0 == ICmpInst::ICMP_ULE && Pred1 == ICmpInst::ICMP_SGT) | ||||
1772 | return getFalse(ITy); | ||||
1773 | if (Pred0 == ICmpInst::ICMP_SLE && Pred1 == ICmpInst::ICMP_SGT && IsNSW) | ||||
1774 | return getFalse(ITy); | ||||
1775 | } | ||||
1776 | } | ||||
1777 | if (C0->getBoolValue() && IsNUW) { | ||||
1778 | if (Delta == 2) | ||||
1779 | if (Pred0 == ICmpInst::ICMP_ULT && Pred1 == ICmpInst::ICMP_UGT) | ||||
1780 | return getFalse(ITy); | ||||
1781 | if (Delta == 1) | ||||
1782 | if (Pred0 == ICmpInst::ICMP_ULE && Pred1 == ICmpInst::ICMP_UGT) | ||||
1783 | return getFalse(ITy); | ||||
1784 | } | ||||
1785 | |||||
1786 | return nullptr; | ||||
1787 | } | ||||
1788 | |||||
1789 | /// Try to eliminate compares with signed or unsigned min/max constants. | ||||
1790 | static Value *simplifyAndOrOfICmpsWithLimitConst(ICmpInst *Cmp0, ICmpInst *Cmp1, | ||||
1791 | bool IsAnd) { | ||||
1792 | // Canonicalize an equality compare as Cmp0. | ||||
1793 | if (Cmp1->isEquality()) | ||||
1794 | std::swap(Cmp0, Cmp1); | ||||
1795 | if (!Cmp0->isEquality()) | ||||
1796 | return nullptr; | ||||
1797 | |||||
1798 | // The non-equality compare must include a common operand (X). Canonicalize | ||||
1799 | // the common operand as operand 0 (the predicate is swapped if the common | ||||
1800 | // operand was operand 1). | ||||
1801 | ICmpInst::Predicate Pred0 = Cmp0->getPredicate(); | ||||
1802 | Value *X = Cmp0->getOperand(0); | ||||
1803 | ICmpInst::Predicate Pred1; | ||||
1804 | bool HasNotOp = match(Cmp1, m_c_ICmp(Pred1, m_Not(m_Specific(X)), m_Value())); | ||||
1805 | if (!HasNotOp && !match(Cmp1, m_c_ICmp(Pred1, m_Specific(X), m_Value()))) | ||||
1806 | return nullptr; | ||||
1807 | if (ICmpInst::isEquality(Pred1)) | ||||
1808 | return nullptr; | ||||
1809 | |||||
1810 | // The equality compare must be against a constant. Flip bits if we matched | ||||
1811 | // a bitwise not. Convert a null pointer constant to an integer zero value. | ||||
1812 | APInt MinMaxC; | ||||
1813 | const APInt *C; | ||||
1814 | if (match(Cmp0->getOperand(1), m_APInt(C))) | ||||
1815 | MinMaxC = HasNotOp ? ~*C : *C; | ||||
1816 | else if (isa<ConstantPointerNull>(Cmp0->getOperand(1))) | ||||
1817 | MinMaxC = APInt::getZero(8); | ||||
1818 | else | ||||
1819 | return nullptr; | ||||
1820 | |||||
1821 | // DeMorganize if this is 'or': P0 || P1 --> !P0 && !P1. | ||||
1822 | if (!IsAnd) { | ||||
1823 | Pred0 = ICmpInst::getInversePredicate(Pred0); | ||||
1824 | Pred1 = ICmpInst::getInversePredicate(Pred1); | ||||
1825 | } | ||||
1826 | |||||
1827 | // Normalize to unsigned compare and unsigned min/max value. | ||||
1828 | // Example for 8-bit: -128 + 128 -> 0; 127 + 128 -> 255 | ||||
1829 | if (ICmpInst::isSigned(Pred1)) { | ||||
1830 | Pred1 = ICmpInst::getUnsignedPredicate(Pred1); | ||||
1831 | MinMaxC += APInt::getSignedMinValue(MinMaxC.getBitWidth()); | ||||
1832 | } | ||||
1833 | |||||
1834 | // (X != MAX) && (X < Y) --> X < Y | ||||
1835 | // (X == MAX) || (X >= Y) --> X >= Y | ||||
1836 | if (MinMaxC.isMaxValue()) | ||||
1837 | if (Pred0 == ICmpInst::ICMP_NE && Pred1 == ICmpInst::ICMP_ULT) | ||||
1838 | return Cmp1; | ||||
1839 | |||||
1840 | // (X != MIN) && (X > Y) --> X > Y | ||||
1841 | // (X == MIN) || (X <= Y) --> X <= Y | ||||
1842 | if (MinMaxC.isMinValue()) | ||||
1843 | if (Pred0 == ICmpInst::ICMP_NE && Pred1 == ICmpInst::ICMP_UGT) | ||||
1844 | return Cmp1; | ||||
1845 | |||||
1846 | return nullptr; | ||||
1847 | } | ||||
1848 | |||||
1849 | /// Try to simplify and/or of icmp with ctpop intrinsic. | ||||
1850 | static Value *simplifyAndOrOfICmpsWithCtpop(ICmpInst *Cmp0, ICmpInst *Cmp1, | ||||
1851 | bool IsAnd) { | ||||
1852 | ICmpInst::Predicate Pred0, Pred1; | ||||
1853 | Value *X; | ||||
1854 | const APInt *C; | ||||
1855 | if (!match(Cmp0, m_ICmp(Pred0, m_Intrinsic<Intrinsic::ctpop>(m_Value(X)), | ||||
1856 | m_APInt(C))) || | ||||
1857 | !match(Cmp1, m_ICmp(Pred1, m_Specific(X), m_ZeroInt())) || C->isZero()) | ||||
1858 | return nullptr; | ||||
1859 | |||||
1860 | // (ctpop(X) == C) || (X != 0) --> X != 0 where C > 0 | ||||
1861 | if (!IsAnd && Pred0 == ICmpInst::ICMP_EQ && Pred1 == ICmpInst::ICMP_NE) | ||||
1862 | return Cmp1; | ||||
1863 | // (ctpop(X) != C) && (X == 0) --> X == 0 where C > 0 | ||||
1864 | if (IsAnd && Pred0 == ICmpInst::ICMP_NE && Pred1 == ICmpInst::ICMP_EQ) | ||||
1865 | return Cmp1; | ||||
1866 | |||||
1867 | return nullptr; | ||||
1868 | } | ||||
1869 | |||||
1870 | static Value *simplifyAndOfICmps(ICmpInst *Op0, ICmpInst *Op1, | ||||
1871 | const SimplifyQuery &Q) { | ||||
1872 | if (Value *X = simplifyUnsignedRangeCheck(Op0, Op1, /*IsAnd=*/true, Q)) | ||||
1873 | return X; | ||||
1874 | if (Value *X = simplifyUnsignedRangeCheck(Op1, Op0, /*IsAnd=*/true, Q)) | ||||
1875 | return X; | ||||
1876 | |||||
1877 | if (Value *X = simplifyAndOrOfICmpsWithConstants(Op0, Op1, true)) | ||||
1878 | return X; | ||||
1879 | |||||
1880 | if (Value *X = simplifyAndOrOfICmpsWithLimitConst(Op0, Op1, true)) | ||||
1881 | return X; | ||||
1882 | |||||
1883 | if (Value *X = simplifyAndOrOfICmpsWithZero(Op0, Op1, true)) | ||||
1884 | return X; | ||||
1885 | |||||
1886 | if (Value *X = simplifyAndOrOfICmpsWithCtpop(Op0, Op1, true)) | ||||
1887 | return X; | ||||
1888 | if (Value *X = simplifyAndOrOfICmpsWithCtpop(Op1, Op0, true)) | ||||
1889 | return X; | ||||
1890 | |||||
1891 | if (Value *X = simplifyAndOfICmpsWithAdd(Op0, Op1, Q.IIQ)) | ||||
1892 | return X; | ||||
1893 | if (Value *X = simplifyAndOfICmpsWithAdd(Op1, Op0, Q.IIQ)) | ||||
1894 | return X; | ||||
1895 | |||||
1896 | return nullptr; | ||||
1897 | } | ||||
1898 | |||||
1899 | static Value *simplifyOrOfICmpsWithAdd(ICmpInst *Op0, ICmpInst *Op1, | ||||
1900 | const InstrInfoQuery &IIQ) { | ||||
1901 | // (icmp (add V, C0), C1) | (icmp V, C0) | ||||
1902 | ICmpInst::Predicate Pred0, Pred1; | ||||
1903 | const APInt *C0, *C1; | ||||
1904 | Value *V; | ||||
1905 | if (!match(Op0, m_ICmp(Pred0, m_Add(m_Value(V), m_APInt(C0)), m_APInt(C1)))) | ||||
1906 | return nullptr; | ||||
1907 | |||||
1908 | if (!match(Op1, m_ICmp(Pred1, m_Specific(V), m_Value()))) | ||||
1909 | return nullptr; | ||||
1910 | |||||
1911 | auto *AddInst = cast<BinaryOperator>(Op0->getOperand(0)); | ||||
1912 | if (AddInst->getOperand(1) != Op1->getOperand(1)) | ||||
1913 | return nullptr; | ||||
1914 | |||||
1915 | Type *ITy = Op0->getType(); | ||||
1916 | bool IsNSW = IIQ.hasNoSignedWrap(AddInst); | ||||
1917 | bool IsNUW = IIQ.hasNoUnsignedWrap(AddInst); | ||||
1918 | |||||
1919 | const APInt Delta = *C1 - *C0; | ||||
1920 | if (C0->isStrictlyPositive()) { | ||||
1921 | if (Delta == 2) { | ||||
1922 | if (Pred0 == ICmpInst::ICMP_UGE && Pred1 == ICmpInst::ICMP_SLE) | ||||
1923 | return getTrue(ITy); | ||||
1924 | if (Pred0 == ICmpInst::ICMP_SGE && Pred1 == ICmpInst::ICMP_SLE && IsNSW) | ||||
1925 | return getTrue(ITy); | ||||
1926 | } | ||||
1927 | if (Delta == 1) { | ||||
1928 | if (Pred0 == ICmpInst::ICMP_UGT && Pred1 == ICmpInst::ICMP_SLE) | ||||
1929 | return getTrue(ITy); | ||||
1930 | if (Pred0 == ICmpInst::ICMP_SGT && Pred1 == ICmpInst::ICMP_SLE && IsNSW) | ||||
1931 | return getTrue(ITy); | ||||
1932 | } | ||||
1933 | } | ||||
1934 | if (C0->getBoolValue() && IsNUW) { | ||||
1935 | if (Delta == 2) | ||||
1936 | if (Pred0 == ICmpInst::ICMP_UGE && Pred1 == ICmpInst::ICMP_ULE) | ||||
1937 | return getTrue(ITy); | ||||
1938 | if (Delta == 1) | ||||
1939 | if (Pred0 == ICmpInst::ICMP_UGT && Pred1 == ICmpInst::ICMP_ULE) | ||||
1940 | return getTrue(ITy); | ||||
1941 | } | ||||
1942 | |||||
1943 | return nullptr; | ||||
1944 | } | ||||
1945 | |||||
1946 | static Value *simplifyOrOfICmps(ICmpInst *Op0, ICmpInst *Op1, | ||||
1947 | const SimplifyQuery &Q) { | ||||
1948 | if (Value *X = simplifyUnsignedRangeCheck(Op0, Op1, /*IsAnd=*/false, Q)) | ||||
1949 | return X; | ||||
1950 | if (Value *X = simplifyUnsignedRangeCheck(Op1, Op0, /*IsAnd=*/false, Q)) | ||||
1951 | return X; | ||||
1952 | |||||
1953 | if (Value *X = simplifyAndOrOfICmpsWithConstants(Op0, Op1, false)) | ||||
1954 | return X; | ||||
1955 | |||||
1956 | if (Value *X = simplifyAndOrOfICmpsWithLimitConst(Op0, Op1, false)) | ||||
1957 | return X; | ||||
1958 | |||||
1959 | if (Value *X = simplifyAndOrOfICmpsWithZero(Op0, Op1, false)) | ||||
1960 | return X; | ||||
1961 | |||||
1962 | if (Value *X = simplifyAndOrOfICmpsWithCtpop(Op0, Op1, false)) | ||||
1963 | return X; | ||||
1964 | if (Value *X = simplifyAndOrOfICmpsWithCtpop(Op1, Op0, false)) | ||||
1965 | return X; | ||||
1966 | |||||
1967 | if (Value *X = simplifyOrOfICmpsWithAdd(Op0, Op1, Q.IIQ)) | ||||
1968 | return X; | ||||
1969 | if (Value *X = simplifyOrOfICmpsWithAdd(Op1, Op0, Q.IIQ)) | ||||
1970 | return X; | ||||
1971 | |||||
1972 | return nullptr; | ||||
1973 | } | ||||
1974 | |||||
1975 | static Value *simplifyAndOrOfFCmps(const TargetLibraryInfo *TLI, FCmpInst *LHS, | ||||
1976 | FCmpInst *RHS, bool IsAnd) { | ||||
1977 | Value *LHS0 = LHS->getOperand(0), *LHS1 = LHS->getOperand(1); | ||||
1978 | Value *RHS0 = RHS->getOperand(0), *RHS1 = RHS->getOperand(1); | ||||
1979 | if (LHS0->getType() != RHS0->getType()) | ||||
1980 | return nullptr; | ||||
1981 | |||||
1982 | FCmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate(); | ||||
1983 | if ((PredL == FCmpInst::FCMP_ORD && PredR == FCmpInst::FCMP_ORD && IsAnd) || | ||||
1984 | (PredL == FCmpInst::FCMP_UNO && PredR == FCmpInst::FCMP_UNO && !IsAnd)) { | ||||
1985 | // (fcmp ord NNAN, X) & (fcmp ord X, Y) --> fcmp ord X, Y | ||||
1986 | // (fcmp ord NNAN, X) & (fcmp ord Y, X) --> fcmp ord Y, X | ||||
1987 | // (fcmp ord X, NNAN) & (fcmp ord X, Y) --> fcmp ord X, Y | ||||
1988 | // (fcmp ord X, NNAN) & (fcmp ord Y, X) --> fcmp ord Y, X | ||||
1989 | // (fcmp uno NNAN, X) | (fcmp uno X, Y) --> fcmp uno X, Y | ||||
1990 | // (fcmp uno NNAN, X) | (fcmp uno Y, X) --> fcmp uno Y, X | ||||
1991 | // (fcmp uno X, NNAN) | (fcmp uno X, Y) --> fcmp uno X, Y | ||||
1992 | // (fcmp uno X, NNAN) | (fcmp uno Y, X) --> fcmp uno Y, X | ||||
1993 | if ((isKnownNeverNaN(LHS0, TLI) && (LHS1 == RHS0 || LHS1 == RHS1)) || | ||||
1994 | (isKnownNeverNaN(LHS1, TLI) && (LHS0 == RHS0 || LHS0 == RHS1))) | ||||
1995 | return RHS; | ||||
1996 | |||||
1997 | // (fcmp ord X, Y) & (fcmp ord NNAN, X) --> fcmp ord X, Y | ||||
1998 | // (fcmp ord Y, X) & (fcmp ord NNAN, X) --> fcmp ord Y, X | ||||
1999 | // (fcmp ord X, Y) & (fcmp ord X, NNAN) --> fcmp ord X, Y | ||||
2000 | // (fcmp ord Y, X) & (fcmp ord X, NNAN) --> fcmp ord Y, X | ||||
2001 | // (fcmp uno X, Y) | (fcmp uno NNAN, X) --> fcmp uno X, Y | ||||
2002 | // (fcmp uno Y, X) | (fcmp uno NNAN, X) --> fcmp uno Y, X | ||||
2003 | // (fcmp uno X, Y) | (fcmp uno X, NNAN) --> fcmp uno X, Y | ||||
2004 | // (fcmp uno Y, X) | (fcmp uno X, NNAN) --> fcmp uno Y, X | ||||
2005 | if ((isKnownNeverNaN(RHS0, TLI) && (RHS1 == LHS0 || RHS1 == LHS1)) || | ||||
2006 | (isKnownNeverNaN(RHS1, TLI) && (RHS0 == LHS0 || RHS0 == LHS1))) | ||||
2007 | return LHS; | ||||
2008 | } | ||||
2009 | |||||
2010 | return nullptr; | ||||
2011 | } | ||||
2012 | |||||
2013 | static Value *simplifyAndOrOfCmps(const SimplifyQuery &Q, Value *Op0, | ||||
2014 | Value *Op1, bool IsAnd) { | ||||
2015 | // Look through casts of the 'and' operands to find compares. | ||||
2016 | auto *Cast0 = dyn_cast<CastInst>(Op0); | ||||
2017 | auto *Cast1 = dyn_cast<CastInst>(Op1); | ||||
2018 | if (Cast0 && Cast1 && Cast0->getOpcode() == Cast1->getOpcode() && | ||||
2019 | Cast0->getSrcTy() == Cast1->getSrcTy()) { | ||||
2020 | Op0 = Cast0->getOperand(0); | ||||
2021 | Op1 = Cast1->getOperand(0); | ||||
2022 | } | ||||
2023 | |||||
2024 | Value *V = nullptr; | ||||
2025 | auto *ICmp0 = dyn_cast<ICmpInst>(Op0); | ||||
2026 | auto *ICmp1 = dyn_cast<ICmpInst>(Op1); | ||||
2027 | if (ICmp0 && ICmp1) | ||||
2028 | V = IsAnd ? simplifyAndOfICmps(ICmp0, ICmp1, Q) | ||||
2029 | : simplifyOrOfICmps(ICmp0, ICmp1, Q); | ||||
2030 | |||||
2031 | auto *FCmp0 = dyn_cast<FCmpInst>(Op0); | ||||
2032 | auto *FCmp1 = dyn_cast<FCmpInst>(Op1); | ||||
2033 | if (FCmp0 && FCmp1) | ||||
2034 | V = simplifyAndOrOfFCmps(Q.TLI, FCmp0, FCmp1, IsAnd); | ||||
2035 | |||||
2036 | if (!V) | ||||
2037 | return nullptr; | ||||
2038 | if (!Cast0) | ||||
2039 | return V; | ||||
2040 | |||||
2041 | // If we looked through casts, we can only handle a constant simplification | ||||
2042 | // because we are not allowed to create a cast instruction here. | ||||
2043 | if (auto *C = dyn_cast<Constant>(V)) | ||||
2044 | return ConstantExpr::getCast(Cast0->getOpcode(), C, Cast0->getType()); | ||||
2045 | |||||
2046 | return nullptr; | ||||
2047 | } | ||||
2048 | |||||
2049 | /// Given a bitwise logic op, check if the operands are add/sub with a common | ||||
2050 | /// source value and inverted constant (identity: C - X -> ~(X + ~C)). | ||||
2051 | static Value *simplifyLogicOfAddSub(Value *Op0, Value *Op1, | ||||
2052 | Instruction::BinaryOps Opcode) { | ||||
2053 | assert(Op0->getType() == Op1->getType() && "Mismatched binop types")(static_cast <bool> (Op0->getType() == Op1->getType () && "Mismatched binop types") ? void (0) : __assert_fail ("Op0->getType() == Op1->getType() && \"Mismatched binop types\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 2053, __extension__ __PRETTY_FUNCTION__)); | ||||
2054 | assert(BinaryOperator::isBitwiseLogicOp(Opcode) && "Expected logic op")(static_cast <bool> (BinaryOperator::isBitwiseLogicOp(Opcode ) && "Expected logic op") ? void (0) : __assert_fail ( "BinaryOperator::isBitwiseLogicOp(Opcode) && \"Expected logic op\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 2054, __extension__ __PRETTY_FUNCTION__)); | ||||
2055 | Value *X; | ||||
2056 | Constant *C1, *C2; | ||||
2057 | if ((match(Op0, m_Add(m_Value(X), m_Constant(C1))) && | ||||
2058 | match(Op1, m_Sub(m_Constant(C2), m_Specific(X)))) || | ||||
2059 | (match(Op1, m_Add(m_Value(X), m_Constant(C1))) && | ||||
2060 | match(Op0, m_Sub(m_Constant(C2), m_Specific(X))))) { | ||||
2061 | if (ConstantExpr::getNot(C1) == C2) { | ||||
2062 | // (X + C) & (~C - X) --> (X + C) & ~(X + C) --> 0 | ||||
2063 | // (X + C) | (~C - X) --> (X + C) | ~(X + C) --> -1 | ||||
2064 | // (X + C) ^ (~C - X) --> (X + C) ^ ~(X + C) --> -1 | ||||
2065 | Type *Ty = Op0->getType(); | ||||
2066 | return Opcode == Instruction::And ? ConstantInt::getNullValue(Ty) | ||||
2067 | : ConstantInt::getAllOnesValue(Ty); | ||||
2068 | } | ||||
2069 | } | ||||
2070 | return nullptr; | ||||
2071 | } | ||||
2072 | |||||
2073 | /// Given operands for an And, see if we can fold the result. | ||||
2074 | /// If not, this returns null. | ||||
2075 | static Value *simplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, | ||||
2076 | unsigned MaxRecurse) { | ||||
2077 | if (Constant *C = foldOrCommuteConstant(Instruction::And, Op0, Op1, Q)) | ||||
2078 | return C; | ||||
2079 | |||||
2080 | // X & poison -> poison | ||||
2081 | if (isa<PoisonValue>(Op1)) | ||||
2082 | return Op1; | ||||
2083 | |||||
2084 | // X & undef -> 0 | ||||
2085 | if (Q.isUndefValue(Op1)) | ||||
2086 | return Constant::getNullValue(Op0->getType()); | ||||
2087 | |||||
2088 | // X & X = X | ||||
2089 | if (Op0 == Op1) | ||||
2090 | return Op0; | ||||
2091 | |||||
2092 | // X & 0 = 0 | ||||
2093 | if (match(Op1, m_Zero())) | ||||
2094 | return Constant::getNullValue(Op0->getType()); | ||||
2095 | |||||
2096 | // X & -1 = X | ||||
2097 | if (match(Op1, m_AllOnes())) | ||||
2098 | return Op0; | ||||
2099 | |||||
2100 | // A & ~A = ~A & A = 0 | ||||
2101 | if (match(Op0, m_Not(m_Specific(Op1))) || match(Op1, m_Not(m_Specific(Op0)))) | ||||
2102 | return Constant::getNullValue(Op0->getType()); | ||||
2103 | |||||
2104 | // (A | ?) & A = A | ||||
2105 | if (match(Op0, m_c_Or(m_Specific(Op1), m_Value()))) | ||||
2106 | return Op1; | ||||
2107 | |||||
2108 | // A & (A | ?) = A | ||||
2109 | if (match(Op1, m_c_Or(m_Specific(Op0), m_Value()))) | ||||
2110 | return Op0; | ||||
2111 | |||||
2112 | // (X | Y) & (X | ~Y) --> X (commuted 8 ways) | ||||
2113 | Value *X, *Y; | ||||
2114 | if (match(Op0, m_c_Or(m_Value(X), m_Not(m_Value(Y)))) && | ||||
2115 | match(Op1, m_c_Or(m_Deferred(X), m_Deferred(Y)))) | ||||
2116 | return X; | ||||
2117 | if (match(Op1, m_c_Or(m_Value(X), m_Not(m_Value(Y)))) && | ||||
2118 | match(Op0, m_c_Or(m_Deferred(X), m_Deferred(Y)))) | ||||
2119 | return X; | ||||
2120 | |||||
2121 | if (Value *V = simplifyLogicOfAddSub(Op0, Op1, Instruction::And)) | ||||
2122 | return V; | ||||
2123 | |||||
2124 | // A mask that only clears known zeros of a shifted value is a no-op. | ||||
2125 | const APInt *Mask; | ||||
2126 | const APInt *ShAmt; | ||||
2127 | if (match(Op1, m_APInt(Mask))) { | ||||
2128 | // If all bits in the inverted and shifted mask are clear: | ||||
2129 | // and (shl X, ShAmt), Mask --> shl X, ShAmt | ||||
2130 | if (match(Op0, m_Shl(m_Value(X), m_APInt(ShAmt))) && | ||||
2131 | (~(*Mask)).lshr(*ShAmt).isZero()) | ||||
2132 | return Op0; | ||||
2133 | |||||
2134 | // If all bits in the inverted and shifted mask are clear: | ||||
2135 | // and (lshr X, ShAmt), Mask --> lshr X, ShAmt | ||||
2136 | if (match(Op0, m_LShr(m_Value(X), m_APInt(ShAmt))) && | ||||
2137 | (~(*Mask)).shl(*ShAmt).isZero()) | ||||
2138 | return Op0; | ||||
2139 | } | ||||
2140 | |||||
2141 | // If we have a multiplication overflow check that is being 'and'ed with a | ||||
2142 | // check that one of the multipliers is not zero, we can omit the 'and', and | ||||
2143 | // only keep the overflow check. | ||||
2144 | if (isCheckForZeroAndMulWithOverflow(Op0, Op1, true)) | ||||
2145 | return Op1; | ||||
2146 | if (isCheckForZeroAndMulWithOverflow(Op1, Op0, true)) | ||||
2147 | return Op0; | ||||
2148 | |||||
2149 | // A & (-A) = A if A is a power of two or zero. | ||||
2150 | if (match(Op0, m_Neg(m_Specific(Op1))) || | ||||
2151 | match(Op1, m_Neg(m_Specific(Op0)))) { | ||||
2152 | if (isKnownToBeAPowerOfTwo(Op0, Q.DL, /*OrZero*/ true, 0, Q.AC, Q.CxtI, | ||||
2153 | Q.DT)) | ||||
2154 | return Op0; | ||||
2155 | if (isKnownToBeAPowerOfTwo(Op1, Q.DL, /*OrZero*/ true, 0, Q.AC, Q.CxtI, | ||||
2156 | Q.DT)) | ||||
2157 | return Op1; | ||||
2158 | } | ||||
2159 | |||||
2160 | // This is a similar pattern used for checking if a value is a power-of-2: | ||||
2161 | // (A - 1) & A --> 0 (if A is a power-of-2 or 0) | ||||
2162 | // A & (A - 1) --> 0 (if A is a power-of-2 or 0) | ||||
2163 | if (match(Op0, m_Add(m_Specific(Op1), m_AllOnes())) && | ||||
2164 | isKnownToBeAPowerOfTwo(Op1, Q.DL, /*OrZero*/ true, 0, Q.AC, Q.CxtI, Q.DT)) | ||||
2165 | return Constant::getNullValue(Op1->getType()); | ||||
2166 | if (match(Op1, m_Add(m_Specific(Op0), m_AllOnes())) && | ||||
2167 | isKnownToBeAPowerOfTwo(Op0, Q.DL, /*OrZero*/ true, 0, Q.AC, Q.CxtI, Q.DT)) | ||||
2168 | return Constant::getNullValue(Op0->getType()); | ||||
2169 | |||||
2170 | if (Value *V = simplifyAndOrOfCmps(Q, Op0, Op1, true)) | ||||
2171 | return V; | ||||
2172 | |||||
2173 | // Try some generic simplifications for associative operations. | ||||
2174 | if (Value *V = | ||||
2175 | simplifyAssociativeBinOp(Instruction::And, Op0, Op1, Q, MaxRecurse)) | ||||
2176 | return V; | ||||
2177 | |||||
2178 | // And distributes over Or. Try some generic simplifications based on this. | ||||
2179 | if (Value *V = expandCommutativeBinOp(Instruction::And, Op0, Op1, | ||||
2180 | Instruction::Or, Q, MaxRecurse)) | ||||
2181 | return V; | ||||
2182 | |||||
2183 | // And distributes over Xor. Try some generic simplifications based on this. | ||||
2184 | if (Value *V = expandCommutativeBinOp(Instruction::And, Op0, Op1, | ||||
2185 | Instruction::Xor, Q, MaxRecurse)) | ||||
2186 | return V; | ||||
2187 | |||||
2188 | if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1)) { | ||||
2189 | if (Op0->getType()->isIntOrIntVectorTy(1)) { | ||||
2190 | // A & (A && B) -> A && B | ||||
2191 | if (match(Op1, m_Select(m_Specific(Op0), m_Value(), m_Zero()))) | ||||
2192 | return Op1; | ||||
2193 | else if (match(Op0, m_Select(m_Specific(Op1), m_Value(), m_Zero()))) | ||||
2194 | return Op0; | ||||
2195 | } | ||||
2196 | // If the operation is with the result of a select instruction, check | ||||
2197 | // whether operating on either branch of the select always yields the same | ||||
2198 | // value. | ||||
2199 | if (Value *V = | ||||
2200 | threadBinOpOverSelect(Instruction::And, Op0, Op1, Q, MaxRecurse)) | ||||
2201 | return V; | ||||
2202 | } | ||||
2203 | |||||
2204 | // If the operation is with the result of a phi instruction, check whether | ||||
2205 | // operating on all incoming values of the phi always yields the same value. | ||||
2206 | if (isa<PHINode>(Op0) || isa<PHINode>(Op1)) | ||||
2207 | if (Value *V = | ||||
2208 | threadBinOpOverPHI(Instruction::And, Op0, Op1, Q, MaxRecurse)) | ||||
2209 | return V; | ||||
2210 | |||||
2211 | // Assuming the effective width of Y is not larger than A, i.e. all bits | ||||
2212 | // from X and Y are disjoint in (X << A) | Y, | ||||
2213 | // if the mask of this AND op covers all bits of X or Y, while it covers | ||||
2214 | // no bits from the other, we can bypass this AND op. E.g., | ||||
2215 | // ((X << A) | Y) & Mask -> Y, | ||||
2216 | // if Mask = ((1 << effective_width_of(Y)) - 1) | ||||
2217 | // ((X << A) | Y) & Mask -> X << A, | ||||
2218 | // if Mask = ((1 << effective_width_of(X)) - 1) << A | ||||
2219 | // SimplifyDemandedBits in InstCombine can optimize the general case. | ||||
2220 | // This pattern aims to help other passes for a common case. | ||||
2221 | Value *XShifted; | ||||
2222 | if (match(Op1, m_APInt(Mask)) && | ||||
2223 | match(Op0, m_c_Or(m_CombineAnd(m_NUWShl(m_Value(X), m_APInt(ShAmt)), | ||||
2224 | m_Value(XShifted)), | ||||
2225 | m_Value(Y)))) { | ||||
2226 | const unsigned Width = Op0->getType()->getScalarSizeInBits(); | ||||
2227 | const unsigned ShftCnt = ShAmt->getLimitedValue(Width); | ||||
2228 | const KnownBits YKnown = computeKnownBits(Y, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
2229 | const unsigned EffWidthY = YKnown.countMaxActiveBits(); | ||||
2230 | if (EffWidthY <= ShftCnt) { | ||||
2231 | const KnownBits XKnown = computeKnownBits(X, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
2232 | const unsigned EffWidthX = XKnown.countMaxActiveBits(); | ||||
2233 | const APInt EffBitsY = APInt::getLowBitsSet(Width, EffWidthY); | ||||
2234 | const APInt EffBitsX = APInt::getLowBitsSet(Width, EffWidthX) << ShftCnt; | ||||
2235 | // If the mask is extracting all bits from X or Y as is, we can skip | ||||
2236 | // this AND op. | ||||
2237 | if (EffBitsY.isSubsetOf(*Mask) && !EffBitsX.intersects(*Mask)) | ||||
2238 | return Y; | ||||
2239 | if (EffBitsX.isSubsetOf(*Mask) && !EffBitsY.intersects(*Mask)) | ||||
2240 | return XShifted; | ||||
2241 | } | ||||
2242 | } | ||||
2243 | |||||
2244 | // ((X | Y) ^ X ) & ((X | Y) ^ Y) --> 0 | ||||
2245 | // ((X | Y) ^ Y ) & ((X | Y) ^ X) --> 0 | ||||
2246 | BinaryOperator *Or; | ||||
2247 | if (match(Op0, m_c_Xor(m_Value(X), | ||||
2248 | m_CombineAnd(m_BinOp(Or), | ||||
2249 | m_c_Or(m_Deferred(X), m_Value(Y))))) && | ||||
2250 | match(Op1, m_c_Xor(m_Specific(Or), m_Specific(Y)))) | ||||
2251 | return Constant::getNullValue(Op0->getType()); | ||||
2252 | |||||
2253 | if (Op0->getType()->isIntOrIntVectorTy(1)) { | ||||
2254 | if (std::optional<bool> Implied = isImpliedCondition(Op0, Op1, Q.DL)) { | ||||
2255 | // If Op0 is true implies Op1 is true, then Op0 is a subset of Op1. | ||||
2256 | if (*Implied == true) | ||||
2257 | return Op0; | ||||
2258 | // If Op0 is true implies Op1 is false, then they are not true together. | ||||
2259 | if (*Implied == false) | ||||
2260 | return ConstantInt::getFalse(Op0->getType()); | ||||
2261 | } | ||||
2262 | if (std::optional<bool> Implied = isImpliedCondition(Op1, Op0, Q.DL)) { | ||||
2263 | // If Op1 is true implies Op0 is true, then Op1 is a subset of Op0. | ||||
2264 | if (*Implied) | ||||
2265 | return Op1; | ||||
2266 | // If Op1 is true implies Op0 is false, then they are not true together. | ||||
2267 | if (!*Implied) | ||||
2268 | return ConstantInt::getFalse(Op1->getType()); | ||||
2269 | } | ||||
2270 | } | ||||
2271 | |||||
2272 | if (Value *V = simplifyByDomEq(Instruction::And, Op0, Op1, Q, MaxRecurse)) | ||||
2273 | return V; | ||||
2274 | |||||
2275 | return nullptr; | ||||
2276 | } | ||||
2277 | |||||
2278 | Value *llvm::simplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { | ||||
2279 | return ::simplifyAndInst(Op0, Op1, Q, RecursionLimit); | ||||
2280 | } | ||||
2281 | |||||
2282 | // TODO: Many of these folds could use LogicalAnd/LogicalOr. | ||||
2283 | static Value *simplifyOrLogic(Value *X, Value *Y) { | ||||
2284 | assert(X->getType() == Y->getType() && "Expected same type for 'or' ops")(static_cast <bool> (X->getType() == Y->getType() && "Expected same type for 'or' ops") ? void (0) : __assert_fail ("X->getType() == Y->getType() && \"Expected same type for 'or' ops\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 2284, __extension__ __PRETTY_FUNCTION__)); | ||||
2285 | Type *Ty = X->getType(); | ||||
2286 | |||||
2287 | // X | ~X --> -1 | ||||
2288 | if (match(Y, m_Not(m_Specific(X)))) | ||||
2289 | return ConstantInt::getAllOnesValue(Ty); | ||||
2290 | |||||
2291 | // X | ~(X & ?) = -1 | ||||
2292 | if (match(Y, m_Not(m_c_And(m_Specific(X), m_Value())))) | ||||
2293 | return ConstantInt::getAllOnesValue(Ty); | ||||
2294 | |||||
2295 | // X | (X & ?) --> X | ||||
2296 | if (match(Y, m_c_And(m_Specific(X), m_Value()))) | ||||
2297 | return X; | ||||
2298 | |||||
2299 | Value *A, *B; | ||||
2300 | |||||
2301 | // (A ^ B) | (A | B) --> A | B | ||||
2302 | // (A ^ B) | (B | A) --> B | A | ||||
2303 | if (match(X, m_Xor(m_Value(A), m_Value(B))) && | ||||
2304 | match(Y, m_c_Or(m_Specific(A), m_Specific(B)))) | ||||
2305 | return Y; | ||||
2306 | |||||
2307 | // ~(A ^ B) | (A | B) --> -1 | ||||
2308 | // ~(A ^ B) | (B | A) --> -1 | ||||
2309 | if (match(X, m_Not(m_Xor(m_Value(A), m_Value(B)))) && | ||||
2310 | match(Y, m_c_Or(m_Specific(A), m_Specific(B)))) | ||||
2311 | return ConstantInt::getAllOnesValue(Ty); | ||||
2312 | |||||
2313 | // (A & ~B) | (A ^ B) --> A ^ B | ||||
2314 | // (~B & A) | (A ^ B) --> A ^ B | ||||
2315 | // (A & ~B) | (B ^ A) --> B ^ A | ||||
2316 | // (~B & A) | (B ^ A) --> B ^ A | ||||
2317 | if (match(X, m_c_And(m_Value(A), m_Not(m_Value(B)))) && | ||||
2318 | match(Y, m_c_Xor(m_Specific(A), m_Specific(B)))) | ||||
2319 | return Y; | ||||
2320 | |||||
2321 | // (~A ^ B) | (A & B) --> ~A ^ B | ||||
2322 | // (B ^ ~A) | (A & B) --> B ^ ~A | ||||
2323 | // (~A ^ B) | (B & A) --> ~A ^ B | ||||
2324 | // (B ^ ~A) | (B & A) --> B ^ ~A | ||||
2325 | if (match(X, m_c_Xor(m_NotForbidUndef(m_Value(A)), m_Value(B))) && | ||||
2326 | match(Y, m_c_And(m_Specific(A), m_Specific(B)))) | ||||
2327 | return X; | ||||
2328 | |||||
2329 | // (~A | B) | (A ^ B) --> -1 | ||||
2330 | // (~A | B) | (B ^ A) --> -1 | ||||
2331 | // (B | ~A) | (A ^ B) --> -1 | ||||
2332 | // (B | ~A) | (B ^ A) --> -1 | ||||
2333 | if (match(X, m_c_Or(m_Not(m_Value(A)), m_Value(B))) && | ||||
2334 | match(Y, m_c_Xor(m_Specific(A), m_Specific(B)))) | ||||
2335 | return ConstantInt::getAllOnesValue(Ty); | ||||
2336 | |||||
2337 | // (~A & B) | ~(A | B) --> ~A | ||||
2338 | // (~A & B) | ~(B | A) --> ~A | ||||
2339 | // (B & ~A) | ~(A | B) --> ~A | ||||
2340 | // (B & ~A) | ~(B | A) --> ~A | ||||
2341 | Value *NotA; | ||||
2342 | if (match(X, | ||||
2343 | m_c_And(m_CombineAnd(m_Value(NotA), m_NotForbidUndef(m_Value(A))), | ||||
2344 | m_Value(B))) && | ||||
2345 | match(Y, m_Not(m_c_Or(m_Specific(A), m_Specific(B))))) | ||||
2346 | return NotA; | ||||
2347 | // The same is true of Logical And | ||||
2348 | // TODO: This could share the logic of the version above if there was a | ||||
2349 | // version of LogicalAnd that allowed more than just i1 types. | ||||
2350 | if (match(X, m_c_LogicalAnd( | ||||
2351 | m_CombineAnd(m_Value(NotA), m_NotForbidUndef(m_Value(A))), | ||||
2352 | m_Value(B))) && | ||||
2353 | match(Y, m_Not(m_c_LogicalOr(m_Specific(A), m_Specific(B))))) | ||||
2354 | return NotA; | ||||
2355 | |||||
2356 | // ~(A ^ B) | (A & B) --> ~(A ^ B) | ||||
2357 | // ~(A ^ B) | (B & A) --> ~(A ^ B) | ||||
2358 | Value *NotAB; | ||||
2359 | if (match(X, m_CombineAnd(m_NotForbidUndef(m_Xor(m_Value(A), m_Value(B))), | ||||
2360 | m_Value(NotAB))) && | ||||
2361 | match(Y, m_c_And(m_Specific(A), m_Specific(B)))) | ||||
2362 | return NotAB; | ||||
2363 | |||||
2364 | // ~(A & B) | (A ^ B) --> ~(A & B) | ||||
2365 | // ~(A & B) | (B ^ A) --> ~(A & B) | ||||
2366 | if (match(X, m_CombineAnd(m_NotForbidUndef(m_And(m_Value(A), m_Value(B))), | ||||
2367 | m_Value(NotAB))) && | ||||
2368 | match(Y, m_c_Xor(m_Specific(A), m_Specific(B)))) | ||||
2369 | return NotAB; | ||||
2370 | |||||
2371 | return nullptr; | ||||
2372 | } | ||||
2373 | |||||
2374 | /// Given operands for an Or, see if we can fold the result. | ||||
2375 | /// If not, this returns null. | ||||
2376 | static Value *simplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, | ||||
2377 | unsigned MaxRecurse) { | ||||
2378 | if (Constant *C = foldOrCommuteConstant(Instruction::Or, Op0, Op1, Q)) | ||||
2379 | return C; | ||||
2380 | |||||
2381 | // X | poison -> poison | ||||
2382 | if (isa<PoisonValue>(Op1)) | ||||
2383 | return Op1; | ||||
2384 | |||||
2385 | // X | undef -> -1 | ||||
2386 | // X | -1 = -1 | ||||
2387 | // Do not return Op1 because it may contain undef elements if it's a vector. | ||||
2388 | if (Q.isUndefValue(Op1) || match(Op1, m_AllOnes())) | ||||
2389 | return Constant::getAllOnesValue(Op0->getType()); | ||||
2390 | |||||
2391 | // X | X = X | ||||
2392 | // X | 0 = X | ||||
2393 | if (Op0 == Op1 || match(Op1, m_Zero())) | ||||
2394 | return Op0; | ||||
2395 | |||||
2396 | if (Value *R = simplifyOrLogic(Op0, Op1)) | ||||
2397 | return R; | ||||
2398 | if (Value *R = simplifyOrLogic(Op1, Op0)) | ||||
2399 | return R; | ||||
2400 | |||||
2401 | if (Value *V = simplifyLogicOfAddSub(Op0, Op1, Instruction::Or)) | ||||
2402 | return V; | ||||
2403 | |||||
2404 | // Rotated -1 is still -1: | ||||
2405 | // (-1 << X) | (-1 >> (C - X)) --> -1 | ||||
2406 | // (-1 >> X) | (-1 << (C - X)) --> -1 | ||||
2407 | // ...with C <= bitwidth (and commuted variants). | ||||
2408 | Value *X, *Y; | ||||
2409 | if ((match(Op0, m_Shl(m_AllOnes(), m_Value(X))) && | ||||
2410 | match(Op1, m_LShr(m_AllOnes(), m_Value(Y)))) || | ||||
2411 | (match(Op1, m_Shl(m_AllOnes(), m_Value(X))) && | ||||
2412 | match(Op0, m_LShr(m_AllOnes(), m_Value(Y))))) { | ||||
2413 | const APInt *C; | ||||
2414 | if ((match(X, m_Sub(m_APInt(C), m_Specific(Y))) || | ||||
2415 | match(Y, m_Sub(m_APInt(C), m_Specific(X)))) && | ||||
2416 | C->ule(X->getType()->getScalarSizeInBits())) { | ||||
2417 | return ConstantInt::getAllOnesValue(X->getType()); | ||||
2418 | } | ||||
2419 | } | ||||
2420 | |||||
2421 | // A funnel shift (rotate) can be decomposed into simpler shifts. See if we | ||||
2422 | // are mixing in another shift that is redundant with the funnel shift. | ||||
2423 | |||||
2424 | // (fshl X, ?, Y) | (shl X, Y) --> fshl X, ?, Y | ||||
2425 | // (shl X, Y) | (fshl X, ?, Y) --> fshl X, ?, Y | ||||
2426 | if (match(Op0, | ||||
2427 | m_Intrinsic<Intrinsic::fshl>(m_Value(X), m_Value(), m_Value(Y))) && | ||||
2428 | match(Op1, m_Shl(m_Specific(X), m_Specific(Y)))) | ||||
2429 | return Op0; | ||||
2430 | if (match(Op1, | ||||
2431 | m_Intrinsic<Intrinsic::fshl>(m_Value(X), m_Value(), m_Value(Y))) && | ||||
2432 | match(Op0, m_Shl(m_Specific(X), m_Specific(Y)))) | ||||
2433 | return Op1; | ||||
2434 | |||||
2435 | // (fshr ?, X, Y) | (lshr X, Y) --> fshr ?, X, Y | ||||
2436 | // (lshr X, Y) | (fshr ?, X, Y) --> fshr ?, X, Y | ||||
2437 | if (match(Op0, | ||||
2438 | m_Intrinsic<Intrinsic::fshr>(m_Value(), m_Value(X), m_Value(Y))) && | ||||
2439 | match(Op1, m_LShr(m_Specific(X), m_Specific(Y)))) | ||||
2440 | return Op0; | ||||
2441 | if (match(Op1, | ||||
2442 | m_Intrinsic<Intrinsic::fshr>(m_Value(), m_Value(X), m_Value(Y))) && | ||||
2443 | match(Op0, m_LShr(m_Specific(X), m_Specific(Y)))) | ||||
2444 | return Op1; | ||||
2445 | |||||
2446 | if (Value *V = simplifyAndOrOfCmps(Q, Op0, Op1, false)) | ||||
2447 | return V; | ||||
2448 | |||||
2449 | // If we have a multiplication overflow check that is being 'and'ed with a | ||||
2450 | // check that one of the multipliers is not zero, we can omit the 'and', and | ||||
2451 | // only keep the overflow check. | ||||
2452 | if (isCheckForZeroAndMulWithOverflow(Op0, Op1, false)) | ||||
2453 | return Op1; | ||||
2454 | if (isCheckForZeroAndMulWithOverflow(Op1, Op0, false)) | ||||
2455 | return Op0; | ||||
2456 | |||||
2457 | // Try some generic simplifications for associative operations. | ||||
2458 | if (Value *V = | ||||
2459 | simplifyAssociativeBinOp(Instruction::Or, Op0, Op1, Q, MaxRecurse)) | ||||
2460 | return V; | ||||
2461 | |||||
2462 | // Or distributes over And. Try some generic simplifications based on this. | ||||
2463 | if (Value *V = expandCommutativeBinOp(Instruction::Or, Op0, Op1, | ||||
2464 | Instruction::And, Q, MaxRecurse)) | ||||
2465 | return V; | ||||
2466 | |||||
2467 | if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1)) { | ||||
2468 | if (Op0->getType()->isIntOrIntVectorTy(1)) { | ||||
2469 | // A | (A || B) -> A || B | ||||
2470 | if (match(Op1, m_Select(m_Specific(Op0), m_One(), m_Value()))) | ||||
2471 | return Op1; | ||||
2472 | else if (match(Op0, m_Select(m_Specific(Op1), m_One(), m_Value()))) | ||||
2473 | return Op0; | ||||
2474 | } | ||||
2475 | // If the operation is with the result of a select instruction, check | ||||
2476 | // whether operating on either branch of the select always yields the same | ||||
2477 | // value. | ||||
2478 | if (Value *V = | ||||
2479 | threadBinOpOverSelect(Instruction::Or, Op0, Op1, Q, MaxRecurse)) | ||||
2480 | return V; | ||||
2481 | } | ||||
2482 | |||||
2483 | // (A & C1)|(B & C2) | ||||
2484 | Value *A, *B; | ||||
2485 | const APInt *C1, *C2; | ||||
2486 | if (match(Op0, m_And(m_Value(A), m_APInt(C1))) && | ||||
2487 | match(Op1, m_And(m_Value(B), m_APInt(C2)))) { | ||||
2488 | if (*C1 == ~*C2) { | ||||
2489 | // (A & C1)|(B & C2) | ||||
2490 | // If we have: ((V + N) & C1) | (V & C2) | ||||
2491 | // .. and C2 = ~C1 and C2 is 0+1+ and (N & C2) == 0 | ||||
2492 | // replace with V+N. | ||||
2493 | Value *N; | ||||
2494 | if (C2->isMask() && // C2 == 0+1+ | ||||
2495 | match(A, m_c_Add(m_Specific(B), m_Value(N)))) { | ||||
2496 | // Add commutes, try both ways. | ||||
2497 | if (MaskedValueIsZero(N, *C2, Q.DL, 0, Q.AC, Q.CxtI, Q.DT)) | ||||
2498 | return A; | ||||
2499 | } | ||||
2500 | // Or commutes, try both ways. | ||||
2501 | if (C1->isMask() && match(B, m_c_Add(m_Specific(A), m_Value(N)))) { | ||||
2502 | // Add commutes, try both ways. | ||||
2503 | if (MaskedValueIsZero(N, *C1, Q.DL, 0, Q.AC, Q.CxtI, Q.DT)) | ||||
2504 | return B; | ||||
2505 | } | ||||
2506 | } | ||||
2507 | } | ||||
2508 | |||||
2509 | // If the operation is with the result of a phi instruction, check whether | ||||
2510 | // operating on all incoming values of the phi always yields the same value. | ||||
2511 | if (isa<PHINode>(Op0) || isa<PHINode>(Op1)) | ||||
2512 | if (Value *V = threadBinOpOverPHI(Instruction::Or, Op0, Op1, Q, MaxRecurse)) | ||||
2513 | return V; | ||||
2514 | |||||
2515 | if (Op0->getType()->isIntOrIntVectorTy(1)) { | ||||
2516 | if (std::optional<bool> Implied = | ||||
2517 | isImpliedCondition(Op0, Op1, Q.DL, false)) { | ||||
2518 | // If Op0 is false implies Op1 is false, then Op1 is a subset of Op0. | ||||
2519 | if (*Implied == false) | ||||
2520 | return Op0; | ||||
2521 | // If Op0 is false implies Op1 is true, then at least one is always true. | ||||
2522 | if (*Implied == true) | ||||
2523 | return ConstantInt::getTrue(Op0->getType()); | ||||
2524 | } | ||||
2525 | if (std::optional<bool> Implied = | ||||
2526 | isImpliedCondition(Op1, Op0, Q.DL, false)) { | ||||
2527 | // If Op1 is false implies Op0 is false, then Op0 is a subset of Op1. | ||||
2528 | if (*Implied == false) | ||||
2529 | return Op1; | ||||
2530 | // If Op1 is false implies Op0 is true, then at least one is always true. | ||||
2531 | if (*Implied == true) | ||||
2532 | return ConstantInt::getTrue(Op1->getType()); | ||||
2533 | } | ||||
2534 | } | ||||
2535 | |||||
2536 | if (Value *V = simplifyByDomEq(Instruction::Or, Op0, Op1, Q, MaxRecurse)) | ||||
2537 | return V; | ||||
2538 | |||||
2539 | return nullptr; | ||||
2540 | } | ||||
2541 | |||||
2542 | Value *llvm::simplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { | ||||
2543 | return ::simplifyOrInst(Op0, Op1, Q, RecursionLimit); | ||||
2544 | } | ||||
2545 | |||||
2546 | /// Given operands for a Xor, see if we can fold the result. | ||||
2547 | /// If not, this returns null. | ||||
2548 | static Value *simplifyXorInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, | ||||
2549 | unsigned MaxRecurse) { | ||||
2550 | if (Constant *C = foldOrCommuteConstant(Instruction::Xor, Op0, Op1, Q)) | ||||
2551 | return C; | ||||
2552 | |||||
2553 | // X ^ poison -> poison | ||||
2554 | if (isa<PoisonValue>(Op1)) | ||||
2555 | return Op1; | ||||
2556 | |||||
2557 | // A ^ undef -> undef | ||||
2558 | if (Q.isUndefValue(Op1)) | ||||
2559 | return Op1; | ||||
2560 | |||||
2561 | // A ^ 0 = A | ||||
2562 | if (match(Op1, m_Zero())) | ||||
2563 | return Op0; | ||||
2564 | |||||
2565 | // A ^ A = 0 | ||||
2566 | if (Op0 == Op1) | ||||
2567 | return Constant::getNullValue(Op0->getType()); | ||||
2568 | |||||
2569 | // A ^ ~A = ~A ^ A = -1 | ||||
2570 | if (match(Op0, m_Not(m_Specific(Op1))) || match(Op1, m_Not(m_Specific(Op0)))) | ||||
2571 | return Constant::getAllOnesValue(Op0->getType()); | ||||
2572 | |||||
2573 | auto foldAndOrNot = [](Value *X, Value *Y) -> Value * { | ||||
2574 | Value *A, *B; | ||||
2575 | // (~A & B) ^ (A | B) --> A -- There are 8 commuted variants. | ||||
2576 | if (match(X, m_c_And(m_Not(m_Value(A)), m_Value(B))) && | ||||
2577 | match(Y, m_c_Or(m_Specific(A), m_Specific(B)))) | ||||
2578 | return A; | ||||
2579 | |||||
2580 | // (~A | B) ^ (A & B) --> ~A -- There are 8 commuted variants. | ||||
2581 | // The 'not' op must contain a complete -1 operand (no undef elements for | ||||
2582 | // vector) for the transform to be safe. | ||||
2583 | Value *NotA; | ||||
2584 | if (match(X, | ||||
2585 | m_c_Or(m_CombineAnd(m_NotForbidUndef(m_Value(A)), m_Value(NotA)), | ||||
2586 | m_Value(B))) && | ||||
2587 | match(Y, m_c_And(m_Specific(A), m_Specific(B)))) | ||||
2588 | return NotA; | ||||
2589 | |||||
2590 | return nullptr; | ||||
2591 | }; | ||||
2592 | if (Value *R = foldAndOrNot(Op0, Op1)) | ||||
2593 | return R; | ||||
2594 | if (Value *R = foldAndOrNot(Op1, Op0)) | ||||
2595 | return R; | ||||
2596 | |||||
2597 | if (Value *V = simplifyLogicOfAddSub(Op0, Op1, Instruction::Xor)) | ||||
2598 | return V; | ||||
2599 | |||||
2600 | // Try some generic simplifications for associative operations. | ||||
2601 | if (Value *V = | ||||
2602 | simplifyAssociativeBinOp(Instruction::Xor, Op0, Op1, Q, MaxRecurse)) | ||||
2603 | return V; | ||||
2604 | |||||
2605 | // Threading Xor over selects and phi nodes is pointless, so don't bother. | ||||
2606 | // Threading over the select in "A ^ select(cond, B, C)" means evaluating | ||||
2607 | // "A^B" and "A^C" and seeing if they are equal; but they are equal if and | ||||
2608 | // only if B and C are equal. If B and C are equal then (since we assume | ||||
2609 | // that operands have already been simplified) "select(cond, B, C)" should | ||||
2610 | // have been simplified to the common value of B and C already. Analysing | ||||
2611 | // "A^B" and "A^C" thus gains nothing, but costs compile time. Similarly | ||||
2612 | // for threading over phi nodes. | ||||
2613 | |||||
2614 | if (Value *V = simplifyByDomEq(Instruction::Xor, Op0, Op1, Q, MaxRecurse)) | ||||
2615 | return V; | ||||
2616 | |||||
2617 | return nullptr; | ||||
2618 | } | ||||
2619 | |||||
2620 | Value *llvm::simplifyXorInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { | ||||
2621 | return ::simplifyXorInst(Op0, Op1, Q, RecursionLimit); | ||||
2622 | } | ||||
2623 | |||||
2624 | static Type *getCompareTy(Value *Op) { | ||||
2625 | return CmpInst::makeCmpResultType(Op->getType()); | ||||
2626 | } | ||||
2627 | |||||
2628 | /// Rummage around inside V looking for something equivalent to the comparison | ||||
2629 | /// "LHS Pred RHS". Return such a value if found, otherwise return null. | ||||
2630 | /// Helper function for analyzing max/min idioms. | ||||
2631 | static Value *extractEquivalentCondition(Value *V, CmpInst::Predicate Pred, | ||||
2632 | Value *LHS, Value *RHS) { | ||||
2633 | SelectInst *SI = dyn_cast<SelectInst>(V); | ||||
2634 | if (!SI) | ||||
2635 | return nullptr; | ||||
2636 | CmpInst *Cmp = dyn_cast<CmpInst>(SI->getCondition()); | ||||
2637 | if (!Cmp) | ||||
2638 | return nullptr; | ||||
2639 | Value *CmpLHS = Cmp->getOperand(0), *CmpRHS = Cmp->getOperand(1); | ||||
2640 | if (Pred == Cmp->getPredicate() && LHS == CmpLHS && RHS == CmpRHS) | ||||
2641 | return Cmp; | ||||
2642 | if (Pred == CmpInst::getSwappedPredicate(Cmp->getPredicate()) && | ||||
2643 | LHS == CmpRHS && RHS == CmpLHS) | ||||
2644 | return Cmp; | ||||
2645 | return nullptr; | ||||
2646 | } | ||||
2647 | |||||
2648 | /// Return true if the underlying object (storage) must be disjoint from | ||||
2649 | /// storage returned by any noalias return call. | ||||
2650 | static bool isAllocDisjoint(const Value *V) { | ||||
2651 | // For allocas, we consider only static ones (dynamic | ||||
2652 | // allocas might be transformed into calls to malloc not simultaneously | ||||
2653 | // live with the compared-to allocation). For globals, we exclude symbols | ||||
2654 | // that might be resolve lazily to symbols in another dynamically-loaded | ||||
2655 | // library (and, thus, could be malloc'ed by the implementation). | ||||
2656 | if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) | ||||
2657 | return AI->getParent() && AI->getFunction() && AI->isStaticAlloca(); | ||||
2658 | if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) | ||||
2659 | return (GV->hasLocalLinkage() || GV->hasHiddenVisibility() || | ||||
2660 | GV->hasProtectedVisibility() || GV->hasGlobalUnnamedAddr()) && | ||||
2661 | !GV->isThreadLocal(); | ||||
2662 | if (const Argument *A = dyn_cast<Argument>(V)) | ||||
2663 | return A->hasByValAttr(); | ||||
2664 | return false; | ||||
2665 | } | ||||
2666 | |||||
2667 | /// Return true if V1 and V2 are each the base of some distict storage region | ||||
2668 | /// [V, object_size(V)] which do not overlap. Note that zero sized regions | ||||
2669 | /// *are* possible, and that zero sized regions do not overlap with any other. | ||||
2670 | static bool haveNonOverlappingStorage(const Value *V1, const Value *V2) { | ||||
2671 | // Global variables always exist, so they always exist during the lifetime | ||||
2672 | // of each other and all allocas. Global variables themselves usually have | ||||
2673 | // non-overlapping storage, but since their addresses are constants, the | ||||
2674 | // case involving two globals does not reach here and is instead handled in | ||||
2675 | // constant folding. | ||||
2676 | // | ||||
2677 | // Two different allocas usually have different addresses... | ||||
2678 | // | ||||
2679 | // However, if there's an @llvm.stackrestore dynamically in between two | ||||
2680 | // allocas, they may have the same address. It's tempting to reduce the | ||||
2681 | // scope of the problem by only looking at *static* allocas here. That would | ||||
2682 | // cover the majority of allocas while significantly reducing the likelihood | ||||
2683 | // of having an @llvm.stackrestore pop up in the middle. However, it's not | ||||
2684 | // actually impossible for an @llvm.stackrestore to pop up in the middle of | ||||
2685 | // an entry block. Also, if we have a block that's not attached to a | ||||
2686 | // function, we can't tell if it's "static" under the current definition. | ||||
2687 | // Theoretically, this problem could be fixed by creating a new kind of | ||||
2688 | // instruction kind specifically for static allocas. Such a new instruction | ||||
2689 | // could be required to be at the top of the entry block, thus preventing it | ||||
2690 | // from being subject to a @llvm.stackrestore. Instcombine could even | ||||
2691 | // convert regular allocas into these special allocas. It'd be nifty. | ||||
2692 | // However, until then, this problem remains open. | ||||
2693 | // | ||||
2694 | // So, we'll assume that two non-empty allocas have different addresses | ||||
2695 | // for now. | ||||
2696 | auto isByValArg = [](const Value *V) { | ||||
2697 | const Argument *A = dyn_cast<Argument>(V); | ||||
2698 | return A && A->hasByValAttr(); | ||||
2699 | }; | ||||
2700 | |||||
2701 | // Byval args are backed by store which does not overlap with each other, | ||||
2702 | // allocas, or globals. | ||||
2703 | if (isByValArg(V1)) | ||||
2704 | return isa<AllocaInst>(V2) || isa<GlobalVariable>(V2) || isByValArg(V2); | ||||
2705 | if (isByValArg(V2)) | ||||
2706 | return isa<AllocaInst>(V1) || isa<GlobalVariable>(V1) || isByValArg(V1); | ||||
2707 | |||||
2708 | return isa<AllocaInst>(V1) && | ||||
2709 | (isa<AllocaInst>(V2) || isa<GlobalVariable>(V2)); | ||||
2710 | } | ||||
2711 | |||||
2712 | // A significant optimization not implemented here is assuming that alloca | ||||
2713 | // addresses are not equal to incoming argument values. They don't *alias*, | ||||
2714 | // as we say, but that doesn't mean they aren't equal, so we take a | ||||
2715 | // conservative approach. | ||||
2716 | // | ||||
2717 | // This is inspired in part by C++11 5.10p1: | ||||
2718 | // "Two pointers of the same type compare equal if and only if they are both | ||||
2719 | // null, both point to the same function, or both represent the same | ||||
2720 | // address." | ||||
2721 | // | ||||
2722 | // This is pretty permissive. | ||||
2723 | // | ||||
2724 | // It's also partly due to C11 6.5.9p6: | ||||
2725 | // "Two pointers compare equal if and only if both are null pointers, both are | ||||
2726 | // pointers to the same object (including a pointer to an object and a | ||||
2727 | // subobject at its beginning) or function, both are pointers to one past the | ||||
2728 | // last element of the same array object, or one is a pointer to one past the | ||||
2729 | // end of one array object and the other is a pointer to the start of a | ||||
2730 | // different array object that happens to immediately follow the first array | ||||
2731 | // object in the address space.) | ||||
2732 | // | ||||
2733 | // C11's version is more restrictive, however there's no reason why an argument | ||||
2734 | // couldn't be a one-past-the-end value for a stack object in the caller and be | ||||
2735 | // equal to the beginning of a stack object in the callee. | ||||
2736 | // | ||||
2737 | // If the C and C++ standards are ever made sufficiently restrictive in this | ||||
2738 | // area, it may be possible to update LLVM's semantics accordingly and reinstate | ||||
2739 | // this optimization. | ||||
2740 | static Constant *computePointerICmp(CmpInst::Predicate Pred, Value *LHS, | ||||
2741 | Value *RHS, const SimplifyQuery &Q) { | ||||
2742 | const DataLayout &DL = Q.DL; | ||||
2743 | const TargetLibraryInfo *TLI = Q.TLI; | ||||
2744 | const DominatorTree *DT = Q.DT; | ||||
2745 | const Instruction *CxtI = Q.CxtI; | ||||
2746 | const InstrInfoQuery &IIQ = Q.IIQ; | ||||
2747 | |||||
2748 | // First, skip past any trivial no-ops. | ||||
2749 | LHS = LHS->stripPointerCasts(); | ||||
2750 | RHS = RHS->stripPointerCasts(); | ||||
2751 | |||||
2752 | // A non-null pointer is not equal to a null pointer. | ||||
2753 | if (isa<ConstantPointerNull>(RHS) && ICmpInst::isEquality(Pred) && | ||||
2754 | llvm::isKnownNonZero(LHS, DL, 0, nullptr, nullptr, nullptr, | ||||
2755 | IIQ.UseInstrInfo)) | ||||
2756 | return ConstantInt::get(getCompareTy(LHS), !CmpInst::isTrueWhenEqual(Pred)); | ||||
2757 | |||||
2758 | // We can only fold certain predicates on pointer comparisons. | ||||
2759 | switch (Pred) { | ||||
2760 | default: | ||||
2761 | return nullptr; | ||||
2762 | |||||
2763 | // Equality comparisons are easy to fold. | ||||
2764 | case CmpInst::ICMP_EQ: | ||||
2765 | case CmpInst::ICMP_NE: | ||||
2766 | break; | ||||
2767 | |||||
2768 | // We can only handle unsigned relational comparisons because 'inbounds' on | ||||
2769 | // a GEP only protects against unsigned wrapping. | ||||
2770 | case CmpInst::ICMP_UGT: | ||||
2771 | case CmpInst::ICMP_UGE: | ||||
2772 | case CmpInst::ICMP_ULT: | ||||
2773 | case CmpInst::ICMP_ULE: | ||||
2774 | // However, we have to switch them to their signed variants to handle | ||||
2775 | // negative indices from the base pointer. | ||||
2776 | Pred = ICmpInst::getSignedPredicate(Pred); | ||||
2777 | break; | ||||
2778 | } | ||||
2779 | |||||
2780 | // Strip off any constant offsets so that we can reason about them. | ||||
2781 | // It's tempting to use getUnderlyingObject or even just stripInBoundsOffsets | ||||
2782 | // here and compare base addresses like AliasAnalysis does, however there are | ||||
2783 | // numerous hazards. AliasAnalysis and its utilities rely on special rules | ||||
2784 | // governing loads and stores which don't apply to icmps. Also, AliasAnalysis | ||||
2785 | // doesn't need to guarantee pointer inequality when it says NoAlias. | ||||
2786 | |||||
2787 | // Even if an non-inbounds GEP occurs along the path we can still optimize | ||||
2788 | // equality comparisons concerning the result. | ||||
2789 | bool AllowNonInbounds = ICmpInst::isEquality(Pred); | ||||
2790 | APInt LHSOffset = stripAndComputeConstantOffsets(DL, LHS, AllowNonInbounds); | ||||
2791 | APInt RHSOffset = stripAndComputeConstantOffsets(DL, RHS, AllowNonInbounds); | ||||
2792 | |||||
2793 | // If LHS and RHS are related via constant offsets to the same base | ||||
2794 | // value, we can replace it with an icmp which just compares the offsets. | ||||
2795 | if (LHS == RHS) | ||||
2796 | return ConstantInt::get(getCompareTy(LHS), | ||||
2797 | ICmpInst::compare(LHSOffset, RHSOffset, Pred)); | ||||
2798 | |||||
2799 | // Various optimizations for (in)equality comparisons. | ||||
2800 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) { | ||||
2801 | // Different non-empty allocations that exist at the same time have | ||||
2802 | // different addresses (if the program can tell). If the offsets are | ||||
2803 | // within the bounds of their allocations (and not one-past-the-end! | ||||
2804 | // so we can't use inbounds!), and their allocations aren't the same, | ||||
2805 | // the pointers are not equal. | ||||
2806 | if (haveNonOverlappingStorage(LHS, RHS)) { | ||||
2807 | uint64_t LHSSize, RHSSize; | ||||
2808 | ObjectSizeOpts Opts; | ||||
2809 | Opts.EvalMode = ObjectSizeOpts::Mode::Min; | ||||
2810 | auto *F = [](Value *V) -> Function * { | ||||
2811 | if (auto *I = dyn_cast<Instruction>(V)) | ||||
2812 | return I->getFunction(); | ||||
2813 | if (auto *A = dyn_cast<Argument>(V)) | ||||
2814 | return A->getParent(); | ||||
2815 | return nullptr; | ||||
2816 | }(LHS); | ||||
2817 | Opts.NullIsUnknownSize = F ? NullPointerIsDefined(F) : true; | ||||
2818 | if (getObjectSize(LHS, LHSSize, DL, TLI, Opts) && | ||||
2819 | getObjectSize(RHS, RHSSize, DL, TLI, Opts) && | ||||
2820 | !LHSOffset.isNegative() && !RHSOffset.isNegative() && | ||||
2821 | LHSOffset.ult(LHSSize) && RHSOffset.ult(RHSSize)) { | ||||
2822 | return ConstantInt::get(getCompareTy(LHS), | ||||
2823 | !CmpInst::isTrueWhenEqual(Pred)); | ||||
2824 | } | ||||
2825 | } | ||||
2826 | |||||
2827 | // If one side of the equality comparison must come from a noalias call | ||||
2828 | // (meaning a system memory allocation function), and the other side must | ||||
2829 | // come from a pointer that cannot overlap with dynamically-allocated | ||||
2830 | // memory within the lifetime of the current function (allocas, byval | ||||
2831 | // arguments, globals), then determine the comparison result here. | ||||
2832 | SmallVector<const Value *, 8> LHSUObjs, RHSUObjs; | ||||
2833 | getUnderlyingObjects(LHS, LHSUObjs); | ||||
2834 | getUnderlyingObjects(RHS, RHSUObjs); | ||||
2835 | |||||
2836 | // Is the set of underlying objects all noalias calls? | ||||
2837 | auto IsNAC = [](ArrayRef<const Value *> Objects) { | ||||
2838 | return all_of(Objects, isNoAliasCall); | ||||
2839 | }; | ||||
2840 | |||||
2841 | // Is the set of underlying objects all things which must be disjoint from | ||||
2842 | // noalias calls. We assume that indexing from such disjoint storage | ||||
2843 | // into the heap is undefined, and thus offsets can be safely ignored. | ||||
2844 | auto IsAllocDisjoint = [](ArrayRef<const Value *> Objects) { | ||||
2845 | return all_of(Objects, ::isAllocDisjoint); | ||||
2846 | }; | ||||
2847 | |||||
2848 | if ((IsNAC(LHSUObjs) && IsAllocDisjoint(RHSUObjs)) || | ||||
2849 | (IsNAC(RHSUObjs) && IsAllocDisjoint(LHSUObjs))) | ||||
2850 | return ConstantInt::get(getCompareTy(LHS), | ||||
2851 | !CmpInst::isTrueWhenEqual(Pred)); | ||||
2852 | |||||
2853 | // Fold comparisons for non-escaping pointer even if the allocation call | ||||
2854 | // cannot be elided. We cannot fold malloc comparison to null. Also, the | ||||
2855 | // dynamic allocation call could be either of the operands. Note that | ||||
2856 | // the other operand can not be based on the alloc - if it were, then | ||||
2857 | // the cmp itself would be a capture. | ||||
2858 | Value *MI = nullptr; | ||||
2859 | if (isAllocLikeFn(LHS, TLI) && | ||||
2860 | llvm::isKnownNonZero(RHS, DL, 0, nullptr, CxtI, DT)) | ||||
2861 | MI = LHS; | ||||
2862 | else if (isAllocLikeFn(RHS, TLI) && | ||||
2863 | llvm::isKnownNonZero(LHS, DL, 0, nullptr, CxtI, DT)) | ||||
2864 | MI = RHS; | ||||
2865 | if (MI) { | ||||
2866 | // FIXME: This is incorrect, see PR54002. While we can assume that the | ||||
2867 | // allocation is at an address that makes the comparison false, this | ||||
2868 | // requires that *all* comparisons to that address be false, which | ||||
2869 | // InstSimplify cannot guarantee. | ||||
2870 | struct CustomCaptureTracker : public CaptureTracker { | ||||
2871 | bool Captured = false; | ||||
2872 | void tooManyUses() override { Captured = true; } | ||||
2873 | bool captured(const Use *U) override { | ||||
2874 | if (auto *ICmp = dyn_cast<ICmpInst>(U->getUser())) { | ||||
2875 | // Comparison against value stored in global variable. Given the | ||||
2876 | // pointer does not escape, its value cannot be guessed and stored | ||||
2877 | // separately in a global variable. | ||||
2878 | unsigned OtherIdx = 1 - U->getOperandNo(); | ||||
2879 | auto *LI = dyn_cast<LoadInst>(ICmp->getOperand(OtherIdx)); | ||||
2880 | if (LI && isa<GlobalVariable>(LI->getPointerOperand())) | ||||
2881 | return false; | ||||
2882 | } | ||||
2883 | |||||
2884 | Captured = true; | ||||
2885 | return true; | ||||
2886 | } | ||||
2887 | }; | ||||
2888 | CustomCaptureTracker Tracker; | ||||
2889 | PointerMayBeCaptured(MI, &Tracker); | ||||
2890 | if (!Tracker.Captured) | ||||
2891 | return ConstantInt::get(getCompareTy(LHS), | ||||
2892 | CmpInst::isFalseWhenEqual(Pred)); | ||||
2893 | } | ||||
2894 | } | ||||
2895 | |||||
2896 | // Otherwise, fail. | ||||
2897 | return nullptr; | ||||
2898 | } | ||||
2899 | |||||
2900 | /// Fold an icmp when its operands have i1 scalar type. | ||||
2901 | static Value *simplifyICmpOfBools(CmpInst::Predicate Pred, Value *LHS, | ||||
2902 | Value *RHS, const SimplifyQuery &Q) { | ||||
2903 | Type *ITy = getCompareTy(LHS); // The return type. | ||||
2904 | Type *OpTy = LHS->getType(); // The operand type. | ||||
2905 | if (!OpTy->isIntOrIntVectorTy(1)) | ||||
2906 | return nullptr; | ||||
2907 | |||||
2908 | // A boolean compared to true/false can be reduced in 14 out of the 20 | ||||
2909 | // (10 predicates * 2 constants) possible combinations. The other | ||||
2910 | // 6 cases require a 'not' of the LHS. | ||||
2911 | |||||
2912 | auto ExtractNotLHS = [](Value *V) -> Value * { | ||||
2913 | Value *X; | ||||
2914 | if (match(V, m_Not(m_Value(X)))) | ||||
2915 | return X; | ||||
2916 | return nullptr; | ||||
2917 | }; | ||||
2918 | |||||
2919 | if (match(RHS, m_Zero())) { | ||||
2920 | switch (Pred) { | ||||
2921 | case CmpInst::ICMP_NE: // X != 0 -> X | ||||
2922 | case CmpInst::ICMP_UGT: // X >u 0 -> X | ||||
2923 | case CmpInst::ICMP_SLT: // X <s 0 -> X | ||||
2924 | return LHS; | ||||
2925 | |||||
2926 | case CmpInst::ICMP_EQ: // not(X) == 0 -> X != 0 -> X | ||||
2927 | case CmpInst::ICMP_ULE: // not(X) <=u 0 -> X >u 0 -> X | ||||
2928 | case CmpInst::ICMP_SGE: // not(X) >=s 0 -> X <s 0 -> X | ||||
2929 | if (Value *X = ExtractNotLHS(LHS)) | ||||
2930 | return X; | ||||
2931 | break; | ||||
2932 | |||||
2933 | case CmpInst::ICMP_ULT: // X <u 0 -> false | ||||
2934 | case CmpInst::ICMP_SGT: // X >s 0 -> false | ||||
2935 | return getFalse(ITy); | ||||
2936 | |||||
2937 | case CmpInst::ICMP_UGE: // X >=u 0 -> true | ||||
2938 | case CmpInst::ICMP_SLE: // X <=s 0 -> true | ||||
2939 | return getTrue(ITy); | ||||
2940 | |||||
2941 | default: | ||||
2942 | break; | ||||
2943 | } | ||||
2944 | } else if (match(RHS, m_One())) { | ||||
2945 | switch (Pred) { | ||||
2946 | case CmpInst::ICMP_EQ: // X == 1 -> X | ||||
2947 | case CmpInst::ICMP_UGE: // X >=u 1 -> X | ||||
2948 | case CmpInst::ICMP_SLE: // X <=s -1 -> X | ||||
2949 | return LHS; | ||||
2950 | |||||
2951 | case CmpInst::ICMP_NE: // not(X) != 1 -> X == 1 -> X | ||||
2952 | case CmpInst::ICMP_ULT: // not(X) <=u 1 -> X >=u 1 -> X | ||||
2953 | case CmpInst::ICMP_SGT: // not(X) >s 1 -> X <=s -1 -> X | ||||
2954 | if (Value *X = ExtractNotLHS(LHS)) | ||||
2955 | return X; | ||||
2956 | break; | ||||
2957 | |||||
2958 | case CmpInst::ICMP_UGT: // X >u 1 -> false | ||||
2959 | case CmpInst::ICMP_SLT: // X <s -1 -> false | ||||
2960 | return getFalse(ITy); | ||||
2961 | |||||
2962 | case CmpInst::ICMP_ULE: // X <=u 1 -> true | ||||
2963 | case CmpInst::ICMP_SGE: // X >=s -1 -> true | ||||
2964 | return getTrue(ITy); | ||||
2965 | |||||
2966 | default: | ||||
2967 | break; | ||||
2968 | } | ||||
2969 | } | ||||
2970 | |||||
2971 | switch (Pred) { | ||||
2972 | default: | ||||
2973 | break; | ||||
2974 | case ICmpInst::ICMP_UGE: | ||||
2975 | if (isImpliedCondition(RHS, LHS, Q.DL).value_or(false)) | ||||
2976 | return getTrue(ITy); | ||||
2977 | break; | ||||
2978 | case ICmpInst::ICMP_SGE: | ||||
2979 | /// For signed comparison, the values for an i1 are 0 and -1 | ||||
2980 | /// respectively. This maps into a truth table of: | ||||
2981 | /// LHS | RHS | LHS >=s RHS | LHS implies RHS | ||||
2982 | /// 0 | 0 | 1 (0 >= 0) | 1 | ||||
2983 | /// 0 | 1 | 1 (0 >= -1) | 1 | ||||
2984 | /// 1 | 0 | 0 (-1 >= 0) | 0 | ||||
2985 | /// 1 | 1 | 1 (-1 >= -1) | 1 | ||||
2986 | if (isImpliedCondition(LHS, RHS, Q.DL).value_or(false)) | ||||
2987 | return getTrue(ITy); | ||||
2988 | break; | ||||
2989 | case ICmpInst::ICMP_ULE: | ||||
2990 | if (isImpliedCondition(LHS, RHS, Q.DL).value_or(false)) | ||||
2991 | return getTrue(ITy); | ||||
2992 | break; | ||||
2993 | case ICmpInst::ICMP_SLE: | ||||
2994 | /// SLE follows the same logic as SGE with the LHS and RHS swapped. | ||||
2995 | if (isImpliedCondition(RHS, LHS, Q.DL).value_or(false)) | ||||
2996 | return getTrue(ITy); | ||||
2997 | break; | ||||
2998 | } | ||||
2999 | |||||
3000 | return nullptr; | ||||
3001 | } | ||||
3002 | |||||
3003 | /// Try hard to fold icmp with zero RHS because this is a common case. | ||||
3004 | static Value *simplifyICmpWithZero(CmpInst::Predicate Pred, Value *LHS, | ||||
3005 | Value *RHS, const SimplifyQuery &Q) { | ||||
3006 | if (!match(RHS, m_Zero())) | ||||
3007 | return nullptr; | ||||
3008 | |||||
3009 | Type *ITy = getCompareTy(LHS); // The return type. | ||||
3010 | switch (Pred) { | ||||
3011 | default: | ||||
3012 | llvm_unreachable("Unknown ICmp predicate!")::llvm::llvm_unreachable_internal("Unknown ICmp predicate!", "llvm/lib/Analysis/InstructionSimplify.cpp" , 3012); | ||||
3013 | case ICmpInst::ICMP_ULT: | ||||
3014 | return getFalse(ITy); | ||||
3015 | case ICmpInst::ICMP_UGE: | ||||
3016 | return getTrue(ITy); | ||||
3017 | case ICmpInst::ICMP_EQ: | ||||
3018 | case ICmpInst::ICMP_ULE: | ||||
3019 | if (isKnownNonZero(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT, Q.IIQ.UseInstrInfo)) | ||||
3020 | return getFalse(ITy); | ||||
3021 | break; | ||||
3022 | case ICmpInst::ICMP_NE: | ||||
3023 | case ICmpInst::ICMP_UGT: | ||||
3024 | if (isKnownNonZero(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT, Q.IIQ.UseInstrInfo)) | ||||
3025 | return getTrue(ITy); | ||||
3026 | break; | ||||
3027 | case ICmpInst::ICMP_SLT: { | ||||
3028 | KnownBits LHSKnown = computeKnownBits(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
3029 | if (LHSKnown.isNegative()) | ||||
3030 | return getTrue(ITy); | ||||
3031 | if (LHSKnown.isNonNegative()) | ||||
3032 | return getFalse(ITy); | ||||
3033 | break; | ||||
3034 | } | ||||
3035 | case ICmpInst::ICMP_SLE: { | ||||
3036 | KnownBits LHSKnown = computeKnownBits(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
3037 | if (LHSKnown.isNegative()) | ||||
3038 | return getTrue(ITy); | ||||
3039 | if (LHSKnown.isNonNegative() && | ||||
3040 | isKnownNonZero(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT)) | ||||
3041 | return getFalse(ITy); | ||||
3042 | break; | ||||
3043 | } | ||||
3044 | case ICmpInst::ICMP_SGE: { | ||||
3045 | KnownBits LHSKnown = computeKnownBits(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
3046 | if (LHSKnown.isNegative()) | ||||
3047 | return getFalse(ITy); | ||||
3048 | if (LHSKnown.isNonNegative()) | ||||
3049 | return getTrue(ITy); | ||||
3050 | break; | ||||
3051 | } | ||||
3052 | case ICmpInst::ICMP_SGT: { | ||||
3053 | KnownBits LHSKnown = computeKnownBits(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
3054 | if (LHSKnown.isNegative()) | ||||
3055 | return getFalse(ITy); | ||||
3056 | if (LHSKnown.isNonNegative() && | ||||
3057 | isKnownNonZero(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT)) | ||||
3058 | return getTrue(ITy); | ||||
3059 | break; | ||||
3060 | } | ||||
3061 | } | ||||
3062 | |||||
3063 | return nullptr; | ||||
3064 | } | ||||
3065 | |||||
3066 | static Value *simplifyICmpWithConstant(CmpInst::Predicate Pred, Value *LHS, | ||||
3067 | Value *RHS, const InstrInfoQuery &IIQ) { | ||||
3068 | Type *ITy = getCompareTy(RHS); // The return type. | ||||
3069 | |||||
3070 | Value *X; | ||||
3071 | // Sign-bit checks can be optimized to true/false after unsigned | ||||
3072 | // floating-point casts: | ||||
3073 | // icmp slt (bitcast (uitofp X)), 0 --> false | ||||
3074 | // icmp sgt (bitcast (uitofp X)), -1 --> true | ||||
3075 | if (match(LHS, m_BitCast(m_UIToFP(m_Value(X))))) { | ||||
3076 | if (Pred == ICmpInst::ICMP_SLT && match(RHS, m_Zero())) | ||||
3077 | return ConstantInt::getFalse(ITy); | ||||
3078 | if (Pred == ICmpInst::ICMP_SGT && match(RHS, m_AllOnes())) | ||||
3079 | return ConstantInt::getTrue(ITy); | ||||
3080 | } | ||||
3081 | |||||
3082 | const APInt *C; | ||||
3083 | if (!match(RHS, m_APIntAllowUndef(C))) | ||||
3084 | return nullptr; | ||||
3085 | |||||
3086 | // Rule out tautological comparisons (eg., ult 0 or uge 0). | ||||
3087 | ConstantRange RHS_CR = ConstantRange::makeExactICmpRegion(Pred, *C); | ||||
3088 | if (RHS_CR.isEmptySet()) | ||||
3089 | return ConstantInt::getFalse(ITy); | ||||
3090 | if (RHS_CR.isFullSet()) | ||||
3091 | return ConstantInt::getTrue(ITy); | ||||
3092 | |||||
3093 | ConstantRange LHS_CR = | ||||
3094 | computeConstantRange(LHS, CmpInst::isSigned(Pred), IIQ.UseInstrInfo); | ||||
3095 | if (!LHS_CR.isFullSet()) { | ||||
3096 | if (RHS_CR.contains(LHS_CR)) | ||||
3097 | return ConstantInt::getTrue(ITy); | ||||
3098 | if (RHS_CR.inverse().contains(LHS_CR)) | ||||
3099 | return ConstantInt::getFalse(ITy); | ||||
3100 | } | ||||
3101 | |||||
3102 | // (mul nuw/nsw X, MulC) != C --> true (if C is not a multiple of MulC) | ||||
3103 | // (mul nuw/nsw X, MulC) == C --> false (if C is not a multiple of MulC) | ||||
3104 | const APInt *MulC; | ||||
3105 | if (ICmpInst::isEquality(Pred) && | ||||
3106 | ((match(LHS, m_NUWMul(m_Value(), m_APIntAllowUndef(MulC))) && | ||||
3107 | *MulC != 0 && C->urem(*MulC) != 0) || | ||||
3108 | (match(LHS, m_NSWMul(m_Value(), m_APIntAllowUndef(MulC))) && | ||||
3109 | *MulC != 0 && C->srem(*MulC) != 0))) | ||||
3110 | return ConstantInt::get(ITy, Pred == ICmpInst::ICMP_NE); | ||||
3111 | |||||
3112 | return nullptr; | ||||
3113 | } | ||||
3114 | |||||
3115 | static Value *simplifyICmpWithBinOpOnLHS(CmpInst::Predicate Pred, | ||||
3116 | BinaryOperator *LBO, Value *RHS, | ||||
3117 | const SimplifyQuery &Q, | ||||
3118 | unsigned MaxRecurse) { | ||||
3119 | Type *ITy = getCompareTy(RHS); // The return type. | ||||
3120 | |||||
3121 | Value *Y = nullptr; | ||||
3122 | // icmp pred (or X, Y), X | ||||
3123 | if (match(LBO, m_c_Or(m_Value(Y), m_Specific(RHS)))) { | ||||
3124 | if (Pred == ICmpInst::ICMP_ULT) | ||||
3125 | return getFalse(ITy); | ||||
3126 | if (Pred == ICmpInst::ICMP_UGE) | ||||
3127 | return getTrue(ITy); | ||||
3128 | |||||
3129 | if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) { | ||||
3130 | KnownBits RHSKnown = computeKnownBits(RHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
3131 | KnownBits YKnown = computeKnownBits(Y, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
3132 | if (RHSKnown.isNonNegative() && YKnown.isNegative()) | ||||
3133 | return Pred == ICmpInst::ICMP_SLT ? getTrue(ITy) : getFalse(ITy); | ||||
3134 | if (RHSKnown.isNegative() || YKnown.isNonNegative()) | ||||
3135 | return Pred == ICmpInst::ICMP_SLT ? getFalse(ITy) : getTrue(ITy); | ||||
3136 | } | ||||
3137 | } | ||||
3138 | |||||
3139 | // icmp pred (and X, Y), X | ||||
3140 | if (match(LBO, m_c_And(m_Value(), m_Specific(RHS)))) { | ||||
3141 | if (Pred == ICmpInst::ICMP_UGT) | ||||
3142 | return getFalse(ITy); | ||||
3143 | if (Pred == ICmpInst::ICMP_ULE) | ||||
3144 | return getTrue(ITy); | ||||
3145 | } | ||||
3146 | |||||
3147 | // icmp pred (urem X, Y), Y | ||||
3148 | if (match(LBO, m_URem(m_Value(), m_Specific(RHS)))) { | ||||
3149 | switch (Pred) { | ||||
3150 | default: | ||||
3151 | break; | ||||
3152 | case ICmpInst::ICMP_SGT: | ||||
3153 | case ICmpInst::ICMP_SGE: { | ||||
3154 | KnownBits Known = computeKnownBits(RHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
3155 | if (!Known.isNonNegative()) | ||||
3156 | break; | ||||
3157 | [[fallthrough]]; | ||||
3158 | } | ||||
3159 | case ICmpInst::ICMP_EQ: | ||||
3160 | case ICmpInst::ICMP_UGT: | ||||
3161 | case ICmpInst::ICMP_UGE: | ||||
3162 | return getFalse(ITy); | ||||
3163 | case ICmpInst::ICMP_SLT: | ||||
3164 | case ICmpInst::ICMP_SLE: { | ||||
3165 | KnownBits Known = computeKnownBits(RHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); | ||||
3166 | if (!Known.isNonNegative()) | ||||
3167 | break; | ||||
3168 | [[fallthrough]]; | ||||
3169 | } | ||||
3170 | case ICmpInst::ICMP_NE: | ||||
3171 | case ICmpInst::ICMP_ULT: | ||||
3172 | case ICmpInst::ICMP_ULE: | ||||
3173 | return getTrue(ITy); | ||||
3174 | } | ||||
3175 | } | ||||
3176 | |||||
3177 | // icmp pred (urem X, Y), X | ||||
3178 | if (match(LBO, m_URem(m_Specific(RHS), m_Value()))) { | ||||
3179 | if (Pred == ICmpInst::ICMP_ULE) | ||||
3180 | return getTrue(ITy); | ||||
3181 | if (Pred == ICmpInst::ICMP_UGT) | ||||
3182 | return getFalse(ITy); | ||||
3183 | } | ||||
3184 | |||||
3185 | // x >>u y <=u x --> true. | ||||
3186 | // x >>u y >u x --> false. | ||||
3187 | // x udiv y <=u x --> true. | ||||
3188 | // x udiv y >u x --> false. | ||||
3189 | if (match(LBO, m_LShr(m_Specific(RHS), m_Value())) || | ||||
3190 | match(LBO, m_UDiv(m_Specific(RHS), m_Value()))) { | ||||
3191 | // icmp pred (X op Y), X | ||||
3192 | if (Pred == ICmpInst::ICMP_UGT) | ||||
3193 | return getFalse(ITy); | ||||
3194 | if (Pred == ICmpInst::ICMP_ULE) | ||||
3195 | return getTrue(ITy); | ||||
3196 | } | ||||
3197 | |||||
3198 | // If x is nonzero: | ||||
3199 | // x >>u C <u x --> true for C != 0. | ||||
3200 | // x >>u C != x --> true for C != 0. | ||||
3201 | // x >>u C >=u x --> false for C != 0. | ||||
3202 | // x >>u C == x --> false for C != 0. | ||||
3203 | // x udiv C <u x --> true for C != 1. | ||||
3204 | // x udiv C != x --> true for C != 1. | ||||
3205 | // x udiv C >=u x --> false for C != 1. | ||||
3206 | // x udiv C == x --> false for C != 1. | ||||
3207 | // TODO: allow non-constant shift amount/divisor | ||||
3208 | const APInt *C; | ||||
3209 | if ((match(LBO, m_LShr(m_Specific(RHS), m_APInt(C))) && *C != 0) || | ||||
3210 | (match(LBO, m_UDiv(m_Specific(RHS), m_APInt(C))) && *C != 1)) { | ||||
3211 | if (isKnownNonZero(RHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT)) { | ||||
3212 | switch (Pred) { | ||||
3213 | default: | ||||
3214 | break; | ||||
3215 | case ICmpInst::ICMP_EQ: | ||||
3216 | case ICmpInst::ICMP_UGE: | ||||
3217 | return getFalse(ITy); | ||||
3218 | case ICmpInst::ICMP_NE: | ||||
3219 | case ICmpInst::ICMP_ULT: | ||||
3220 | return getTrue(ITy); | ||||
3221 | case ICmpInst::ICMP_UGT: | ||||
3222 | case ICmpInst::ICMP_ULE: | ||||
3223 | // UGT/ULE are handled by the more general case just above | ||||
3224 | llvm_unreachable("Unexpected UGT/ULE, should have been handled")::llvm::llvm_unreachable_internal("Unexpected UGT/ULE, should have been handled" , "llvm/lib/Analysis/InstructionSimplify.cpp", 3224); | ||||
3225 | } | ||||
3226 | } | ||||
3227 | } | ||||
3228 | |||||
3229 | // (x*C1)/C2 <= x for C1 <= C2. | ||||
3230 | // This holds even if the multiplication overflows: Assume that x != 0 and | ||||
3231 | // arithmetic is modulo M. For overflow to occur we must have C1 >= M/x and | ||||
3232 | // thus C2 >= M/x. It follows that (x*C1)/C2 <= (M-1)/C2 <= ((M-1)*x)/M < x. | ||||
3233 | // | ||||
3234 | // Additionally, either the multiplication and division might be represented | ||||
3235 | // as shifts: | ||||
3236 | // (x*C1)>>C2 <= x for C1 < 2**C2. | ||||
3237 | // (x<<C1)/C2 <= x for 2**C1 < C2. | ||||
3238 | const APInt *C1, *C2; | ||||
3239 | if ((match(LBO, m_UDiv(m_Mul(m_Specific(RHS), m_APInt(C1)), m_APInt(C2))) && | ||||
3240 | C1->ule(*C2)) || | ||||
3241 | (match(LBO, m_LShr(m_Mul(m_Specific(RHS), m_APInt(C1)), m_APInt(C2))) && | ||||
3242 | C1->ule(APInt(C2->getBitWidth(), 1) << *C2)) || | ||||
3243 | (match(LBO, m_UDiv(m_Shl(m_Specific(RHS), m_APInt(C1)), m_APInt(C2))) && | ||||
3244 | (APInt(C1->getBitWidth(), 1) << *C1).ule(*C2))) { | ||||
3245 | if (Pred == ICmpInst::ICMP_UGT) | ||||
3246 | return getFalse(ITy); | ||||
3247 | if (Pred == ICmpInst::ICMP_ULE) | ||||
3248 | return getTrue(ITy); | ||||
3249 | } | ||||
3250 | |||||
3251 | // (sub C, X) == X, C is odd --> false | ||||
3252 | // (sub C, X) != X, C is odd --> true | ||||
3253 | if (match(LBO, m_Sub(m_APIntAllowUndef(C), m_Specific(RHS))) && | ||||
3254 | (*C & 1) == 1 && ICmpInst::isEquality(Pred)) | ||||
3255 | return (Pred == ICmpInst::ICMP_EQ) ? getFalse(ITy) : getTrue(ITy); | ||||
3256 | |||||
3257 | return nullptr; | ||||
3258 | } | ||||
3259 | |||||
3260 | // If only one of the icmp's operands has NSW flags, try to prove that: | ||||
3261 | // | ||||
3262 | // icmp slt (x + C1), (x +nsw C2) | ||||
3263 | // | ||||
3264 | // is equivalent to: | ||||
3265 | // | ||||
3266 | // icmp slt C1, C2 | ||||
3267 | // | ||||
3268 | // which is true if x + C2 has the NSW flags set and: | ||||
3269 | // *) C1 < C2 && C1 >= 0, or | ||||
3270 | // *) C2 < C1 && C1 <= 0. | ||||
3271 | // | ||||
3272 | static bool trySimplifyICmpWithAdds(CmpInst::Predicate Pred, Value *LHS, | ||||
3273 | Value *RHS) { | ||||
3274 | // TODO: only support icmp slt for now. | ||||
3275 | if (Pred != CmpInst::ICMP_SLT) | ||||
3276 | return false; | ||||
3277 | |||||
3278 | // Canonicalize nsw add as RHS. | ||||
3279 | if (!match(RHS, m_NSWAdd(m_Value(), m_Value()))) | ||||
3280 | std::swap(LHS, RHS); | ||||
3281 | if (!match(RHS, m_NSWAdd(m_Value(), m_Value()))) | ||||
3282 | return false; | ||||
3283 | |||||
3284 | Value *X; | ||||
3285 | const APInt *C1, *C2; | ||||
3286 | if (!match(LHS, m_c_Add(m_Value(X), m_APInt(C1))) || | ||||
3287 | !match(RHS, m_c_Add(m_Specific(X), m_APInt(C2)))) | ||||
3288 | return false; | ||||
3289 | |||||
3290 | return (C1->slt(*C2) && C1->isNonNegative()) || | ||||
3291 | (C2->slt(*C1) && C1->isNonPositive()); | ||||
3292 | } | ||||
3293 | |||||
3294 | /// TODO: A large part of this logic is duplicated in InstCombine's | ||||
3295 | /// foldICmpBinOp(). We should be able to share that and avoid the code | ||||
3296 | /// duplication. | ||||
3297 | static Value *simplifyICmpWithBinOp(CmpInst::Predicate Pred, Value *LHS, | ||||
3298 | Value *RHS, const SimplifyQuery &Q, | ||||
3299 | unsigned MaxRecurse) { | ||||
3300 | BinaryOperator *LBO = dyn_cast<BinaryOperator>(LHS); | ||||
3301 | BinaryOperator *RBO = dyn_cast<BinaryOperator>(RHS); | ||||
3302 | if (MaxRecurse && (LBO || RBO)) { | ||||
3303 | // Analyze the case when either LHS or RHS is an add instruction. | ||||
3304 | Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr; | ||||
3305 | // LHS = A + B (or A and B are null); RHS = C + D (or C and D are null). | ||||
3306 | bool NoLHSWrapProblem = false, NoRHSWrapProblem = false; | ||||
3307 | if (LBO && LBO->getOpcode() == Instruction::Add) { | ||||
3308 | A = LBO->getOperand(0); | ||||
3309 | B = LBO->getOperand(1); | ||||
3310 | NoLHSWrapProblem = | ||||
3311 | ICmpInst::isEquality(Pred) || | ||||
3312 | (CmpInst::isUnsigned(Pred) && | ||||
3313 | Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(LBO))) || | ||||
3314 | (CmpInst::isSigned(Pred) && | ||||
3315 | Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(LBO))); | ||||
3316 | } | ||||
3317 | if (RBO && RBO->getOpcode() == Instruction::Add) { | ||||
3318 | C = RBO->getOperand(0); | ||||
3319 | D = RBO->getOperand(1); | ||||
3320 | NoRHSWrapProblem = | ||||
3321 | ICmpInst::isEquality(Pred) || | ||||
3322 | (CmpInst::isUnsigned(Pred) && | ||||
3323 | Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(RBO))) || | ||||
3324 | (CmpInst::isSigned(Pred) && | ||||
3325 | Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(RBO))); | ||||
3326 | } | ||||
3327 | |||||
3328 | // icmp (X+Y), X -> icmp Y, 0 for equalities or if there is no overflow. | ||||
3329 | if ((A == RHS || B == RHS) && NoLHSWrapProblem) | ||||
3330 | if (Value *V = simplifyICmpInst(Pred, A == RHS ? B : A, | ||||
3331 | Constant::getNullValue(RHS->getType()), Q, | ||||
3332 | MaxRecurse - 1)) | ||||
3333 | return V; | ||||
3334 | |||||
3335 | // icmp X, (X+Y) -> icmp 0, Y for equalities or if there is no overflow. | ||||
3336 | if ((C == LHS || D == LHS) && NoRHSWrapProblem) | ||||
3337 | if (Value *V = | ||||
3338 | simplifyICmpInst(Pred, Constant::getNullValue(LHS->getType()), | ||||
3339 | C == LHS ? D : C, Q, MaxRecurse - 1)) | ||||
3340 | return V; | ||||
3341 | |||||
3342 | // icmp (X+Y), (X+Z) -> icmp Y,Z for equalities or if there is no overflow. | ||||
3343 | bool CanSimplify = (NoLHSWrapProblem && NoRHSWrapProblem) || | ||||
3344 | trySimplifyICmpWithAdds(Pred, LHS, RHS); | ||||
3345 | if (A && C && (A == C || A == D || B == C || B == D) && CanSimplify) { | ||||
3346 | // Determine Y and Z in the form icmp (X+Y), (X+Z). | ||||
3347 | Value *Y, *Z; | ||||
3348 | if (A == C) { | ||||
3349 | // C + B == C + D -> B == D | ||||
3350 | Y = B; | ||||
3351 | Z = D; | ||||
3352 | } else if (A == D) { | ||||
3353 | // D + B == C + D -> B == C | ||||
3354 | Y = B; | ||||
3355 | Z = C; | ||||
3356 | } else if (B == C) { | ||||
3357 | // A + C == C + D -> A == D | ||||
3358 | Y = A; | ||||
3359 | Z = D; | ||||
3360 | } else { | ||||
3361 | assert(B == D)(static_cast <bool> (B == D) ? void (0) : __assert_fail ("B == D", "llvm/lib/Analysis/InstructionSimplify.cpp", 3361 , __extension__ __PRETTY_FUNCTION__)); | ||||
3362 | // A + D == C + D -> A == C | ||||
3363 | Y = A; | ||||
3364 | Z = C; | ||||
3365 | } | ||||
3366 | if (Value *V = simplifyICmpInst(Pred, Y, Z, Q, MaxRecurse - 1)) | ||||
3367 | return V; | ||||
3368 | } | ||||
3369 | } | ||||
3370 | |||||
3371 | if (LBO) | ||||
3372 | if (Value *V = simplifyICmpWithBinOpOnLHS(Pred, LBO, RHS, Q, MaxRecurse)) | ||||
3373 | return V; | ||||
3374 | |||||
3375 | if (RBO) | ||||
3376 | if (Value *V = simplifyICmpWithBinOpOnLHS( | ||||
3377 | ICmpInst::getSwappedPredicate(Pred), RBO, LHS, Q, MaxRecurse)) | ||||
3378 | return V; | ||||
3379 | |||||
3380 | // 0 - (zext X) pred C | ||||
3381 | if (!CmpInst::isUnsigned(Pred) && match(LHS, m_Neg(m_ZExt(m_Value())))) { | ||||
3382 | const APInt *C; | ||||
3383 | if (match(RHS, m_APInt(C))) { | ||||
3384 | if (C->isStrictlyPositive()) { | ||||
3385 | if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_NE) | ||||
3386 | return ConstantInt::getTrue(getCompareTy(RHS)); | ||||
3387 | if (Pred == ICmpInst::ICMP_SGE || Pred == ICmpInst::ICMP_EQ) | ||||
3388 | return ConstantInt::getFalse(getCompareTy(RHS)); | ||||
3389 | } | ||||
3390 | if (C->isNonNegative()) { | ||||
3391 | if (Pred == ICmpInst::ICMP_SLE) | ||||
3392 | return ConstantInt::getTrue(getCompareTy(RHS)); | ||||
3393 | if (Pred == ICmpInst::ICMP_SGT) | ||||
3394 | return ConstantInt::getFalse(getCompareTy(RHS)); | ||||
3395 | } | ||||
3396 | } | ||||
3397 | } | ||||
3398 | |||||
3399 | // If C2 is a power-of-2 and C is not: | ||||
3400 | // (C2 << X) == C --> false | ||||
3401 | // (C2 << X) != C --> true | ||||
3402 | const APInt *C; | ||||
3403 | if (match(LHS, m_Shl(m_Power2(), m_Value())) && | ||||
3404 | match(RHS, m_APIntAllowUndef(C)) && !C->isPowerOf2()) { | ||||
3405 | // C2 << X can equal zero in some circumstances. | ||||
3406 | // This simplification might be unsafe if C is zero. | ||||
3407 | // | ||||
3408 | // We know it is safe if: | ||||
3409 | // - The shift is nsw. We can't shift out the one bit. | ||||
3410 | // - The shift is nuw. We can't shift out the one bit. | ||||
3411 | // - C2 is one. | ||||
3412 | // - C isn't zero. | ||||
3413 | if (Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(LBO)) || | ||||
3414 | Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(LBO)) || | ||||
3415 | match(LHS, m_Shl(m_One(), m_Value())) || !C->isZero()) { | ||||
3416 | if (Pred == ICmpInst::ICMP_EQ) | ||||
3417 | return ConstantInt::getFalse(getCompareTy(RHS)); | ||||
3418 | if (Pred == ICmpInst::ICMP_NE) | ||||
3419 | return ConstantInt::getTrue(getCompareTy(RHS)); | ||||
3420 | } | ||||
3421 | } | ||||
3422 | |||||
3423 | // TODO: This is overly constrained. LHS can be any power-of-2. | ||||
3424 | // (1 << X) >u 0x8000 --> false | ||||
3425 | // (1 << X) <=u 0x8000 --> true | ||||
3426 | if (match(LHS, m_Shl(m_One(), m_Value())) && match(RHS, m_SignMask())) { | ||||
3427 | if (Pred == ICmpInst::ICMP_UGT) | ||||
3428 | return ConstantInt::getFalse(getCompareTy(RHS)); | ||||
3429 | if (Pred == ICmpInst::ICMP_ULE) | ||||
3430 | return ConstantInt::getTrue(getCompareTy(RHS)); | ||||
3431 | } | ||||
3432 | |||||
3433 | if (!MaxRecurse || !LBO || !RBO || LBO->getOpcode() != RBO->getOpcode()) | ||||
3434 | return nullptr; | ||||
3435 | |||||
3436 | if (LBO->getOperand(0) == RBO->getOperand(0)) { | ||||
3437 | switch (LBO->getOpcode()) { | ||||
3438 | default: | ||||
3439 | break; | ||||
3440 | case Instruction::Shl: | ||||
3441 | bool NUW = Q.IIQ.hasNoUnsignedWrap(LBO) && Q.IIQ.hasNoUnsignedWrap(RBO); | ||||
3442 | bool NSW = Q.IIQ.hasNoSignedWrap(LBO) && Q.IIQ.hasNoSignedWrap(RBO); | ||||
3443 | if (!NUW || (ICmpInst::isSigned(Pred) && !NSW) || | ||||
3444 | !isKnownNonZero(LBO->getOperand(0), Q.DL)) | ||||
3445 | break; | ||||
3446 | if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(1), | ||||
3447 | RBO->getOperand(1), Q, MaxRecurse - 1)) | ||||
3448 | return V; | ||||
3449 | } | ||||
3450 | } | ||||
3451 | |||||
3452 | if (LBO->getOperand(1) == RBO->getOperand(1)) { | ||||
3453 | switch (LBO->getOpcode()) { | ||||
3454 | default: | ||||
3455 | break; | ||||
3456 | case Instruction::UDiv: | ||||
3457 | case Instruction::LShr: | ||||
3458 | if (ICmpInst::isSigned(Pred) || !Q.IIQ.isExact(LBO) || | ||||
3459 | !Q.IIQ.isExact(RBO)) | ||||
3460 | break; | ||||
3461 | if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(0), | ||||
3462 | RBO->getOperand(0), Q, MaxRecurse - 1)) | ||||
3463 | return V; | ||||
3464 | break; | ||||
3465 | case Instruction::SDiv: | ||||
3466 | if (!ICmpInst::isEquality(Pred) || !Q.IIQ.isExact(LBO) || | ||||
3467 | !Q.IIQ.isExact(RBO)) | ||||
3468 | break; | ||||
3469 | if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(0), | ||||
3470 | RBO->getOperand(0), Q, MaxRecurse - 1)) | ||||
3471 | return V; | ||||
3472 | break; | ||||
3473 | case Instruction::AShr: | ||||
3474 | if (!Q.IIQ.isExact(LBO) || !Q.IIQ.isExact(RBO)) | ||||
3475 | break; | ||||
3476 | if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(0), | ||||
3477 | RBO->getOperand(0), Q, MaxRecurse - 1)) | ||||
3478 | return V; | ||||
3479 | break; | ||||
3480 | case Instruction::Shl: { | ||||
3481 | bool NUW = Q.IIQ.hasNoUnsignedWrap(LBO) && Q.IIQ.hasNoUnsignedWrap(RBO); | ||||
3482 | bool NSW = Q.IIQ.hasNoSignedWrap(LBO) && Q.IIQ.hasNoSignedWrap(RBO); | ||||
3483 | if (!NUW && !NSW) | ||||
3484 | break; | ||||
3485 | if (!NSW && ICmpInst::isSigned(Pred)) | ||||
3486 | break; | ||||
3487 | if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(0), | ||||
3488 | RBO->getOperand(0), Q, MaxRecurse - 1)) | ||||
3489 | return V; | ||||
3490 | break; | ||||
3491 | } | ||||
3492 | } | ||||
3493 | } | ||||
3494 | return nullptr; | ||||
3495 | } | ||||
3496 | |||||
3497 | /// simplify integer comparisons where at least one operand of the compare | ||||
3498 | /// matches an integer min/max idiom. | ||||
3499 | static Value *simplifyICmpWithMinMax(CmpInst::Predicate Pred, Value *LHS, | ||||
3500 | Value *RHS, const SimplifyQuery &Q, | ||||
3501 | unsigned MaxRecurse) { | ||||
3502 | Type *ITy = getCompareTy(LHS); // The return type. | ||||
3503 | Value *A, *B; | ||||
3504 | CmpInst::Predicate P = CmpInst::BAD_ICMP_PREDICATE; | ||||
3505 | CmpInst::Predicate EqP; // Chosen so that "A == max/min(A,B)" iff "A EqP B". | ||||
3506 | |||||
3507 | // Signed variants on "max(a,b)>=a -> true". | ||||
3508 | if (match(LHS, m_SMax(m_Value(A), m_Value(B))) && (A == RHS || B == RHS)) { | ||||
3509 | if (A != RHS) | ||||
3510 | std::swap(A, B); // smax(A, B) pred A. | ||||
3511 | EqP = CmpInst::ICMP_SGE; // "A == smax(A, B)" iff "A sge B". | ||||
3512 | // We analyze this as smax(A, B) pred A. | ||||
3513 | P = Pred; | ||||
3514 | } else if (match(RHS, m_SMax(m_Value(A), m_Value(B))) && | ||||
3515 | (A == LHS || B == LHS)) { | ||||
3516 | if (A != LHS) | ||||
3517 | std::swap(A, B); // A pred smax(A, B). | ||||
3518 | EqP = CmpInst::ICMP_SGE; // "A == smax(A, B)" iff "A sge B". | ||||
3519 | // We analyze this as smax(A, B) swapped-pred A. | ||||
3520 | P = CmpInst::getSwappedPredicate(Pred); | ||||
3521 | } else if (match(LHS, m_SMin(m_Value(A), m_Value(B))) && | ||||
3522 | (A == RHS || B == RHS)) { | ||||
3523 | if (A != RHS) | ||||
3524 | std::swap(A, B); // smin(A, B) pred A. | ||||
3525 | EqP = CmpInst::ICMP_SLE; // "A == smin(A, B)" iff "A sle B". | ||||
3526 | // We analyze this as smax(-A, -B) swapped-pred -A. | ||||
3527 | // Note that we do not need to actually form -A or -B thanks to EqP. | ||||
3528 | P = CmpInst::getSwappedPredicate(Pred); | ||||
3529 | } else if (match(RHS, m_SMin(m_Value(A), m_Value(B))) && | ||||
3530 | (A == LHS || B == LHS)) { | ||||
3531 | if (A != LHS) | ||||
3532 | std::swap(A, B); // A pred smin(A, B). | ||||
3533 | EqP = CmpInst::ICMP_SLE; // "A == smin(A, B)" iff "A sle B". | ||||
3534 | // We analyze this as smax(-A, -B) pred -A. | ||||
3535 | // Note that we do not need to actually form -A or -B thanks to EqP. | ||||
3536 | P = Pred; | ||||
3537 | } | ||||
3538 | if (P != CmpInst::BAD_ICMP_PREDICATE) { | ||||
3539 | // Cases correspond to "max(A, B) p A". | ||||
3540 | switch (P) { | ||||
3541 | default: | ||||
3542 | break; | ||||
3543 | case CmpInst::ICMP_EQ: | ||||
3544 | case CmpInst::ICMP_SLE: | ||||
3545 | // Equivalent to "A EqP B". This may be the same as the condition tested | ||||
3546 | // in the max/min; if so, we can just return that. | ||||
3547 | if (Value *V = extractEquivalentCondition(LHS, EqP, A, B)) | ||||
3548 | return V; | ||||
3549 | if (Value *V = extractEquivalentCondition(RHS, EqP, A, B)) | ||||
3550 | return V; | ||||
3551 | // Otherwise, see if "A EqP B" simplifies. | ||||
3552 | if (MaxRecurse) | ||||
3553 | if (Value *V = simplifyICmpInst(EqP, A, B, Q, MaxRecurse - 1)) | ||||
3554 | return V; | ||||
3555 | break; | ||||
3556 | case CmpInst::ICMP_NE: | ||||
3557 | case CmpInst::ICMP_SGT: { | ||||
3558 | CmpInst::Predicate InvEqP = CmpInst::getInversePredicate(EqP); | ||||
3559 | // Equivalent to "A InvEqP B". This may be the same as the condition | ||||
3560 | // tested in the max/min; if so, we can just return that. | ||||
3561 | if (Value *V = extractEquivalentCondition(LHS, InvEqP, A, B)) | ||||
3562 | return V; | ||||
3563 | if (Value *V = extractEquivalentCondition(RHS, InvEqP, A, B)) | ||||
3564 | return V; | ||||
3565 | // Otherwise, see if "A InvEqP B" simplifies. | ||||
3566 | if (MaxRecurse) | ||||
3567 | if (Value *V = simplifyICmpInst(InvEqP, A, B, Q, MaxRecurse - 1)) | ||||
3568 | return V; | ||||
3569 | break; | ||||
3570 | } | ||||
3571 | case CmpInst::ICMP_SGE: | ||||
3572 | // Always true. | ||||
3573 | return getTrue(ITy); | ||||
3574 | case CmpInst::ICMP_SLT: | ||||
3575 | // Always false. | ||||
3576 | return getFalse(ITy); | ||||
3577 | } | ||||
3578 | } | ||||
3579 | |||||
3580 | // Unsigned variants on "max(a,b)>=a -> true". | ||||
3581 | P = CmpInst::BAD_ICMP_PREDICATE; | ||||
3582 | if (match(LHS, m_UMax(m_Value(A), m_Value(B))) && (A == RHS || B == RHS)) { | ||||
3583 | if (A != RHS) | ||||
3584 | std::swap(A, B); // umax(A, B) pred A. | ||||
3585 | EqP = CmpInst::ICMP_UGE; // "A == umax(A, B)" iff "A uge B". | ||||
3586 | // We analyze this as umax(A, B) pred A. | ||||
3587 | P = Pred; | ||||
3588 | } else if (match(RHS, m_UMax(m_Value(A), m_Value(B))) && | ||||
3589 | (A == LHS || B == LHS)) { | ||||
3590 | if (A != LHS) | ||||
3591 | std::swap(A, B); // A pred umax(A, B). | ||||
3592 | EqP = CmpInst::ICMP_UGE; // "A == umax(A, B)" iff "A uge B". | ||||
3593 | // We analyze this as umax(A, B) swapped-pred A. | ||||
3594 | P = CmpInst::getSwappedPredicate(Pred); | ||||
3595 | } else if (match(LHS, m_UMin(m_Value(A), m_Value(B))) && | ||||
3596 | (A == RHS || B == RHS)) { | ||||
3597 | if (A != RHS) | ||||
3598 | std::swap(A, B); // umin(A, B) pred A. | ||||
3599 | EqP = CmpInst::ICMP_ULE; // "A == umin(A, B)" iff "A ule B". | ||||
3600 | // We analyze this as umax(-A, -B) swapped-pred -A. | ||||
3601 | // Note that we do not need to actually form -A or -B thanks to EqP. | ||||
3602 | P = CmpInst::getSwappedPredicate(Pred); | ||||
3603 | } else if (match(RHS, m_UMin(m_Value(A), m_Value(B))) && | ||||
3604 | (A == LHS || B == LHS)) { | ||||
3605 | if (A != LHS) | ||||
3606 | std::swap(A, B); // A pred umin(A, B). | ||||
3607 | EqP = CmpInst::ICMP_ULE; // "A == umin(A, B)" iff "A ule B". | ||||
3608 | // We analyze this as umax(-A, -B) pred -A. | ||||
3609 | // Note that we do not need to actually form -A or -B thanks to EqP. | ||||
3610 | P = Pred; | ||||
3611 | } | ||||
3612 | if (P != CmpInst::BAD_ICMP_PREDICATE) { | ||||
3613 | // Cases correspond to "max(A, B) p A". | ||||
3614 | switch (P) { | ||||
3615 | default: | ||||
3616 | break; | ||||
3617 | case CmpInst::ICMP_EQ: | ||||
3618 | case CmpInst::ICMP_ULE: | ||||
3619 | // Equivalent to "A EqP B". This may be the same as the condition tested | ||||
3620 | // in the max/min; if so, we can just return that. | ||||
3621 | if (Value *V = extractEquivalentCondition(LHS, EqP, A, B)) | ||||
3622 | return V; | ||||
3623 | if (Value *V = extractEquivalentCondition(RHS, EqP, A, B)) | ||||
3624 | return V; | ||||
3625 | // Otherwise, see if "A EqP B" simplifies. | ||||
3626 | if (MaxRecurse) | ||||
3627 | if (Value *V = simplifyICmpInst(EqP, A, B, Q, MaxRecurse - 1)) | ||||
3628 | return V; | ||||
3629 | break; | ||||
3630 | case CmpInst::ICMP_NE: | ||||
3631 | case CmpInst::ICMP_UGT: { | ||||
3632 | CmpInst::Predicate InvEqP = CmpInst::getInversePredicate(EqP); | ||||
3633 | // Equivalent to "A InvEqP B". This may be the same as the condition | ||||
3634 | // tested in the max/min; if so, we can just return that. | ||||
3635 | if (Value *V = extractEquivalentCondition(LHS, InvEqP, A, B)) | ||||
3636 | return V; | ||||
3637 | if (Value *V = extractEquivalentCondition(RHS, InvEqP, A, B)) | ||||
3638 | return V; | ||||
3639 | // Otherwise, see if "A InvEqP B" simplifies. | ||||
3640 | if (MaxRecurse) | ||||
3641 | if (Value *V = simplifyICmpInst(InvEqP, A, B, Q, MaxRecurse - 1)) | ||||
3642 | return V; | ||||
3643 | break; | ||||
3644 | } | ||||
3645 | case CmpInst::ICMP_UGE: | ||||
3646 | return getTrue(ITy); | ||||
3647 | case CmpInst::ICMP_ULT: | ||||
3648 | return getFalse(ITy); | ||||
3649 | } | ||||
3650 | } | ||||
3651 | |||||
3652 | // Comparing 1 each of min/max with a common operand? | ||||
3653 | // Canonicalize min operand to RHS. | ||||
3654 | if (match(LHS, m_UMin(m_Value(), m_Value())) || | ||||
3655 | match(LHS, m_SMin(m_Value(), m_Value()))) { | ||||
3656 | std::swap(LHS, RHS); | ||||
3657 | Pred = ICmpInst::getSwappedPredicate(Pred); | ||||
3658 | } | ||||
3659 | |||||
3660 | Value *C, *D; | ||||
3661 | if (match(LHS, m_SMax(m_Value(A), m_Value(B))) && | ||||
3662 | match(RHS, m_SMin(m_Value(C), m_Value(D))) && | ||||
3663 | (A == C || A == D || B == C || B == D)) { | ||||
3664 | // smax(A, B) >=s smin(A, D) --> true | ||||
3665 | if (Pred == CmpInst::ICMP_SGE) | ||||
3666 | return getTrue(ITy); | ||||
3667 | // smax(A, B) <s smin(A, D) --> false | ||||
3668 | if (Pred == CmpInst::ICMP_SLT) | ||||
3669 | return getFalse(ITy); | ||||
3670 | } else if (match(LHS, m_UMax(m_Value(A), m_Value(B))) && | ||||
3671 | match(RHS, m_UMin(m_Value(C), m_Value(D))) && | ||||
3672 | (A == C || A == D || B == C || B == D)) { | ||||
3673 | // umax(A, B) >=u umin(A, D) --> true | ||||
3674 | if (Pred == CmpInst::ICMP_UGE) | ||||
3675 | return getTrue(ITy); | ||||
3676 | // umax(A, B) <u umin(A, D) --> false | ||||
3677 | if (Pred == CmpInst::ICMP_ULT) | ||||
3678 | return getFalse(ITy); | ||||
3679 | } | ||||
3680 | |||||
3681 | return nullptr; | ||||
3682 | } | ||||
3683 | |||||
3684 | static Value *simplifyICmpWithDominatingAssume(CmpInst::Predicate Predicate, | ||||
3685 | Value *LHS, Value *RHS, | ||||
3686 | const SimplifyQuery &Q) { | ||||
3687 | // Gracefully handle instructions that have not been inserted yet. | ||||
3688 | if (!Q.AC || !Q.CxtI || !Q.CxtI->getParent()) | ||||
3689 | return nullptr; | ||||
3690 | |||||
3691 | for (Value *AssumeBaseOp : {LHS, RHS}) { | ||||
3692 | for (auto &AssumeVH : Q.AC->assumptionsFor(AssumeBaseOp)) { | ||||
3693 | if (!AssumeVH) | ||||
3694 | continue; | ||||
3695 | |||||
3696 | CallInst *Assume = cast<CallInst>(AssumeVH); | ||||
3697 | if (std::optional<bool> Imp = isImpliedCondition( | ||||
3698 | Assume->getArgOperand(0), Predicate, LHS, RHS, Q.DL)) | ||||
3699 | if (isValidAssumeForContext(Assume, Q.CxtI, Q.DT)) | ||||
3700 | return ConstantInt::get(getCompareTy(LHS), *Imp); | ||||
3701 | } | ||||
3702 | } | ||||
3703 | |||||
3704 | return nullptr; | ||||
3705 | } | ||||
3706 | |||||
3707 | /// Given operands for an ICmpInst, see if we can fold the result. | ||||
3708 | /// If not, this returns null. | ||||
3709 | static Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, | ||||
3710 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
3711 | CmpInst::Predicate Pred = (CmpInst::Predicate)Predicate; | ||||
3712 | assert(CmpInst::isIntPredicate(Pred) && "Not an integer compare!")(static_cast <bool> (CmpInst::isIntPredicate(Pred) && "Not an integer compare!") ? void (0) : __assert_fail ("CmpInst::isIntPredicate(Pred) && \"Not an integer compare!\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 3712, __extension__ __PRETTY_FUNCTION__)); | ||||
3713 | |||||
3714 | if (Constant *CLHS = dyn_cast<Constant>(LHS)) { | ||||
3715 | if (Constant *CRHS = dyn_cast<Constant>(RHS)) | ||||
3716 | return ConstantFoldCompareInstOperands(Pred, CLHS, CRHS, Q.DL, Q.TLI); | ||||
3717 | |||||
3718 | // If we have a constant, make sure it is on the RHS. | ||||
3719 | std::swap(LHS, RHS); | ||||
3720 | Pred = CmpInst::getSwappedPredicate(Pred); | ||||
3721 | } | ||||
3722 | assert(!isa<UndefValue>(LHS) && "Unexpected icmp undef,%X")(static_cast <bool> (!isa<UndefValue>(LHS) && "Unexpected icmp undef,%X") ? void (0) : __assert_fail ("!isa<UndefValue>(LHS) && \"Unexpected icmp undef,%X\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 3722, __extension__ __PRETTY_FUNCTION__)); | ||||
3723 | |||||
3724 | Type *ITy = getCompareTy(LHS); // The return type. | ||||
3725 | |||||
3726 | // icmp poison, X -> poison | ||||
3727 | if (isa<PoisonValue>(RHS)) | ||||
3728 | return PoisonValue::get(ITy); | ||||
3729 | |||||
3730 | // For EQ and NE, we can always pick a value for the undef to make the | ||||
3731 | // predicate pass or fail, so we can return undef. | ||||
3732 | // Matches behavior in llvm::ConstantFoldCompareInstruction. | ||||
3733 | if (Q.isUndefValue(RHS) && ICmpInst::isEquality(Pred)) | ||||
3734 | return UndefValue::get(ITy); | ||||
3735 | |||||
3736 | // icmp X, X -> true/false | ||||
3737 | // icmp X, undef -> true/false because undef could be X. | ||||
3738 | if (LHS == RHS || Q.isUndefValue(RHS)) | ||||
3739 | return ConstantInt::get(ITy, CmpInst::isTrueWhenEqual(Pred)); | ||||
3740 | |||||
3741 | if (Value *V = simplifyICmpOfBools(Pred, LHS, RHS, Q)) | ||||
3742 | return V; | ||||
3743 | |||||
3744 | // TODO: Sink/common this with other potentially expensive calls that use | ||||
3745 | // ValueTracking? See comment below for isKnownNonEqual(). | ||||
3746 | if (Value *V = simplifyICmpWithZero(Pred, LHS, RHS, Q)) | ||||
3747 | return V; | ||||
3748 | |||||
3749 | if (Value *V = simplifyICmpWithConstant(Pred, LHS, RHS, Q.IIQ)) | ||||
3750 | return V; | ||||
3751 | |||||
3752 | // If both operands have range metadata, use the metadata | ||||
3753 | // to simplify the comparison. | ||||
3754 | if (isa<Instruction>(RHS) && isa<Instruction>(LHS)) { | ||||
3755 | auto RHS_Instr = cast<Instruction>(RHS); | ||||
3756 | auto LHS_Instr = cast<Instruction>(LHS); | ||||
3757 | |||||
3758 | if (Q.IIQ.getMetadata(RHS_Instr, LLVMContext::MD_range) && | ||||
3759 | Q.IIQ.getMetadata(LHS_Instr, LLVMContext::MD_range)) { | ||||
3760 | auto RHS_CR = getConstantRangeFromMetadata( | ||||
3761 | *RHS_Instr->getMetadata(LLVMContext::MD_range)); | ||||
3762 | auto LHS_CR = getConstantRangeFromMetadata( | ||||
3763 | *LHS_Instr->getMetadata(LLVMContext::MD_range)); | ||||
3764 | |||||
3765 | if (LHS_CR.icmp(Pred, RHS_CR)) | ||||
3766 | return ConstantInt::getTrue(RHS->getContext()); | ||||
3767 | |||||
3768 | if (LHS_CR.icmp(CmpInst::getInversePredicate(Pred), RHS_CR)) | ||||
3769 | return ConstantInt::getFalse(RHS->getContext()); | ||||
3770 | } | ||||
3771 | } | ||||
3772 | |||||
3773 | // Compare of cast, for example (zext X) != 0 -> X != 0 | ||||
3774 | if (isa<CastInst>(LHS) && (isa<Constant>(RHS) || isa<CastInst>(RHS))) { | ||||
3775 | Instruction *LI = cast<CastInst>(LHS); | ||||
3776 | Value *SrcOp = LI->getOperand(0); | ||||
3777 | Type *SrcTy = SrcOp->getType(); | ||||
3778 | Type *DstTy = LI->getType(); | ||||
3779 | |||||
3780 | // Turn icmp (ptrtoint x), (ptrtoint/constant) into a compare of the input | ||||
3781 | // if the integer type is the same size as the pointer type. | ||||
3782 | if (MaxRecurse && isa<PtrToIntInst>(LI) && | ||||
3783 | Q.DL.getTypeSizeInBits(SrcTy) == DstTy->getPrimitiveSizeInBits()) { | ||||
3784 | if (Constant *RHSC = dyn_cast<Constant>(RHS)) { | ||||
3785 | // Transfer the cast to the constant. | ||||
3786 | if (Value *V = simplifyICmpInst(Pred, SrcOp, | ||||
3787 | ConstantExpr::getIntToPtr(RHSC, SrcTy), | ||||
3788 | Q, MaxRecurse - 1)) | ||||
3789 | return V; | ||||
3790 | } else if (PtrToIntInst *RI = dyn_cast<PtrToIntInst>(RHS)) { | ||||
3791 | if (RI->getOperand(0)->getType() == SrcTy) | ||||
3792 | // Compare without the cast. | ||||
3793 | if (Value *V = simplifyICmpInst(Pred, SrcOp, RI->getOperand(0), Q, | ||||
3794 | MaxRecurse - 1)) | ||||
3795 | return V; | ||||
3796 | } | ||||
3797 | } | ||||
3798 | |||||
3799 | if (isa<ZExtInst>(LHS)) { | ||||
3800 | // Turn icmp (zext X), (zext Y) into a compare of X and Y if they have the | ||||
3801 | // same type. | ||||
3802 | if (ZExtInst *RI = dyn_cast<ZExtInst>(RHS)) { | ||||
3803 | if (MaxRecurse && SrcTy == RI->getOperand(0)->getType()) | ||||
3804 | // Compare X and Y. Note that signed predicates become unsigned. | ||||
3805 | if (Value *V = | ||||
3806 | simplifyICmpInst(ICmpInst::getUnsignedPredicate(Pred), SrcOp, | ||||
3807 | RI->getOperand(0), Q, MaxRecurse - 1)) | ||||
3808 | return V; | ||||
3809 | } | ||||
3810 | // Fold (zext X) ule (sext X), (zext X) sge (sext X) to true. | ||||
3811 | else if (SExtInst *RI = dyn_cast<SExtInst>(RHS)) { | ||||
3812 | if (SrcOp == RI->getOperand(0)) { | ||||
3813 | if (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_SGE) | ||||
3814 | return ConstantInt::getTrue(ITy); | ||||
3815 | if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_SLT) | ||||
3816 | return ConstantInt::getFalse(ITy); | ||||
3817 | } | ||||
3818 | } | ||||
3819 | // Turn icmp (zext X), Cst into a compare of X and Cst if Cst is extended | ||||
3820 | // too. If not, then try to deduce the result of the comparison. | ||||
3821 | else if (ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) { | ||||
3822 | // Compute the constant that would happen if we truncated to SrcTy then | ||||
3823 | // reextended to DstTy. | ||||
3824 | Constant *Trunc = ConstantExpr::getTrunc(CI, SrcTy); | ||||
3825 | Constant *RExt = ConstantExpr::getCast(CastInst::ZExt, Trunc, DstTy); | ||||
3826 | |||||
3827 | // If the re-extended constant didn't change then this is effectively | ||||
3828 | // also a case of comparing two zero-extended values. | ||||
3829 | if (RExt == CI && MaxRecurse) | ||||
3830 | if (Value *V = simplifyICmpInst(ICmpInst::getUnsignedPredicate(Pred), | ||||
3831 | SrcOp, Trunc, Q, MaxRecurse - 1)) | ||||
3832 | return V; | ||||
3833 | |||||
3834 | // Otherwise the upper bits of LHS are zero while RHS has a non-zero bit | ||||
3835 | // there. Use this to work out the result of the comparison. | ||||
3836 | if (RExt != CI) { | ||||
3837 | switch (Pred) { | ||||
3838 | default: | ||||
3839 | llvm_unreachable("Unknown ICmp predicate!")::llvm::llvm_unreachable_internal("Unknown ICmp predicate!", "llvm/lib/Analysis/InstructionSimplify.cpp" , 3839); | ||||
3840 | // LHS <u RHS. | ||||
3841 | case ICmpInst::ICMP_EQ: | ||||
3842 | case ICmpInst::ICMP_UGT: | ||||
3843 | case ICmpInst::ICMP_UGE: | ||||
3844 | return ConstantInt::getFalse(CI->getContext()); | ||||
3845 | |||||
3846 | case ICmpInst::ICMP_NE: | ||||
3847 | case ICmpInst::ICMP_ULT: | ||||
3848 | case ICmpInst::ICMP_ULE: | ||||
3849 | return ConstantInt::getTrue(CI->getContext()); | ||||
3850 | |||||
3851 | // LHS is non-negative. If RHS is negative then LHS >s LHS. If RHS | ||||
3852 | // is non-negative then LHS <s RHS. | ||||
3853 | case ICmpInst::ICMP_SGT: | ||||
3854 | case ICmpInst::ICMP_SGE: | ||||
3855 | return CI->getValue().isNegative() | ||||
3856 | ? ConstantInt::getTrue(CI->getContext()) | ||||
3857 | : ConstantInt::getFalse(CI->getContext()); | ||||
3858 | |||||
3859 | case ICmpInst::ICMP_SLT: | ||||
3860 | case ICmpInst::ICMP_SLE: | ||||
3861 | return CI->getValue().isNegative() | ||||
3862 | ? ConstantInt::getFalse(CI->getContext()) | ||||
3863 | : ConstantInt::getTrue(CI->getContext()); | ||||
3864 | } | ||||
3865 | } | ||||
3866 | } | ||||
3867 | } | ||||
3868 | |||||
3869 | if (isa<SExtInst>(LHS)) { | ||||
3870 | // Turn icmp (sext X), (sext Y) into a compare of X and Y if they have the | ||||
3871 | // same type. | ||||
3872 | if (SExtInst *RI = dyn_cast<SExtInst>(RHS)) { | ||||
3873 | if (MaxRecurse && SrcTy == RI->getOperand(0)->getType()) | ||||
3874 | // Compare X and Y. Note that the predicate does not change. | ||||
3875 | if (Value *V = simplifyICmpInst(Pred, SrcOp, RI->getOperand(0), Q, | ||||
3876 | MaxRecurse - 1)) | ||||
3877 | return V; | ||||
3878 | } | ||||
3879 | // Fold (sext X) uge (zext X), (sext X) sle (zext X) to true. | ||||
3880 | else if (ZExtInst *RI = dyn_cast<ZExtInst>(RHS)) { | ||||
3881 | if (SrcOp == RI->getOperand(0)) { | ||||
3882 | if (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_SLE) | ||||
3883 | return ConstantInt::getTrue(ITy); | ||||
3884 | if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_SGT) | ||||
3885 | return ConstantInt::getFalse(ITy); | ||||
3886 | } | ||||
3887 | } | ||||
3888 | // Turn icmp (sext X), Cst into a compare of X and Cst if Cst is extended | ||||
3889 | // too. If not, then try to deduce the result of the comparison. | ||||
3890 | else if (ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) { | ||||
3891 | // Compute the constant that would happen if we truncated to SrcTy then | ||||
3892 | // reextended to DstTy. | ||||
3893 | Constant *Trunc = ConstantExpr::getTrunc(CI, SrcTy); | ||||
3894 | Constant *RExt = ConstantExpr::getCast(CastInst::SExt, Trunc, DstTy); | ||||
3895 | |||||
3896 | // If the re-extended constant didn't change then this is effectively | ||||
3897 | // also a case of comparing two sign-extended values. | ||||
3898 | if (RExt == CI && MaxRecurse) | ||||
3899 | if (Value *V = | ||||
3900 | simplifyICmpInst(Pred, SrcOp, Trunc, Q, MaxRecurse - 1)) | ||||
3901 | return V; | ||||
3902 | |||||
3903 | // Otherwise the upper bits of LHS are all equal, while RHS has varying | ||||
3904 | // bits there. Use this to work out the result of the comparison. | ||||
3905 | if (RExt != CI) { | ||||
3906 | switch (Pred) { | ||||
3907 | default: | ||||
3908 | llvm_unreachable("Unknown ICmp predicate!")::llvm::llvm_unreachable_internal("Unknown ICmp predicate!", "llvm/lib/Analysis/InstructionSimplify.cpp" , 3908); | ||||
3909 | case ICmpInst::ICMP_EQ: | ||||
3910 | return ConstantInt::getFalse(CI->getContext()); | ||||
3911 | case ICmpInst::ICMP_NE: | ||||
3912 | return ConstantInt::getTrue(CI->getContext()); | ||||
3913 | |||||
3914 | // If RHS is non-negative then LHS <s RHS. If RHS is negative then | ||||
3915 | // LHS >s RHS. | ||||
3916 | case ICmpInst::ICMP_SGT: | ||||
3917 | case ICmpInst::ICMP_SGE: | ||||
3918 | return CI->getValue().isNegative() | ||||
3919 | ? ConstantInt::getTrue(CI->getContext()) | ||||
3920 | : ConstantInt::getFalse(CI->getContext()); | ||||
3921 | case ICmpInst::ICMP_SLT: | ||||
3922 | case ICmpInst::ICMP_SLE: | ||||
3923 | return CI->getValue().isNegative() | ||||
3924 | ? ConstantInt::getFalse(CI->getContext()) | ||||
3925 | : ConstantInt::getTrue(CI->getContext()); | ||||
3926 | |||||
3927 | // If LHS is non-negative then LHS <u RHS. If LHS is negative then | ||||
3928 | // LHS >u RHS. | ||||
3929 | case ICmpInst::ICMP_UGT: | ||||
3930 | case ICmpInst::ICMP_UGE: | ||||
3931 | // Comparison is true iff the LHS <s 0. | ||||
3932 | if (MaxRecurse) | ||||
3933 | if (Value *V = simplifyICmpInst(ICmpInst::ICMP_SLT, SrcOp, | ||||
3934 | Constant::getNullValue(SrcTy), Q, | ||||
3935 | MaxRecurse - 1)) | ||||
3936 | return V; | ||||
3937 | break; | ||||
3938 | case ICmpInst::ICMP_ULT: | ||||
3939 | case ICmpInst::ICMP_ULE: | ||||
3940 | // Comparison is true iff the LHS >=s 0. | ||||
3941 | if (MaxRecurse) | ||||
3942 | if (Value *V = simplifyICmpInst(ICmpInst::ICMP_SGE, SrcOp, | ||||
3943 | Constant::getNullValue(SrcTy), Q, | ||||
3944 | MaxRecurse - 1)) | ||||
3945 | return V; | ||||
3946 | break; | ||||
3947 | } | ||||
3948 | } | ||||
3949 | } | ||||
3950 | } | ||||
3951 | } | ||||
3952 | |||||
3953 | // icmp eq|ne X, Y -> false|true if X != Y | ||||
3954 | // This is potentially expensive, and we have already computedKnownBits for | ||||
3955 | // compares with 0 above here, so only try this for a non-zero compare. | ||||
3956 | if (ICmpInst::isEquality(Pred) && !match(RHS, m_Zero()) && | ||||
3957 | isKnownNonEqual(LHS, RHS, Q.DL, Q.AC, Q.CxtI, Q.DT, Q.IIQ.UseInstrInfo)) { | ||||
3958 | return Pred == ICmpInst::ICMP_NE ? getTrue(ITy) : getFalse(ITy); | ||||
3959 | } | ||||
3960 | |||||
3961 | if (Value *V = simplifyICmpWithBinOp(Pred, LHS, RHS, Q, MaxRecurse)) | ||||
3962 | return V; | ||||
3963 | |||||
3964 | if (Value *V = simplifyICmpWithMinMax(Pred, LHS, RHS, Q, MaxRecurse)) | ||||
3965 | return V; | ||||
3966 | |||||
3967 | if (Value *V = simplifyICmpWithDominatingAssume(Pred, LHS, RHS, Q)) | ||||
3968 | return V; | ||||
3969 | |||||
3970 | if (std::optional<bool> Res = | ||||
3971 | isImpliedByDomCondition(Pred, LHS, RHS, Q.CxtI, Q.DL)) | ||||
3972 | return ConstantInt::getBool(ITy, *Res); | ||||
3973 | |||||
3974 | // Simplify comparisons of related pointers using a powerful, recursive | ||||
3975 | // GEP-walk when we have target data available.. | ||||
3976 | if (LHS->getType()->isPointerTy()) | ||||
3977 | if (auto *C = computePointerICmp(Pred, LHS, RHS, Q)) | ||||
3978 | return C; | ||||
3979 | if (auto *CLHS = dyn_cast<PtrToIntOperator>(LHS)) | ||||
3980 | if (auto *CRHS = dyn_cast<PtrToIntOperator>(RHS)) | ||||
3981 | if (Q.DL.getTypeSizeInBits(CLHS->getPointerOperandType()) == | ||||
3982 | Q.DL.getTypeSizeInBits(CLHS->getType()) && | ||||
3983 | Q.DL.getTypeSizeInBits(CRHS->getPointerOperandType()) == | ||||
3984 | Q.DL.getTypeSizeInBits(CRHS->getType())) | ||||
3985 | if (auto *C = computePointerICmp(Pred, CLHS->getPointerOperand(), | ||||
3986 | CRHS->getPointerOperand(), Q)) | ||||
3987 | return C; | ||||
3988 | |||||
3989 | // If the comparison is with the result of a select instruction, check whether | ||||
3990 | // comparing with either branch of the select always yields the same value. | ||||
3991 | if (isa<SelectInst>(LHS) || isa<SelectInst>(RHS)) | ||||
3992 | if (Value *V = threadCmpOverSelect(Pred, LHS, RHS, Q, MaxRecurse)) | ||||
3993 | return V; | ||||
3994 | |||||
3995 | // If the comparison is with the result of a phi instruction, check whether | ||||
3996 | // doing the compare with each incoming phi value yields a common result. | ||||
3997 | if (isa<PHINode>(LHS) || isa<PHINode>(RHS)) | ||||
3998 | if (Value *V = threadCmpOverPHI(Pred, LHS, RHS, Q, MaxRecurse)) | ||||
3999 | return V; | ||||
4000 | |||||
4001 | return nullptr; | ||||
4002 | } | ||||
4003 | |||||
4004 | Value *llvm::simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, | ||||
4005 | const SimplifyQuery &Q) { | ||||
4006 | return ::simplifyICmpInst(Predicate, LHS, RHS, Q, RecursionLimit); | ||||
4007 | } | ||||
4008 | |||||
4009 | /// Given operands for an FCmpInst, see if we can fold the result. | ||||
4010 | /// If not, this returns null. | ||||
4011 | static Value *simplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, | ||||
4012 | FastMathFlags FMF, const SimplifyQuery &Q, | ||||
4013 | unsigned MaxRecurse) { | ||||
4014 | CmpInst::Predicate Pred = (CmpInst::Predicate)Predicate; | ||||
4015 | assert(CmpInst::isFPPredicate(Pred) && "Not an FP compare!")(static_cast <bool> (CmpInst::isFPPredicate(Pred) && "Not an FP compare!") ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Pred) && \"Not an FP compare!\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 4015, __extension__ __PRETTY_FUNCTION__)); | ||||
4016 | |||||
4017 | if (Constant *CLHS = dyn_cast<Constant>(LHS)) { | ||||
4018 | if (Constant *CRHS = dyn_cast<Constant>(RHS)) | ||||
4019 | return ConstantFoldCompareInstOperands(Pred, CLHS, CRHS, Q.DL, Q.TLI, | ||||
4020 | Q.CxtI); | ||||
4021 | |||||
4022 | // If we have a constant, make sure it is on the RHS. | ||||
4023 | std::swap(LHS, RHS); | ||||
4024 | Pred = CmpInst::getSwappedPredicate(Pred); | ||||
4025 | } | ||||
4026 | |||||
4027 | // Fold trivial predicates. | ||||
4028 | Type *RetTy = getCompareTy(LHS); | ||||
4029 | if (Pred == FCmpInst::FCMP_FALSE) | ||||
4030 | return getFalse(RetTy); | ||||
4031 | if (Pred == FCmpInst::FCMP_TRUE) | ||||
4032 | return getTrue(RetTy); | ||||
4033 | |||||
4034 | // Fold (un)ordered comparison if we can determine there are no NaNs. | ||||
4035 | if (Pred == FCmpInst::FCMP_UNO || Pred == FCmpInst::FCMP_ORD) | ||||
4036 | if (FMF.noNaNs() || | ||||
4037 | (isKnownNeverNaN(LHS, Q.TLI) && isKnownNeverNaN(RHS, Q.TLI))) | ||||
4038 | return ConstantInt::get(RetTy, Pred == FCmpInst::FCMP_ORD); | ||||
4039 | |||||
4040 | // NaN is unordered; NaN is not ordered. | ||||
4041 | assert((FCmpInst::isOrdered(Pred) || FCmpInst::isUnordered(Pred)) &&(static_cast <bool> ((FCmpInst::isOrdered(Pred) || FCmpInst ::isUnordered(Pred)) && "Comparison must be either ordered or unordered" ) ? void (0) : __assert_fail ("(FCmpInst::isOrdered(Pred) || FCmpInst::isUnordered(Pred)) && \"Comparison must be either ordered or unordered\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 4042, __extension__ __PRETTY_FUNCTION__)) | ||||
4042 | "Comparison must be either ordered or unordered")(static_cast <bool> ((FCmpInst::isOrdered(Pred) || FCmpInst ::isUnordered(Pred)) && "Comparison must be either ordered or unordered" ) ? void (0) : __assert_fail ("(FCmpInst::isOrdered(Pred) || FCmpInst::isUnordered(Pred)) && \"Comparison must be either ordered or unordered\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 4042, __extension__ __PRETTY_FUNCTION__)); | ||||
4043 | if (match(RHS, m_NaN())) | ||||
4044 | return ConstantInt::get(RetTy, CmpInst::isUnordered(Pred)); | ||||
4045 | |||||
4046 | // fcmp pred x, poison and fcmp pred poison, x | ||||
4047 | // fold to poison | ||||
4048 | if (isa<PoisonValue>(LHS) || isa<PoisonValue>(RHS)) | ||||
4049 | return PoisonValue::get(RetTy); | ||||
4050 | |||||
4051 | // fcmp pred x, undef and fcmp pred undef, x | ||||
4052 | // fold to true if unordered, false if ordered | ||||
4053 | if (Q.isUndefValue(LHS) || Q.isUndefValue(RHS)) { | ||||
4054 | // Choosing NaN for the undef will always make unordered comparison succeed | ||||
4055 | // and ordered comparison fail. | ||||
4056 | return ConstantInt::get(RetTy, CmpInst::isUnordered(Pred)); | ||||
4057 | } | ||||
4058 | |||||
4059 | // fcmp x,x -> true/false. Not all compares are foldable. | ||||
4060 | if (LHS == RHS) { | ||||
4061 | if (CmpInst::isTrueWhenEqual(Pred)) | ||||
4062 | return getTrue(RetTy); | ||||
4063 | if (CmpInst::isFalseWhenEqual(Pred)) | ||||
4064 | return getFalse(RetTy); | ||||
4065 | } | ||||
4066 | |||||
4067 | // Handle fcmp with constant RHS. | ||||
4068 | // TODO: Use match with a specific FP value, so these work with vectors with | ||||
4069 | // undef lanes. | ||||
4070 | const APFloat *C; | ||||
4071 | if (match(RHS, m_APFloat(C))) { | ||||
4072 | // Check whether the constant is an infinity. | ||||
4073 | if (C->isInfinity()) { | ||||
4074 | if (C->isNegative()) { | ||||
4075 | switch (Pred) { | ||||
4076 | case FCmpInst::FCMP_OLT: | ||||
4077 | // No value is ordered and less than negative infinity. | ||||
4078 | return getFalse(RetTy); | ||||
4079 | case FCmpInst::FCMP_UGE: | ||||
4080 | // All values are unordered with or at least negative infinity. | ||||
4081 | return getTrue(RetTy); | ||||
4082 | default: | ||||
4083 | break; | ||||
4084 | } | ||||
4085 | } else { | ||||
4086 | switch (Pred) { | ||||
4087 | case FCmpInst::FCMP_OGT: | ||||
4088 | // No value is ordered and greater than infinity. | ||||
4089 | return getFalse(RetTy); | ||||
4090 | case FCmpInst::FCMP_ULE: | ||||
4091 | // All values are unordered with and at most infinity. | ||||
4092 | return getTrue(RetTy); | ||||
4093 | default: | ||||
4094 | break; | ||||
4095 | } | ||||
4096 | } | ||||
4097 | |||||
4098 | // LHS == Inf | ||||
4099 | if (Pred == FCmpInst::FCMP_OEQ && isKnownNeverInfinity(LHS, Q.TLI)) | ||||
4100 | return getFalse(RetTy); | ||||
4101 | // LHS != Inf | ||||
4102 | if (Pred == FCmpInst::FCMP_UNE && isKnownNeverInfinity(LHS, Q.TLI)) | ||||
4103 | return getTrue(RetTy); | ||||
4104 | // LHS == Inf || LHS == NaN | ||||
4105 | if (Pred == FCmpInst::FCMP_UEQ && isKnownNeverInfinity(LHS, Q.TLI) && | ||||
4106 | isKnownNeverNaN(LHS, Q.TLI)) | ||||
4107 | return getFalse(RetTy); | ||||
4108 | // LHS != Inf && LHS != NaN | ||||
4109 | if (Pred == FCmpInst::FCMP_ONE && isKnownNeverInfinity(LHS, Q.TLI) && | ||||
4110 | isKnownNeverNaN(LHS, Q.TLI)) | ||||
4111 | return getTrue(RetTy); | ||||
4112 | } | ||||
4113 | if (C->isNegative() && !C->isNegZero()) { | ||||
4114 | assert(!C->isNaN() && "Unexpected NaN constant!")(static_cast <bool> (!C->isNaN() && "Unexpected NaN constant!" ) ? void (0) : __assert_fail ("!C->isNaN() && \"Unexpected NaN constant!\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 4114, __extension__ __PRETTY_FUNCTION__)); | ||||
4115 | // TODO: We can catch more cases by using a range check rather than | ||||
4116 | // relying on CannotBeOrderedLessThanZero. | ||||
4117 | switch (Pred) { | ||||
4118 | case FCmpInst::FCMP_UGE: | ||||
4119 | case FCmpInst::FCMP_UGT: | ||||
4120 | case FCmpInst::FCMP_UNE: | ||||
4121 | // (X >= 0) implies (X > C) when (C < 0) | ||||
4122 | if (CannotBeOrderedLessThanZero(LHS, Q.TLI)) | ||||
4123 | return getTrue(RetTy); | ||||
4124 | break; | ||||
4125 | case FCmpInst::FCMP_OEQ: | ||||
4126 | case FCmpInst::FCMP_OLE: | ||||
4127 | case FCmpInst::FCMP_OLT: | ||||
4128 | // (X >= 0) implies !(X < C) when (C < 0) | ||||
4129 | if (CannotBeOrderedLessThanZero(LHS, Q.TLI)) | ||||
4130 | return getFalse(RetTy); | ||||
4131 | break; | ||||
4132 | default: | ||||
4133 | break; | ||||
4134 | } | ||||
4135 | } | ||||
4136 | |||||
4137 | // Check comparison of [minnum/maxnum with constant] with other constant. | ||||
4138 | const APFloat *C2; | ||||
4139 | if ((match(LHS, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_APFloat(C2))) && | ||||
4140 | *C2 < *C) || | ||||
4141 | (match(LHS, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_APFloat(C2))) && | ||||
4142 | *C2 > *C)) { | ||||
4143 | bool IsMaxNum = | ||||
4144 | cast<IntrinsicInst>(LHS)->getIntrinsicID() == Intrinsic::maxnum; | ||||
4145 | // The ordered relationship and minnum/maxnum guarantee that we do not | ||||
4146 | // have NaN constants, so ordered/unordered preds are handled the same. | ||||
4147 | switch (Pred) { | ||||
4148 | case FCmpInst::FCMP_OEQ: | ||||
4149 | case FCmpInst::FCMP_UEQ: | ||||
4150 | // minnum(X, LesserC) == C --> false | ||||
4151 | // maxnum(X, GreaterC) == C --> false | ||||
4152 | return getFalse(RetTy); | ||||
4153 | case FCmpInst::FCMP_ONE: | ||||
4154 | case FCmpInst::FCMP_UNE: | ||||
4155 | // minnum(X, LesserC) != C --> true | ||||
4156 | // maxnum(X, GreaterC) != C --> true | ||||
4157 | return getTrue(RetTy); | ||||
4158 | case FCmpInst::FCMP_OGE: | ||||
4159 | case FCmpInst::FCMP_UGE: | ||||
4160 | case FCmpInst::FCMP_OGT: | ||||
4161 | case FCmpInst::FCMP_UGT: | ||||
4162 | // minnum(X, LesserC) >= C --> false | ||||
4163 | // minnum(X, LesserC) > C --> false | ||||
4164 | // maxnum(X, GreaterC) >= C --> true | ||||
4165 | // maxnum(X, GreaterC) > C --> true | ||||
4166 | return ConstantInt::get(RetTy, IsMaxNum); | ||||
4167 | case FCmpInst::FCMP_OLE: | ||||
4168 | case FCmpInst::FCMP_ULE: | ||||
4169 | case FCmpInst::FCMP_OLT: | ||||
4170 | case FCmpInst::FCMP_ULT: | ||||
4171 | // minnum(X, LesserC) <= C --> true | ||||
4172 | // minnum(X, LesserC) < C --> true | ||||
4173 | // maxnum(X, GreaterC) <= C --> false | ||||
4174 | // maxnum(X, GreaterC) < C --> false | ||||
4175 | return ConstantInt::get(RetTy, !IsMaxNum); | ||||
4176 | default: | ||||
4177 | // TRUE/FALSE/ORD/UNO should be handled before this. | ||||
4178 | llvm_unreachable("Unexpected fcmp predicate")::llvm::llvm_unreachable_internal("Unexpected fcmp predicate" , "llvm/lib/Analysis/InstructionSimplify.cpp", 4178); | ||||
4179 | } | ||||
4180 | } | ||||
4181 | } | ||||
4182 | |||||
4183 | if (match(RHS, m_AnyZeroFP())) { | ||||
4184 | switch (Pred) { | ||||
4185 | case FCmpInst::FCMP_OGE: | ||||
4186 | case FCmpInst::FCMP_ULT: | ||||
4187 | // Positive or zero X >= 0.0 --> true | ||||
4188 | // Positive or zero X < 0.0 --> false | ||||
4189 | if ((FMF.noNaNs() || isKnownNeverNaN(LHS, Q.TLI)) && | ||||
4190 | CannotBeOrderedLessThanZero(LHS, Q.TLI)) | ||||
4191 | return Pred == FCmpInst::FCMP_OGE ? getTrue(RetTy) : getFalse(RetTy); | ||||
4192 | break; | ||||
4193 | case FCmpInst::FCMP_UGE: | ||||
4194 | case FCmpInst::FCMP_OLT: | ||||
4195 | // Positive or zero or nan X >= 0.0 --> true | ||||
4196 | // Positive or zero or nan X < 0.0 --> false | ||||
4197 | if (CannotBeOrderedLessThanZero(LHS, Q.TLI)) | ||||
4198 | return Pred == FCmpInst::FCMP_UGE ? getTrue(RetTy) : getFalse(RetTy); | ||||
4199 | break; | ||||
4200 | default: | ||||
4201 | break; | ||||
4202 | } | ||||
4203 | } | ||||
4204 | |||||
4205 | // If the comparison is with the result of a select instruction, check whether | ||||
4206 | // comparing with either branch of the select always yields the same value. | ||||
4207 | if (isa<SelectInst>(LHS) || isa<SelectInst>(RHS)) | ||||
4208 | if (Value *V = threadCmpOverSelect(Pred, LHS, RHS, Q, MaxRecurse)) | ||||
4209 | return V; | ||||
4210 | |||||
4211 | // If the comparison is with the result of a phi instruction, check whether | ||||
4212 | // doing the compare with each incoming phi value yields a common result. | ||||
4213 | if (isa<PHINode>(LHS) || isa<PHINode>(RHS)) | ||||
4214 | if (Value *V = threadCmpOverPHI(Pred, LHS, RHS, Q, MaxRecurse)) | ||||
4215 | return V; | ||||
4216 | |||||
4217 | return nullptr; | ||||
4218 | } | ||||
4219 | |||||
4220 | Value *llvm::simplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, | ||||
4221 | FastMathFlags FMF, const SimplifyQuery &Q) { | ||||
4222 | return ::simplifyFCmpInst(Predicate, LHS, RHS, FMF, Q, RecursionLimit); | ||||
4223 | } | ||||
4224 | |||||
4225 | static Value *simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp, | ||||
4226 | const SimplifyQuery &Q, | ||||
4227 | bool AllowRefinement, | ||||
4228 | unsigned MaxRecurse) { | ||||
4229 | // Trivial replacement. | ||||
4230 | if (V == Op) | ||||
4231 | return RepOp; | ||||
4232 | |||||
4233 | // We cannot replace a constant, and shouldn't even try. | ||||
4234 | if (isa<Constant>(Op)) | ||||
4235 | return nullptr; | ||||
4236 | |||||
4237 | auto *I = dyn_cast<Instruction>(V); | ||||
4238 | if (!I || !is_contained(I->operands(), Op)) | ||||
4239 | return nullptr; | ||||
4240 | |||||
4241 | if (Op->getType()->isVectorTy()) { | ||||
4242 | // For vector types, the simplification must hold per-lane, so forbid | ||||
4243 | // potentially cross-lane operations like shufflevector. | ||||
4244 | assert(I->getType()->isVectorTy() && "Vector type mismatch")(static_cast <bool> (I->getType()->isVectorTy() && "Vector type mismatch") ? void (0) : __assert_fail ("I->getType()->isVectorTy() && \"Vector type mismatch\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 4244, __extension__ __PRETTY_FUNCTION__)); | ||||
4245 | if (isa<ShuffleVectorInst>(I) || isa<CallBase>(I)) | ||||
4246 | return nullptr; | ||||
4247 | } | ||||
4248 | |||||
4249 | // Replace Op with RepOp in instruction operands. | ||||
4250 | SmallVector<Value *, 8> NewOps(I->getNumOperands()); | ||||
4251 | transform(I->operands(), NewOps.begin(), | ||||
4252 | [&](Value *V) { return V == Op ? RepOp : V; }); | ||||
4253 | |||||
4254 | if (!AllowRefinement) { | ||||
4255 | // General InstSimplify functions may refine the result, e.g. by returning | ||||
4256 | // a constant for a potentially poison value. To avoid this, implement only | ||||
4257 | // a few non-refining but profitable transforms here. | ||||
4258 | |||||
4259 | if (auto *BO = dyn_cast<BinaryOperator>(I)) { | ||||
4260 | unsigned Opcode = BO->getOpcode(); | ||||
4261 | // id op x -> x, x op id -> x | ||||
4262 | if (NewOps[0] == ConstantExpr::getBinOpIdentity(Opcode, I->getType())) | ||||
4263 | return NewOps[1]; | ||||
4264 | if (NewOps[1] == ConstantExpr::getBinOpIdentity(Opcode, I->getType(), | ||||
4265 | /* RHS */ true)) | ||||
4266 | return NewOps[0]; | ||||
4267 | |||||
4268 | // x & x -> x, x | x -> x | ||||
4269 | if ((Opcode == Instruction::And || Opcode == Instruction::Or) && | ||||
4270 | NewOps[0] == NewOps[1]) | ||||
4271 | return NewOps[0]; | ||||
4272 | |||||
4273 | // If we are substituting an absorber constant into a binop and extra | ||||
4274 | // poison can't leak if we remove the select -- because both operands of | ||||
4275 | // the binop are based on the same value -- then it may be safe to replace | ||||
4276 | // the value with the absorber constant. Examples: | ||||
4277 | // (Op == 0) ? 0 : (Op & -Op) --> Op & -Op | ||||
4278 | // (Op == 0) ? 0 : (Op * (binop Op, C)) --> Op * (binop Op, C) | ||||
4279 | // (Op == -1) ? -1 : (Op | (binop C, Op) --> Op | (binop C, Op) | ||||
4280 | if (RepOp == ConstantExpr::getBinOpAbsorber(Opcode, I->getType()) && | ||||
4281 | impliesPoison(BO, Op)) | ||||
4282 | return RepOp; | ||||
4283 | } | ||||
4284 | |||||
4285 | if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) { | ||||
4286 | // getelementptr x, 0 -> x | ||||
4287 | if (NewOps.size() == 2 && match(NewOps[1], m_Zero()) && | ||||
4288 | !GEP->isInBounds()) | ||||
4289 | return NewOps[0]; | ||||
4290 | } | ||||
4291 | } else if (MaxRecurse) { | ||||
4292 | // The simplification queries below may return the original value. Consider: | ||||
4293 | // %div = udiv i32 %arg, %arg2 | ||||
4294 | // %mul = mul nsw i32 %div, %arg2 | ||||
4295 | // %cmp = icmp eq i32 %mul, %arg | ||||
4296 | // %sel = select i1 %cmp, i32 %div, i32 undef | ||||
4297 | // Replacing %arg by %mul, %div becomes "udiv i32 %mul, %arg2", which | ||||
4298 | // simplifies back to %arg. This can only happen because %mul does not | ||||
4299 | // dominate %div. To ensure a consistent return value contract, we make sure | ||||
4300 | // that this case returns nullptr as well. | ||||
4301 | auto PreventSelfSimplify = [V](Value *Simplified) { | ||||
4302 | return Simplified != V ? Simplified : nullptr; | ||||
4303 | }; | ||||
4304 | |||||
4305 | if (auto *B = dyn_cast<BinaryOperator>(I)) | ||||
4306 | return PreventSelfSimplify(simplifyBinOp(B->getOpcode(), NewOps[0], | ||||
4307 | NewOps[1], Q, MaxRecurse - 1)); | ||||
4308 | |||||
4309 | if (CmpInst *C = dyn_cast<CmpInst>(I)) | ||||
4310 | return PreventSelfSimplify(simplifyCmpInst(C->getPredicate(), NewOps[0], | ||||
4311 | NewOps[1], Q, MaxRecurse - 1)); | ||||
4312 | |||||
4313 | if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) | ||||
4314 | return PreventSelfSimplify(simplifyGEPInst( | ||||
4315 | GEP->getSourceElementType(), NewOps[0], ArrayRef(NewOps).slice(1), | ||||
4316 | GEP->isInBounds(), Q, MaxRecurse - 1)); | ||||
4317 | |||||
4318 | if (isa<SelectInst>(I)) | ||||
4319 | return PreventSelfSimplify(simplifySelectInst( | ||||
4320 | NewOps[0], NewOps[1], NewOps[2], Q, MaxRecurse - 1)); | ||||
4321 | // TODO: We could hand off more cases to instsimplify here. | ||||
4322 | } | ||||
4323 | |||||
4324 | // If all operands are constant after substituting Op for RepOp then we can | ||||
4325 | // constant fold the instruction. | ||||
4326 | SmallVector<Constant *, 8> ConstOps; | ||||
4327 | for (Value *NewOp : NewOps) { | ||||
4328 | if (Constant *ConstOp = dyn_cast<Constant>(NewOp)) | ||||
4329 | ConstOps.push_back(ConstOp); | ||||
4330 | else | ||||
4331 | return nullptr; | ||||
4332 | } | ||||
4333 | |||||
4334 | // Consider: | ||||
4335 | // %cmp = icmp eq i32 %x, 2147483647 | ||||
4336 | // %add = add nsw i32 %x, 1 | ||||
4337 | // %sel = select i1 %cmp, i32 -2147483648, i32 %add | ||||
4338 | // | ||||
4339 | // We can't replace %sel with %add unless we strip away the flags (which | ||||
4340 | // will be done in InstCombine). | ||||
4341 | // TODO: This may be unsound, because it only catches some forms of | ||||
4342 | // refinement. | ||||
4343 | if (!AllowRefinement && canCreatePoison(cast<Operator>(I))) | ||||
4344 | return nullptr; | ||||
4345 | |||||
4346 | return ConstantFoldInstOperands(I, ConstOps, Q.DL, Q.TLI); | ||||
4347 | } | ||||
4348 | |||||
4349 | Value *llvm::simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp, | ||||
4350 | const SimplifyQuery &Q, | ||||
4351 | bool AllowRefinement) { | ||||
4352 | return ::simplifyWithOpReplaced(V, Op, RepOp, Q, AllowRefinement, | ||||
4353 | RecursionLimit); | ||||
4354 | } | ||||
4355 | |||||
4356 | /// Try to simplify a select instruction when its condition operand is an | ||||
4357 | /// integer comparison where one operand of the compare is a constant. | ||||
4358 | static Value *simplifySelectBitTest(Value *TrueVal, Value *FalseVal, Value *X, | ||||
4359 | const APInt *Y, bool TrueWhenUnset) { | ||||
4360 | const APInt *C; | ||||
4361 | |||||
4362 | // (X & Y) == 0 ? X & ~Y : X --> X | ||||
4363 | // (X & Y) != 0 ? X & ~Y : X --> X & ~Y | ||||
4364 | if (FalseVal == X && match(TrueVal, m_And(m_Specific(X), m_APInt(C))) && | ||||
4365 | *Y == ~*C) | ||||
4366 | return TrueWhenUnset ? FalseVal : TrueVal; | ||||
4367 | |||||
4368 | // (X & Y) == 0 ? X : X & ~Y --> X & ~Y | ||||
4369 | // (X & Y) != 0 ? X : X & ~Y --> X | ||||
4370 | if (TrueVal == X && match(FalseVal, m_And(m_Specific(X), m_APInt(C))) && | ||||
4371 | *Y == ~*C) | ||||
4372 | return TrueWhenUnset ? FalseVal : TrueVal; | ||||
4373 | |||||
4374 | if (Y->isPowerOf2()) { | ||||
4375 | // (X & Y) == 0 ? X | Y : X --> X | Y | ||||
4376 | // (X & Y) != 0 ? X | Y : X --> X | ||||
4377 | if (FalseVal == X && match(TrueVal, m_Or(m_Specific(X), m_APInt(C))) && | ||||
4378 | *Y == *C) | ||||
4379 | return TrueWhenUnset ? TrueVal : FalseVal; | ||||
4380 | |||||
4381 | // (X & Y) == 0 ? X : X | Y --> X | ||||
4382 | // (X & Y) != 0 ? X : X | Y --> X | Y | ||||
4383 | if (TrueVal == X && match(FalseVal, m_Or(m_Specific(X), m_APInt(C))) && | ||||
4384 | *Y == *C) | ||||
4385 | return TrueWhenUnset ? TrueVal : FalseVal; | ||||
4386 | } | ||||
4387 | |||||
4388 | return nullptr; | ||||
4389 | } | ||||
4390 | |||||
4391 | static Value *simplifyCmpSelOfMaxMin(Value *CmpLHS, Value *CmpRHS, | ||||
4392 | ICmpInst::Predicate Pred, Value *TVal, | ||||
4393 | Value *FVal) { | ||||
4394 | // Canonicalize common cmp+sel operand as CmpLHS. | ||||
4395 | if (CmpRHS == TVal || CmpRHS == FVal) { | ||||
4396 | std::swap(CmpLHS, CmpRHS); | ||||
4397 | Pred = ICmpInst::getSwappedPredicate(Pred); | ||||
4398 | } | ||||
4399 | |||||
4400 | // Canonicalize common cmp+sel operand as TVal. | ||||
4401 | if (CmpLHS == FVal) { | ||||
4402 | std::swap(TVal, FVal); | ||||
4403 | Pred = ICmpInst::getInversePredicate(Pred); | ||||
4404 | } | ||||
4405 | |||||
4406 | // A vector select may be shuffling together elements that are equivalent | ||||
4407 | // based on the max/min/select relationship. | ||||
4408 | Value *X = CmpLHS, *Y = CmpRHS; | ||||
4409 | bool PeekedThroughSelectShuffle = false; | ||||
4410 | auto *Shuf = dyn_cast<ShuffleVectorInst>(FVal); | ||||
4411 | if (Shuf && Shuf->isSelect()) { | ||||
4412 | if (Shuf->getOperand(0) == Y) | ||||
4413 | FVal = Shuf->getOperand(1); | ||||
4414 | else if (Shuf->getOperand(1) == Y) | ||||
4415 | FVal = Shuf->getOperand(0); | ||||
4416 | else | ||||
4417 | return nullptr; | ||||
4418 | PeekedThroughSelectShuffle = true; | ||||
4419 | } | ||||
4420 | |||||
4421 | // (X pred Y) ? X : max/min(X, Y) | ||||
4422 | auto *MMI = dyn_cast<MinMaxIntrinsic>(FVal); | ||||
4423 | if (!MMI || TVal != X || | ||||
4424 | !match(FVal, m_c_MaxOrMin(m_Specific(X), m_Specific(Y)))) | ||||
4425 | return nullptr; | ||||
4426 | |||||
4427 | // (X > Y) ? X : max(X, Y) --> max(X, Y) | ||||
4428 | // (X >= Y) ? X : max(X, Y) --> max(X, Y) | ||||
4429 | // (X < Y) ? X : min(X, Y) --> min(X, Y) | ||||
4430 | // (X <= Y) ? X : min(X, Y) --> min(X, Y) | ||||
4431 | // | ||||
4432 | // The equivalence allows a vector select (shuffle) of max/min and Y. Ex: | ||||
4433 | // (X > Y) ? X : (Z ? max(X, Y) : Y) | ||||
4434 | // If Z is true, this reduces as above, and if Z is false: | ||||
4435 | // (X > Y) ? X : Y --> max(X, Y) | ||||
4436 | ICmpInst::Predicate MMPred = MMI->getPredicate(); | ||||
4437 | if (MMPred == CmpInst::getStrictPredicate(Pred)) | ||||
4438 | return MMI; | ||||
4439 | |||||
4440 | // Other transforms are not valid with a shuffle. | ||||
4441 | if (PeekedThroughSelectShuffle) | ||||
4442 | return nullptr; | ||||
4443 | |||||
4444 | // (X == Y) ? X : max/min(X, Y) --> max/min(X, Y) | ||||
4445 | if (Pred == CmpInst::ICMP_EQ) | ||||
4446 | return MMI; | ||||
4447 | |||||
4448 | // (X != Y) ? X : max/min(X, Y) --> X | ||||
4449 | if (Pred == CmpInst::ICMP_NE) | ||||
4450 | return X; | ||||
4451 | |||||
4452 | // (X < Y) ? X : max(X, Y) --> X | ||||
4453 | // (X <= Y) ? X : max(X, Y) --> X | ||||
4454 | // (X > Y) ? X : min(X, Y) --> X | ||||
4455 | // (X >= Y) ? X : min(X, Y) --> X | ||||
4456 | ICmpInst::Predicate InvPred = CmpInst::getInversePredicate(Pred); | ||||
4457 | if (MMPred == CmpInst::getStrictPredicate(InvPred)) | ||||
4458 | return X; | ||||
4459 | |||||
4460 | return nullptr; | ||||
4461 | } | ||||
4462 | |||||
4463 | /// An alternative way to test if a bit is set or not uses sgt/slt instead of | ||||
4464 | /// eq/ne. | ||||
4465 | static Value *simplifySelectWithFakeICmpEq(Value *CmpLHS, Value *CmpRHS, | ||||
4466 | ICmpInst::Predicate Pred, | ||||
4467 | Value *TrueVal, Value *FalseVal) { | ||||
4468 | Value *X; | ||||
4469 | APInt Mask; | ||||
4470 | if (!decomposeBitTestICmp(CmpLHS, CmpRHS, Pred, X, Mask)) | ||||
4471 | return nullptr; | ||||
4472 | |||||
4473 | return simplifySelectBitTest(TrueVal, FalseVal, X, &Mask, | ||||
4474 | Pred == ICmpInst::ICMP_EQ); | ||||
4475 | } | ||||
4476 | |||||
4477 | /// Try to simplify a select instruction when its condition operand is an | ||||
4478 | /// integer comparison. | ||||
4479 | static Value *simplifySelectWithICmpCond(Value *CondVal, Value *TrueVal, | ||||
4480 | Value *FalseVal, | ||||
4481 | const SimplifyQuery &Q, | ||||
4482 | unsigned MaxRecurse) { | ||||
4483 | ICmpInst::Predicate Pred; | ||||
4484 | Value *CmpLHS, *CmpRHS; | ||||
4485 | if (!match(CondVal, m_ICmp(Pred, m_Value(CmpLHS), m_Value(CmpRHS)))) | ||||
4486 | return nullptr; | ||||
4487 | |||||
4488 | if (Value *V = simplifyCmpSelOfMaxMin(CmpLHS, CmpRHS, Pred, TrueVal, FalseVal)) | ||||
4489 | return V; | ||||
4490 | |||||
4491 | // Canonicalize ne to eq predicate. | ||||
4492 | if (Pred == ICmpInst::ICMP_NE) { | ||||
4493 | Pred = ICmpInst::ICMP_EQ; | ||||
4494 | std::swap(TrueVal, FalseVal); | ||||
4495 | } | ||||
4496 | |||||
4497 | // Check for integer min/max with a limit constant: | ||||
4498 | // X > MIN_INT ? X : MIN_INT --> X | ||||
4499 | // X < MAX_INT ? X : MAX_INT --> X | ||||
4500 | if (TrueVal->getType()->isIntOrIntVectorTy()) { | ||||
4501 | Value *X, *Y; | ||||
4502 | SelectPatternFlavor SPF = | ||||
4503 | matchDecomposedSelectPattern(cast<ICmpInst>(CondVal), TrueVal, FalseVal, | ||||
4504 | X, Y) | ||||
4505 | .Flavor; | ||||
4506 | if (SelectPatternResult::isMinOrMax(SPF) && Pred == getMinMaxPred(SPF)) { | ||||
4507 | APInt LimitC = getMinMaxLimit(getInverseMinMaxFlavor(SPF), | ||||
4508 | X->getType()->getScalarSizeInBits()); | ||||
4509 | if (match(Y, m_SpecificInt(LimitC))) | ||||
4510 | return X; | ||||
4511 | } | ||||
4512 | } | ||||
4513 | |||||
4514 | if (Pred == ICmpInst::ICMP_EQ && match(CmpRHS, m_Zero())) { | ||||
4515 | Value *X; | ||||
4516 | const APInt *Y; | ||||
4517 | if (match(CmpLHS, m_And(m_Value(X), m_APInt(Y)))) | ||||
4518 | if (Value *V = simplifySelectBitTest(TrueVal, FalseVal, X, Y, | ||||
4519 | /*TrueWhenUnset=*/true)) | ||||
4520 | return V; | ||||
4521 | |||||
4522 | // Test for a bogus zero-shift-guard-op around funnel-shift or rotate. | ||||
4523 | Value *ShAmt; | ||||
4524 | auto isFsh = m_CombineOr(m_FShl(m_Value(X), m_Value(), m_Value(ShAmt)), | ||||
4525 | m_FShr(m_Value(), m_Value(X), m_Value(ShAmt))); | ||||
4526 | // (ShAmt == 0) ? fshl(X, *, ShAmt) : X --> X | ||||
4527 | // (ShAmt == 0) ? fshr(*, X, ShAmt) : X --> X | ||||
4528 | if (match(TrueVal, isFsh) && FalseVal == X && CmpLHS == ShAmt) | ||||
4529 | return X; | ||||
4530 | |||||
4531 | // Test for a zero-shift-guard-op around rotates. These are used to | ||||
4532 | // avoid UB from oversized shifts in raw IR rotate patterns, but the | ||||
4533 | // intrinsics do not have that problem. | ||||
4534 | // We do not allow this transform for the general funnel shift case because | ||||
4535 | // that would not preserve the poison safety of the original code. | ||||
4536 | auto isRotate = | ||||
4537 | m_CombineOr(m_FShl(m_Value(X), m_Deferred(X), m_Value(ShAmt)), | ||||
4538 | m_FShr(m_Value(X), m_Deferred(X), m_Value(ShAmt))); | ||||
4539 | // (ShAmt == 0) ? X : fshl(X, X, ShAmt) --> fshl(X, X, ShAmt) | ||||
4540 | // (ShAmt == 0) ? X : fshr(X, X, ShAmt) --> fshr(X, X, ShAmt) | ||||
4541 | if (match(FalseVal, isRotate) && TrueVal == X && CmpLHS == ShAmt && | ||||
4542 | Pred == ICmpInst::ICMP_EQ) | ||||
4543 | return FalseVal; | ||||
4544 | |||||
4545 | // X == 0 ? abs(X) : -abs(X) --> -abs(X) | ||||
4546 | // X == 0 ? -abs(X) : abs(X) --> abs(X) | ||||
4547 | if (match(TrueVal, m_Intrinsic<Intrinsic::abs>(m_Specific(CmpLHS))) && | ||||
4548 | match(FalseVal, m_Neg(m_Intrinsic<Intrinsic::abs>(m_Specific(CmpLHS))))) | ||||
4549 | return FalseVal; | ||||
4550 | if (match(TrueVal, | ||||
4551 | m_Neg(m_Intrinsic<Intrinsic::abs>(m_Specific(CmpLHS)))) && | ||||
4552 | match(FalseVal, m_Intrinsic<Intrinsic::abs>(m_Specific(CmpLHS)))) | ||||
4553 | return FalseVal; | ||||
4554 | } | ||||
4555 | |||||
4556 | // Check for other compares that behave like bit test. | ||||
4557 | if (Value *V = | ||||
4558 | simplifySelectWithFakeICmpEq(CmpLHS, CmpRHS, Pred, TrueVal, FalseVal)) | ||||
4559 | return V; | ||||
4560 | |||||
4561 | // If we have a scalar equality comparison, then we know the value in one of | ||||
4562 | // the arms of the select. See if substituting this value into the arm and | ||||
4563 | // simplifying the result yields the same value as the other arm. | ||||
4564 | if (Pred == ICmpInst::ICMP_EQ) { | ||||
4565 | if (simplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, Q, | ||||
4566 | /* AllowRefinement */ false, | ||||
4567 | MaxRecurse) == TrueVal || | ||||
4568 | simplifyWithOpReplaced(FalseVal, CmpRHS, CmpLHS, Q, | ||||
4569 | /* AllowRefinement */ false, | ||||
4570 | MaxRecurse) == TrueVal) | ||||
4571 | return FalseVal; | ||||
4572 | if (simplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, Q, | ||||
4573 | /* AllowRefinement */ true, | ||||
4574 | MaxRecurse) == FalseVal || | ||||
4575 | simplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, Q, | ||||
4576 | /* AllowRefinement */ true, | ||||
4577 | MaxRecurse) == FalseVal) | ||||
4578 | return FalseVal; | ||||
4579 | } | ||||
4580 | |||||
4581 | return nullptr; | ||||
4582 | } | ||||
4583 | |||||
4584 | /// Try to simplify a select instruction when its condition operand is a | ||||
4585 | /// floating-point comparison. | ||||
4586 | static Value *simplifySelectWithFCmp(Value *Cond, Value *T, Value *F, | ||||
4587 | const SimplifyQuery &Q) { | ||||
4588 | FCmpInst::Predicate Pred; | ||||
4589 | if (!match(Cond, m_FCmp(Pred, m_Specific(T), m_Specific(F))) && | ||||
4590 | !match(Cond, m_FCmp(Pred, m_Specific(F), m_Specific(T)))) | ||||
4591 | return nullptr; | ||||
4592 | |||||
4593 | // This transform is safe if we do not have (do not care about) -0.0 or if | ||||
4594 | // at least one operand is known to not be -0.0. Otherwise, the select can | ||||
4595 | // change the sign of a zero operand. | ||||
4596 | bool HasNoSignedZeros = | ||||
4597 | Q.CxtI && isa<FPMathOperator>(Q.CxtI) && Q.CxtI->hasNoSignedZeros(); | ||||
4598 | const APFloat *C; | ||||
4599 | if (HasNoSignedZeros || (match(T, m_APFloat(C)) && C->isNonZero()) || | ||||
4600 | (match(F, m_APFloat(C)) && C->isNonZero())) { | ||||
4601 | // (T == F) ? T : F --> F | ||||
4602 | // (F == T) ? T : F --> F | ||||
4603 | if (Pred == FCmpInst::FCMP_OEQ) | ||||
4604 | return F; | ||||
4605 | |||||
4606 | // (T != F) ? T : F --> T | ||||
4607 | // (F != T) ? T : F --> T | ||||
4608 | if (Pred == FCmpInst::FCMP_UNE) | ||||
4609 | return T; | ||||
4610 | } | ||||
4611 | |||||
4612 | return nullptr; | ||||
4613 | } | ||||
4614 | |||||
4615 | /// Given operands for a SelectInst, see if we can fold the result. | ||||
4616 | /// If not, this returns null. | ||||
4617 | static Value *simplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, | ||||
4618 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
4619 | if (auto *CondC = dyn_cast<Constant>(Cond)) { | ||||
4620 | if (auto *TrueC = dyn_cast<Constant>(TrueVal)) | ||||
4621 | if (auto *FalseC = dyn_cast<Constant>(FalseVal)) | ||||
4622 | return ConstantFoldSelectInstruction(CondC, TrueC, FalseC); | ||||
4623 | |||||
4624 | // select poison, X, Y -> poison | ||||
4625 | if (isa<PoisonValue>(CondC)) | ||||
4626 | return PoisonValue::get(TrueVal->getType()); | ||||
4627 | |||||
4628 | // select undef, X, Y -> X or Y | ||||
4629 | if (Q.isUndefValue(CondC)) | ||||
4630 | return isa<Constant>(FalseVal) ? FalseVal : TrueVal; | ||||
4631 | |||||
4632 | // select true, X, Y --> X | ||||
4633 | // select false, X, Y --> Y | ||||
4634 | // For vectors, allow undef/poison elements in the condition to match the | ||||
4635 | // defined elements, so we can eliminate the select. | ||||
4636 | if (match(CondC, m_One())) | ||||
4637 | return TrueVal; | ||||
4638 | if (match(CondC, m_Zero())) | ||||
4639 | return FalseVal; | ||||
4640 | } | ||||
4641 | |||||
4642 | assert(Cond->getType()->isIntOrIntVectorTy(1) &&(static_cast <bool> (Cond->getType()->isIntOrIntVectorTy (1) && "Select must have bool or bool vector condition" ) ? void (0) : __assert_fail ("Cond->getType()->isIntOrIntVectorTy(1) && \"Select must have bool or bool vector condition\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 4643, __extension__ __PRETTY_FUNCTION__)) | ||||
4643 | "Select must have bool or bool vector condition")(static_cast <bool> (Cond->getType()->isIntOrIntVectorTy (1) && "Select must have bool or bool vector condition" ) ? void (0) : __assert_fail ("Cond->getType()->isIntOrIntVectorTy(1) && \"Select must have bool or bool vector condition\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 4643, __extension__ __PRETTY_FUNCTION__)); | ||||
4644 | assert(TrueVal->getType() == FalseVal->getType() &&(static_cast <bool> (TrueVal->getType() == FalseVal-> getType() && "Select must have same types for true/false ops" ) ? void (0) : __assert_fail ("TrueVal->getType() == FalseVal->getType() && \"Select must have same types for true/false ops\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 4645, __extension__ __PRETTY_FUNCTION__)) | ||||
4645 | "Select must have same types for true/false ops")(static_cast <bool> (TrueVal->getType() == FalseVal-> getType() && "Select must have same types for true/false ops" ) ? void (0) : __assert_fail ("TrueVal->getType() == FalseVal->getType() && \"Select must have same types for true/false ops\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 4645, __extension__ __PRETTY_FUNCTION__)); | ||||
4646 | |||||
4647 | if (Cond->getType() == TrueVal->getType()) { | ||||
4648 | // select i1 Cond, i1 true, i1 false --> i1 Cond | ||||
4649 | if (match(TrueVal, m_One()) && match(FalseVal, m_ZeroInt())) | ||||
4650 | return Cond; | ||||
4651 | |||||
4652 | // (X && Y) ? X : Y --> Y (commuted 2 ways) | ||||
4653 | if (match(Cond, m_c_LogicalAnd(m_Specific(TrueVal), m_Specific(FalseVal)))) | ||||
4654 | return FalseVal; | ||||
4655 | |||||
4656 | // (X || Y) ? X : Y --> X (commuted 2 ways) | ||||
4657 | if (match(Cond, m_c_LogicalOr(m_Specific(TrueVal), m_Specific(FalseVal)))) | ||||
4658 | return TrueVal; | ||||
4659 | |||||
4660 | // (X || Y) ? false : X --> false (commuted 2 ways) | ||||
4661 | if (match(Cond, m_c_LogicalOr(m_Specific(FalseVal), m_Value())) && | ||||
4662 | match(TrueVal, m_ZeroInt())) | ||||
4663 | return ConstantInt::getFalse(Cond->getType()); | ||||
4664 | |||||
4665 | // Match patterns that end in logical-and. | ||||
4666 | if (match(FalseVal, m_ZeroInt())) { | ||||
4667 | // !(X || Y) && X --> false (commuted 2 ways) | ||||
4668 | if (match(Cond, m_Not(m_c_LogicalOr(m_Specific(TrueVal), m_Value())))) | ||||
4669 | return ConstantInt::getFalse(Cond->getType()); | ||||
4670 | // X && !(X || Y) --> false (commuted 2 ways) | ||||
4671 | if (match(TrueVal, m_Not(m_c_LogicalOr(m_Specific(Cond), m_Value())))) | ||||
4672 | return ConstantInt::getFalse(Cond->getType()); | ||||
4673 | |||||
4674 | // (X || Y) && Y --> Y (commuted 2 ways) | ||||
4675 | if (match(Cond, m_c_LogicalOr(m_Specific(TrueVal), m_Value()))) | ||||
4676 | return TrueVal; | ||||
4677 | // Y && (X || Y) --> Y (commuted 2 ways) | ||||
4678 | if (match(TrueVal, m_c_LogicalOr(m_Specific(Cond), m_Value()))) | ||||
4679 | return Cond; | ||||
4680 | |||||
4681 | // (X || Y) && (X || !Y) --> X (commuted 8 ways) | ||||
4682 | Value *X, *Y; | ||||
4683 | if (match(Cond, m_c_LogicalOr(m_Value(X), m_Not(m_Value(Y)))) && | ||||
4684 | match(TrueVal, m_c_LogicalOr(m_Specific(X), m_Specific(Y)))) | ||||
4685 | return X; | ||||
4686 | if (match(TrueVal, m_c_LogicalOr(m_Value(X), m_Not(m_Value(Y)))) && | ||||
4687 | match(Cond, m_c_LogicalOr(m_Specific(X), m_Specific(Y)))) | ||||
4688 | return X; | ||||
4689 | } | ||||
4690 | |||||
4691 | // Match patterns that end in logical-or. | ||||
4692 | if (match(TrueVal, m_One())) { | ||||
4693 | // !(X && Y) || X --> true (commuted 2 ways) | ||||
4694 | if (match(Cond, m_Not(m_c_LogicalAnd(m_Specific(FalseVal), m_Value())))) | ||||
4695 | return ConstantInt::getTrue(Cond->getType()); | ||||
4696 | // X || !(X && Y) --> true (commuted 2 ways) | ||||
4697 | if (match(FalseVal, m_Not(m_c_LogicalAnd(m_Specific(Cond), m_Value())))) | ||||
4698 | return ConstantInt::getTrue(Cond->getType()); | ||||
4699 | |||||
4700 | // (X && Y) || Y --> Y (commuted 2 ways) | ||||
4701 | if (match(Cond, m_c_LogicalAnd(m_Specific(FalseVal), m_Value()))) | ||||
4702 | return FalseVal; | ||||
4703 | // Y || (X && Y) --> Y (commuted 2 ways) | ||||
4704 | if (match(FalseVal, m_c_LogicalAnd(m_Specific(Cond), m_Value()))) | ||||
4705 | return Cond; | ||||
4706 | } | ||||
4707 | } | ||||
4708 | |||||
4709 | // select ?, X, X -> X | ||||
4710 | if (TrueVal == FalseVal) | ||||
4711 | return TrueVal; | ||||
4712 | |||||
4713 | if (Cond == TrueVal) { | ||||
4714 | // select i1 X, i1 X, i1 false --> X (logical-and) | ||||
4715 | if (match(FalseVal, m_ZeroInt())) | ||||
4716 | return Cond; | ||||
4717 | // select i1 X, i1 X, i1 true --> true | ||||
4718 | if (match(FalseVal, m_One())) | ||||
4719 | return ConstantInt::getTrue(Cond->getType()); | ||||
4720 | } | ||||
4721 | if (Cond == FalseVal) { | ||||
4722 | // select i1 X, i1 true, i1 X --> X (logical-or) | ||||
4723 | if (match(TrueVal, m_One())) | ||||
4724 | return Cond; | ||||
4725 | // select i1 X, i1 false, i1 X --> false | ||||
4726 | if (match(TrueVal, m_ZeroInt())) | ||||
4727 | return ConstantInt::getFalse(Cond->getType()); | ||||
4728 | } | ||||
4729 | |||||
4730 | // If the true or false value is poison, we can fold to the other value. | ||||
4731 | // If the true or false value is undef, we can fold to the other value as | ||||
4732 | // long as the other value isn't poison. | ||||
4733 | // select ?, poison, X -> X | ||||
4734 | // select ?, undef, X -> X | ||||
4735 | if (isa<PoisonValue>(TrueVal) || | ||||
4736 | (Q.isUndefValue(TrueVal) && | ||||
4737 | isGuaranteedNotToBePoison(FalseVal, Q.AC, Q.CxtI, Q.DT))) | ||||
4738 | return FalseVal; | ||||
4739 | // select ?, X, poison -> X | ||||
4740 | // select ?, X, undef -> X | ||||
4741 | if (isa<PoisonValue>(FalseVal) || | ||||
4742 | (Q.isUndefValue(FalseVal) && | ||||
4743 | isGuaranteedNotToBePoison(TrueVal, Q.AC, Q.CxtI, Q.DT))) | ||||
4744 | return TrueVal; | ||||
4745 | |||||
4746 | // Deal with partial undef vector constants: select ?, VecC, VecC' --> VecC'' | ||||
4747 | Constant *TrueC, *FalseC; | ||||
4748 | if (isa<FixedVectorType>(TrueVal->getType()) && | ||||
4749 | match(TrueVal, m_Constant(TrueC)) && | ||||
4750 | match(FalseVal, m_Constant(FalseC))) { | ||||
4751 | unsigned NumElts = | ||||
4752 | cast<FixedVectorType>(TrueC->getType())->getNumElements(); | ||||
4753 | SmallVector<Constant *, 16> NewC; | ||||
4754 | for (unsigned i = 0; i != NumElts; ++i) { | ||||
4755 | // Bail out on incomplete vector constants. | ||||
4756 | Constant *TEltC = TrueC->getAggregateElement(i); | ||||
4757 | Constant *FEltC = FalseC->getAggregateElement(i); | ||||
4758 | if (!TEltC || !FEltC) | ||||
4759 | break; | ||||
4760 | |||||
4761 | // If the elements match (undef or not), that value is the result. If only | ||||
4762 | // one element is undef, choose the defined element as the safe result. | ||||
4763 | if (TEltC == FEltC) | ||||
4764 | NewC.push_back(TEltC); | ||||
4765 | else if (isa<PoisonValue>(TEltC) || | ||||
4766 | (Q.isUndefValue(TEltC) && isGuaranteedNotToBePoison(FEltC))) | ||||
4767 | NewC.push_back(FEltC); | ||||
4768 | else if (isa<PoisonValue>(FEltC) || | ||||
4769 | (Q.isUndefValue(FEltC) && isGuaranteedNotToBePoison(TEltC))) | ||||
4770 | NewC.push_back(TEltC); | ||||
4771 | else | ||||
4772 | break; | ||||
4773 | } | ||||
4774 | if (NewC.size() == NumElts) | ||||
4775 | return ConstantVector::get(NewC); | ||||
4776 | } | ||||
4777 | |||||
4778 | if (Value *V = | ||||
4779 | simplifySelectWithICmpCond(Cond, TrueVal, FalseVal, Q, MaxRecurse)) | ||||
4780 | return V; | ||||
4781 | |||||
4782 | if (Value *V = simplifySelectWithFCmp(Cond, TrueVal, FalseVal, Q)) | ||||
4783 | return V; | ||||
4784 | |||||
4785 | if (Value *V = foldSelectWithBinaryOp(Cond, TrueVal, FalseVal)) | ||||
4786 | return V; | ||||
4787 | |||||
4788 | std::optional<bool> Imp = isImpliedByDomCondition(Cond, Q.CxtI, Q.DL); | ||||
4789 | if (Imp) | ||||
4790 | return *Imp ? TrueVal : FalseVal; | ||||
4791 | |||||
4792 | return nullptr; | ||||
4793 | } | ||||
4794 | |||||
4795 | Value *llvm::simplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, | ||||
4796 | const SimplifyQuery &Q) { | ||||
4797 | return ::simplifySelectInst(Cond, TrueVal, FalseVal, Q, RecursionLimit); | ||||
4798 | } | ||||
4799 | |||||
4800 | /// Given operands for an GetElementPtrInst, see if we can fold the result. | ||||
4801 | /// If not, this returns null. | ||||
4802 | static Value *simplifyGEPInst(Type *SrcTy, Value *Ptr, | ||||
4803 | ArrayRef<Value *> Indices, bool InBounds, | ||||
4804 | const SimplifyQuery &Q, unsigned) { | ||||
4805 | // The type of the GEP pointer operand. | ||||
4806 | unsigned AS = | ||||
4807 | cast<PointerType>(Ptr->getType()->getScalarType())->getAddressSpace(); | ||||
4808 | |||||
4809 | // getelementptr P -> P. | ||||
4810 | if (Indices.empty()) | ||||
4811 | return Ptr; | ||||
4812 | |||||
4813 | // Compute the (pointer) type returned by the GEP instruction. | ||||
4814 | Type *LastType = GetElementPtrInst::getIndexedType(SrcTy, Indices); | ||||
4815 | Type *GEPTy = PointerType::get(LastType, AS); | ||||
4816 | if (VectorType *VT = dyn_cast<VectorType>(Ptr->getType())) | ||||
4817 | GEPTy = VectorType::get(GEPTy, VT->getElementCount()); | ||||
4818 | else { | ||||
4819 | for (Value *Op : Indices) { | ||||
4820 | // If one of the operands is a vector, the result type is a vector of | ||||
4821 | // pointers. All vector operands must have the same number of elements. | ||||
4822 | if (VectorType *VT = dyn_cast<VectorType>(Op->getType())) { | ||||
4823 | GEPTy = VectorType::get(GEPTy, VT->getElementCount()); | ||||
4824 | break; | ||||
4825 | } | ||||
4826 | } | ||||
4827 | } | ||||
4828 | |||||
4829 | // For opaque pointers an all-zero GEP is a no-op. For typed pointers, | ||||
4830 | // it may be equivalent to a bitcast. | ||||
4831 | if (Ptr->getType()->getScalarType()->isOpaquePointerTy() && | ||||
4832 | Ptr->getType() == GEPTy && | ||||
4833 | all_of(Indices, [](const auto *V) { return match(V, m_Zero()); })) | ||||
4834 | return Ptr; | ||||
4835 | |||||
4836 | // getelementptr poison, idx -> poison | ||||
4837 | // getelementptr baseptr, poison -> poison | ||||
4838 | if (isa<PoisonValue>(Ptr) || | ||||
4839 | any_of(Indices, [](const auto *V) { return isa<PoisonValue>(V); })) | ||||
4840 | return PoisonValue::get(GEPTy); | ||||
4841 | |||||
4842 | if (Q.isUndefValue(Ptr)) | ||||
4843 | // If inbounds, we can choose an out-of-bounds pointer as a base pointer. | ||||
4844 | return InBounds ? PoisonValue::get(GEPTy) : UndefValue::get(GEPTy); | ||||
4845 | |||||
4846 | bool IsScalableVec = | ||||
4847 | isa<ScalableVectorType>(SrcTy) || any_of(Indices, [](const Value *V) { | ||||
4848 | return isa<ScalableVectorType>(V->getType()); | ||||
4849 | }); | ||||
4850 | |||||
4851 | if (Indices.size() == 1) { | ||||
4852 | // getelementptr P, 0 -> P. | ||||
4853 | if (match(Indices[0], m_Zero()) && Ptr->getType() == GEPTy) | ||||
4854 | return Ptr; | ||||
4855 | |||||
4856 | Type *Ty = SrcTy; | ||||
4857 | if (!IsScalableVec && Ty->isSized()) { | ||||
4858 | Value *P; | ||||
4859 | uint64_t C; | ||||
4860 | uint64_t TyAllocSize = Q.DL.getTypeAllocSize(Ty); | ||||
4861 | // getelementptr P, N -> P if P points to a type of zero size. | ||||
4862 | if (TyAllocSize == 0 && Ptr->getType() == GEPTy) | ||||
4863 | return Ptr; | ||||
4864 | |||||
4865 | // The following transforms are only safe if the ptrtoint cast | ||||
4866 | // doesn't truncate the pointers. | ||||
4867 | if (Indices[0]->getType()->getScalarSizeInBits() == | ||||
4868 | Q.DL.getPointerSizeInBits(AS)) { | ||||
4869 | auto CanSimplify = [GEPTy, &P, Ptr]() -> bool { | ||||
4870 | return P->getType() == GEPTy && | ||||
4871 | getUnderlyingObject(P) == getUnderlyingObject(Ptr); | ||||
4872 | }; | ||||
4873 | // getelementptr V, (sub P, V) -> P if P points to a type of size 1. | ||||
4874 | if (TyAllocSize == 1 && | ||||
4875 | match(Indices[0], | ||||
4876 | m_Sub(m_PtrToInt(m_Value(P)), m_PtrToInt(m_Specific(Ptr)))) && | ||||
4877 | CanSimplify()) | ||||
4878 | return P; | ||||
4879 | |||||
4880 | // getelementptr V, (ashr (sub P, V), C) -> P if P points to a type of | ||||
4881 | // size 1 << C. | ||||
4882 | if (match(Indices[0], m_AShr(m_Sub(m_PtrToInt(m_Value(P)), | ||||
4883 | m_PtrToInt(m_Specific(Ptr))), | ||||
4884 | m_ConstantInt(C))) && | ||||
4885 | TyAllocSize == 1ULL << C && CanSimplify()) | ||||
4886 | return P; | ||||
4887 | |||||
4888 | // getelementptr V, (sdiv (sub P, V), C) -> P if P points to a type of | ||||
4889 | // size C. | ||||
4890 | if (match(Indices[0], m_SDiv(m_Sub(m_PtrToInt(m_Value(P)), | ||||
4891 | m_PtrToInt(m_Specific(Ptr))), | ||||
4892 | m_SpecificInt(TyAllocSize))) && | ||||
4893 | CanSimplify()) | ||||
4894 | return P; | ||||
4895 | } | ||||
4896 | } | ||||
4897 | } | ||||
4898 | |||||
4899 | if (!IsScalableVec && Q.DL.getTypeAllocSize(LastType) == 1 && | ||||
4900 | all_of(Indices.drop_back(1), | ||||
4901 | [](Value *Idx) { return match(Idx, m_Zero()); })) { | ||||
4902 | unsigned IdxWidth = | ||||
4903 | Q.DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()); | ||||
4904 | if (Q.DL.getTypeSizeInBits(Indices.back()->getType()) == IdxWidth) { | ||||
4905 | APInt BasePtrOffset(IdxWidth, 0); | ||||
4906 | Value *StrippedBasePtr = | ||||
4907 | Ptr->stripAndAccumulateInBoundsConstantOffsets(Q.DL, BasePtrOffset); | ||||
4908 | |||||
4909 | // Avoid creating inttoptr of zero here: While LLVMs treatment of | ||||
4910 | // inttoptr is generally conservative, this particular case is folded to | ||||
4911 | // a null pointer, which will have incorrect provenance. | ||||
4912 | |||||
4913 | // gep (gep V, C), (sub 0, V) -> C | ||||
4914 | if (match(Indices.back(), | ||||
4915 | m_Sub(m_Zero(), m_PtrToInt(m_Specific(StrippedBasePtr)))) && | ||||
4916 | !BasePtrOffset.isZero()) { | ||||
4917 | auto *CI = ConstantInt::get(GEPTy->getContext(), BasePtrOffset); | ||||
4918 | return ConstantExpr::getIntToPtr(CI, GEPTy); | ||||
4919 | } | ||||
4920 | // gep (gep V, C), (xor V, -1) -> C-1 | ||||
4921 | if (match(Indices.back(), | ||||
4922 | m_Xor(m_PtrToInt(m_Specific(StrippedBasePtr)), m_AllOnes())) && | ||||
4923 | !BasePtrOffset.isOne()) { | ||||
4924 | auto *CI = ConstantInt::get(GEPTy->getContext(), BasePtrOffset - 1); | ||||
4925 | return ConstantExpr::getIntToPtr(CI, GEPTy); | ||||
4926 | } | ||||
4927 | } | ||||
4928 | } | ||||
4929 | |||||
4930 | // Check to see if this is constant foldable. | ||||
4931 | if (!isa<Constant>(Ptr) || | ||||
4932 | !all_of(Indices, [](Value *V) { return isa<Constant>(V); })) | ||||
4933 | return nullptr; | ||||
4934 | |||||
4935 | if (!ConstantExpr::isSupportedGetElementPtr(SrcTy)) | ||||
4936 | return ConstantFoldGetElementPtr(SrcTy, cast<Constant>(Ptr), InBounds, | ||||
4937 | std::nullopt, Indices); | ||||
4938 | |||||
4939 | auto *CE = ConstantExpr::getGetElementPtr(SrcTy, cast<Constant>(Ptr), Indices, | ||||
4940 | InBounds); | ||||
4941 | return ConstantFoldConstant(CE, Q.DL); | ||||
4942 | } | ||||
4943 | |||||
4944 | Value *llvm::simplifyGEPInst(Type *SrcTy, Value *Ptr, ArrayRef<Value *> Indices, | ||||
4945 | bool InBounds, const SimplifyQuery &Q) { | ||||
4946 | return ::simplifyGEPInst(SrcTy, Ptr, Indices, InBounds, Q, RecursionLimit); | ||||
4947 | } | ||||
4948 | |||||
4949 | /// Given operands for an InsertValueInst, see if we can fold the result. | ||||
4950 | /// If not, this returns null. | ||||
4951 | static Value *simplifyInsertValueInst(Value *Agg, Value *Val, | ||||
4952 | ArrayRef<unsigned> Idxs, | ||||
4953 | const SimplifyQuery &Q, unsigned) { | ||||
4954 | if (Constant *CAgg = dyn_cast<Constant>(Agg)) | ||||
4955 | if (Constant *CVal = dyn_cast<Constant>(Val)) | ||||
4956 | return ConstantFoldInsertValueInstruction(CAgg, CVal, Idxs); | ||||
4957 | |||||
4958 | // insertvalue x, poison, n -> x | ||||
4959 | // insertvalue x, undef, n -> x if x cannot be poison | ||||
4960 | if (isa<PoisonValue>(Val) || | ||||
4961 | (Q.isUndefValue(Val) && isGuaranteedNotToBePoison(Agg))) | ||||
4962 | return Agg; | ||||
4963 | |||||
4964 | // insertvalue x, (extractvalue y, n), n | ||||
4965 | if (ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Val)) | ||||
4966 | if (EV->getAggregateOperand()->getType() == Agg->getType() && | ||||
4967 | EV->getIndices() == Idxs) { | ||||
4968 | // insertvalue poison, (extractvalue y, n), n -> y | ||||
4969 | // insertvalue undef, (extractvalue y, n), n -> y if y cannot be poison | ||||
4970 | if (isa<PoisonValue>(Agg) || | ||||
4971 | (Q.isUndefValue(Agg) && | ||||
4972 | isGuaranteedNotToBePoison(EV->getAggregateOperand()))) | ||||
4973 | return EV->getAggregateOperand(); | ||||
4974 | |||||
4975 | // insertvalue y, (extractvalue y, n), n -> y | ||||
4976 | if (Agg == EV->getAggregateOperand()) | ||||
4977 | return Agg; | ||||
4978 | } | ||||
4979 | |||||
4980 | return nullptr; | ||||
4981 | } | ||||
4982 | |||||
4983 | Value *llvm::simplifyInsertValueInst(Value *Agg, Value *Val, | ||||
4984 | ArrayRef<unsigned> Idxs, | ||||
4985 | const SimplifyQuery &Q) { | ||||
4986 | return ::simplifyInsertValueInst(Agg, Val, Idxs, Q, RecursionLimit); | ||||
4987 | } | ||||
4988 | |||||
4989 | Value *llvm::simplifyInsertElementInst(Value *Vec, Value *Val, Value *Idx, | ||||
4990 | const SimplifyQuery &Q) { | ||||
4991 | // Try to constant fold. | ||||
4992 | auto *VecC = dyn_cast<Constant>(Vec); | ||||
4993 | auto *ValC = dyn_cast<Constant>(Val); | ||||
4994 | auto *IdxC = dyn_cast<Constant>(Idx); | ||||
4995 | if (VecC && ValC && IdxC) | ||||
4996 | return ConstantExpr::getInsertElement(VecC, ValC, IdxC); | ||||
4997 | |||||
4998 | // For fixed-length vector, fold into poison if index is out of bounds. | ||||
4999 | if (auto *CI = dyn_cast<ConstantInt>(Idx)) { | ||||
5000 | if (isa<FixedVectorType>(Vec->getType()) && | ||||
5001 | CI->uge(cast<FixedVectorType>(Vec->getType())->getNumElements())) | ||||
5002 | return PoisonValue::get(Vec->getType()); | ||||
5003 | } | ||||
5004 | |||||
5005 | // If index is undef, it might be out of bounds (see above case) | ||||
5006 | if (Q.isUndefValue(Idx)) | ||||
5007 | return PoisonValue::get(Vec->getType()); | ||||
5008 | |||||
5009 | // If the scalar is poison, or it is undef and there is no risk of | ||||
5010 | // propagating poison from the vector value, simplify to the vector value. | ||||
5011 | if (isa<PoisonValue>(Val) || | ||||
5012 | (Q.isUndefValue(Val) && isGuaranteedNotToBePoison(Vec))) | ||||
5013 | return Vec; | ||||
5014 | |||||
5015 | // If we are extracting a value from a vector, then inserting it into the same | ||||
5016 | // place, that's the input vector: | ||||
5017 | // insertelt Vec, (extractelt Vec, Idx), Idx --> Vec | ||||
5018 | if (match(Val, m_ExtractElt(m_Specific(Vec), m_Specific(Idx)))) | ||||
5019 | return Vec; | ||||
5020 | |||||
5021 | return nullptr; | ||||
5022 | } | ||||
5023 | |||||
5024 | /// Given operands for an ExtractValueInst, see if we can fold the result. | ||||
5025 | /// If not, this returns null. | ||||
5026 | static Value *simplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs, | ||||
5027 | const SimplifyQuery &, unsigned) { | ||||
5028 | if (auto *CAgg = dyn_cast<Constant>(Agg)) | ||||
5029 | return ConstantFoldExtractValueInstruction(CAgg, Idxs); | ||||
5030 | |||||
5031 | // extractvalue x, (insertvalue y, elt, n), n -> elt | ||||
5032 | unsigned NumIdxs = Idxs.size(); | ||||
5033 | for (auto *IVI = dyn_cast<InsertValueInst>(Agg); IVI != nullptr; | ||||
5034 | IVI = dyn_cast<InsertValueInst>(IVI->getAggregateOperand())) { | ||||
5035 | ArrayRef<unsigned> InsertValueIdxs = IVI->getIndices(); | ||||
5036 | unsigned NumInsertValueIdxs = InsertValueIdxs.size(); | ||||
5037 | unsigned NumCommonIdxs = std::min(NumInsertValueIdxs, NumIdxs); | ||||
5038 | if (InsertValueIdxs.slice(0, NumCommonIdxs) == | ||||
5039 | Idxs.slice(0, NumCommonIdxs)) { | ||||
5040 | if (NumIdxs == NumInsertValueIdxs) | ||||
5041 | return IVI->getInsertedValueOperand(); | ||||
5042 | break; | ||||
5043 | } | ||||
5044 | } | ||||
5045 | |||||
5046 | return nullptr; | ||||
5047 | } | ||||
5048 | |||||
5049 | Value *llvm::simplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs, | ||||
5050 | const SimplifyQuery &Q) { | ||||
5051 | return ::simplifyExtractValueInst(Agg, Idxs, Q, RecursionLimit); | ||||
5052 | } | ||||
5053 | |||||
5054 | /// Given operands for an ExtractElementInst, see if we can fold the result. | ||||
5055 | /// If not, this returns null. | ||||
5056 | static Value *simplifyExtractElementInst(Value *Vec, Value *Idx, | ||||
5057 | const SimplifyQuery &Q, unsigned) { | ||||
5058 | auto *VecVTy = cast<VectorType>(Vec->getType()); | ||||
5059 | if (auto *CVec = dyn_cast<Constant>(Vec)) { | ||||
5060 | if (auto *CIdx = dyn_cast<Constant>(Idx)) | ||||
5061 | return ConstantExpr::getExtractElement(CVec, CIdx); | ||||
5062 | |||||
5063 | if (Q.isUndefValue(Vec)) | ||||
5064 | return UndefValue::get(VecVTy->getElementType()); | ||||
5065 | } | ||||
5066 | |||||
5067 | // An undef extract index can be arbitrarily chosen to be an out-of-range | ||||
5068 | // index value, which would result in the instruction being poison. | ||||
5069 | if (Q.isUndefValue(Idx)) | ||||
5070 | return PoisonValue::get(VecVTy->getElementType()); | ||||
5071 | |||||
5072 | // If extracting a specified index from the vector, see if we can recursively | ||||
5073 | // find a previously computed scalar that was inserted into the vector. | ||||
5074 | if (auto *IdxC = dyn_cast<ConstantInt>(Idx)) { | ||||
5075 | // For fixed-length vector, fold into undef if index is out of bounds. | ||||
5076 | unsigned MinNumElts = VecVTy->getElementCount().getKnownMinValue(); | ||||
5077 | if (isa<FixedVectorType>(VecVTy) && IdxC->getValue().uge(MinNumElts)) | ||||
5078 | return PoisonValue::get(VecVTy->getElementType()); | ||||
5079 | // Handle case where an element is extracted from a splat. | ||||
5080 | if (IdxC->getValue().ult(MinNumElts)) | ||||
5081 | if (auto *Splat = getSplatValue(Vec)) | ||||
5082 | return Splat; | ||||
5083 | if (Value *Elt = findScalarElement(Vec, IdxC->getZExtValue())) | ||||
5084 | return Elt; | ||||
5085 | } else { | ||||
5086 | // extractelt x, (insertelt y, elt, n), n -> elt | ||||
5087 | // If the possibly-variable indices are trivially known to be equal | ||||
5088 | // (because they are the same operand) then use the value that was | ||||
5089 | // inserted directly. | ||||
5090 | auto *IE = dyn_cast<InsertElementInst>(Vec); | ||||
5091 | if (IE && IE->getOperand(2) == Idx) | ||||
5092 | return IE->getOperand(1); | ||||
5093 | |||||
5094 | // The index is not relevant if our vector is a splat. | ||||
5095 | if (Value *Splat = getSplatValue(Vec)) | ||||
5096 | return Splat; | ||||
5097 | } | ||||
5098 | return nullptr; | ||||
5099 | } | ||||
5100 | |||||
5101 | Value *llvm::simplifyExtractElementInst(Value *Vec, Value *Idx, | ||||
5102 | const SimplifyQuery &Q) { | ||||
5103 | return ::simplifyExtractElementInst(Vec, Idx, Q, RecursionLimit); | ||||
5104 | } | ||||
5105 | |||||
5106 | /// See if we can fold the given phi. If not, returns null. | ||||
5107 | static Value *simplifyPHINode(PHINode *PN, ArrayRef<Value *> IncomingValues, | ||||
5108 | const SimplifyQuery &Q) { | ||||
5109 | // WARNING: no matter how worthwhile it may seem, we can not perform PHI CSE | ||||
5110 | // here, because the PHI we may succeed simplifying to was not | ||||
5111 | // def-reachable from the original PHI! | ||||
5112 | |||||
5113 | // If all of the PHI's incoming values are the same then replace the PHI node | ||||
5114 | // with the common value. | ||||
5115 | Value *CommonValue = nullptr; | ||||
5116 | bool HasUndefInput = false; | ||||
5117 | for (Value *Incoming : IncomingValues) { | ||||
5118 | // If the incoming value is the phi node itself, it can safely be skipped. | ||||
5119 | if (Incoming == PN) | ||||
5120 | continue; | ||||
5121 | if (Q.isUndefValue(Incoming)) { | ||||
5122 | // Remember that we saw an undef value, but otherwise ignore them. | ||||
5123 | HasUndefInput = true; | ||||
5124 | continue; | ||||
5125 | } | ||||
5126 | if (CommonValue && Incoming != CommonValue) | ||||
5127 | return nullptr; // Not the same, bail out. | ||||
5128 | CommonValue = Incoming; | ||||
5129 | } | ||||
5130 | |||||
5131 | // If CommonValue is null then all of the incoming values were either undef or | ||||
5132 | // equal to the phi node itself. | ||||
5133 | if (!CommonValue) | ||||
5134 | return UndefValue::get(PN->getType()); | ||||
5135 | |||||
5136 | if (HasUndefInput) { | ||||
5137 | // If we have a PHI node like phi(X, undef, X), where X is defined by some | ||||
5138 | // instruction, we cannot return X as the result of the PHI node unless it | ||||
5139 | // dominates the PHI block. | ||||
5140 | return valueDominatesPHI(CommonValue, PN, Q.DT) ? CommonValue : nullptr; | ||||
5141 | } | ||||
5142 | |||||
5143 | return CommonValue; | ||||
5144 | } | ||||
5145 | |||||
5146 | static Value *simplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty, | ||||
5147 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
5148 | if (auto *C = dyn_cast<Constant>(Op)) | ||||
5149 | return ConstantFoldCastOperand(CastOpc, C, Ty, Q.DL); | ||||
5150 | |||||
5151 | if (auto *CI = dyn_cast<CastInst>(Op)) { | ||||
5152 | auto *Src = CI->getOperand(0); | ||||
5153 | Type *SrcTy = Src->getType(); | ||||
5154 | Type *MidTy = CI->getType(); | ||||
5155 | Type *DstTy = Ty; | ||||
5156 | if (Src->getType() == Ty) { | ||||
5157 | auto FirstOp = static_cast<Instruction::CastOps>(CI->getOpcode()); | ||||
5158 | auto SecondOp = static_cast<Instruction::CastOps>(CastOpc); | ||||
5159 | Type *SrcIntPtrTy = | ||||
5160 | SrcTy->isPtrOrPtrVectorTy() ? Q.DL.getIntPtrType(SrcTy) : nullptr; | ||||
5161 | Type *MidIntPtrTy = | ||||
5162 | MidTy->isPtrOrPtrVectorTy() ? Q.DL.getIntPtrType(MidTy) : nullptr; | ||||
5163 | Type *DstIntPtrTy = | ||||
5164 | DstTy->isPtrOrPtrVectorTy() ? Q.DL.getIntPtrType(DstTy) : nullptr; | ||||
5165 | if (CastInst::isEliminableCastPair(FirstOp, SecondOp, SrcTy, MidTy, DstTy, | ||||
5166 | SrcIntPtrTy, MidIntPtrTy, | ||||
5167 | DstIntPtrTy) == Instruction::BitCast) | ||||
5168 | return Src; | ||||
5169 | } | ||||
5170 | } | ||||
5171 | |||||
5172 | // bitcast x -> x | ||||
5173 | if (CastOpc == Instruction::BitCast) | ||||
5174 | if (Op->getType() == Ty) | ||||
5175 | return Op; | ||||
5176 | |||||
5177 | return nullptr; | ||||
5178 | } | ||||
5179 | |||||
5180 | Value *llvm::simplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty, | ||||
5181 | const SimplifyQuery &Q) { | ||||
5182 | return ::simplifyCastInst(CastOpc, Op, Ty, Q, RecursionLimit); | ||||
5183 | } | ||||
5184 | |||||
5185 | /// For the given destination element of a shuffle, peek through shuffles to | ||||
5186 | /// match a root vector source operand that contains that element in the same | ||||
5187 | /// vector lane (ie, the same mask index), so we can eliminate the shuffle(s). | ||||
5188 | static Value *foldIdentityShuffles(int DestElt, Value *Op0, Value *Op1, | ||||
5189 | int MaskVal, Value *RootVec, | ||||
5190 | unsigned MaxRecurse) { | ||||
5191 | if (!MaxRecurse--) | ||||
5192 | return nullptr; | ||||
5193 | |||||
5194 | // Bail out if any mask value is undefined. That kind of shuffle may be | ||||
5195 | // simplified further based on demanded bits or other folds. | ||||
5196 | if (MaskVal == -1) | ||||
5197 | return nullptr; | ||||
5198 | |||||
5199 | // The mask value chooses which source operand we need to look at next. | ||||
5200 | int InVecNumElts = cast<FixedVectorType>(Op0->getType())->getNumElements(); | ||||
5201 | int RootElt = MaskVal; | ||||
5202 | Value *SourceOp = Op0; | ||||
5203 | if (MaskVal >= InVecNumElts) { | ||||
5204 | RootElt = MaskVal - InVecNumElts; | ||||
5205 | SourceOp = Op1; | ||||
5206 | } | ||||
5207 | |||||
5208 | // If the source operand is a shuffle itself, look through it to find the | ||||
5209 | // matching root vector. | ||||
5210 | if (auto *SourceShuf = dyn_cast<ShuffleVectorInst>(SourceOp)) { | ||||
5211 | return foldIdentityShuffles( | ||||
5212 | DestElt, SourceShuf->getOperand(0), SourceShuf->getOperand(1), | ||||
5213 | SourceShuf->getMaskValue(RootElt), RootVec, MaxRecurse); | ||||
5214 | } | ||||
5215 | |||||
5216 | // TODO: Look through bitcasts? What if the bitcast changes the vector element | ||||
5217 | // size? | ||||
5218 | |||||
5219 | // The source operand is not a shuffle. Initialize the root vector value for | ||||
5220 | // this shuffle if that has not been done yet. | ||||
5221 | if (!RootVec) | ||||
5222 | RootVec = SourceOp; | ||||
5223 | |||||
5224 | // Give up as soon as a source operand does not match the existing root value. | ||||
5225 | if (RootVec != SourceOp) | ||||
5226 | return nullptr; | ||||
5227 | |||||
5228 | // The element must be coming from the same lane in the source vector | ||||
5229 | // (although it may have crossed lanes in intermediate shuffles). | ||||
5230 | if (RootElt != DestElt) | ||||
5231 | return nullptr; | ||||
5232 | |||||
5233 | return RootVec; | ||||
5234 | } | ||||
5235 | |||||
5236 | static Value *simplifyShuffleVectorInst(Value *Op0, Value *Op1, | ||||
5237 | ArrayRef<int> Mask, Type *RetTy, | ||||
5238 | const SimplifyQuery &Q, | ||||
5239 | unsigned MaxRecurse) { | ||||
5240 | if (all_of(Mask, [](int Elem) { return Elem == UndefMaskElem; })) | ||||
5241 | return UndefValue::get(RetTy); | ||||
5242 | |||||
5243 | auto *InVecTy = cast<VectorType>(Op0->getType()); | ||||
5244 | unsigned MaskNumElts = Mask.size(); | ||||
5245 | ElementCount InVecEltCount = InVecTy->getElementCount(); | ||||
5246 | |||||
5247 | bool Scalable = InVecEltCount.isScalable(); | ||||
5248 | |||||
5249 | SmallVector<int, 32> Indices; | ||||
5250 | Indices.assign(Mask.begin(), Mask.end()); | ||||
5251 | |||||
5252 | // Canonicalization: If mask does not select elements from an input vector, | ||||
5253 | // replace that input vector with poison. | ||||
5254 | if (!Scalable) { | ||||
5255 | bool MaskSelects0 = false, MaskSelects1 = false; | ||||
5256 | unsigned InVecNumElts = InVecEltCount.getKnownMinValue(); | ||||
5257 | for (unsigned i = 0; i != MaskNumElts; ++i) { | ||||
5258 | if (Indices[i] == -1) | ||||
5259 | continue; | ||||
5260 | if ((unsigned)Indices[i] < InVecNumElts) | ||||
5261 | MaskSelects0 = true; | ||||
5262 | else | ||||
5263 | MaskSelects1 = true; | ||||
5264 | } | ||||
5265 | if (!MaskSelects0) | ||||
5266 | Op0 = PoisonValue::get(InVecTy); | ||||
5267 | if (!MaskSelects1) | ||||
5268 | Op1 = PoisonValue::get(InVecTy); | ||||
5269 | } | ||||
5270 | |||||
5271 | auto *Op0Const = dyn_cast<Constant>(Op0); | ||||
5272 | auto *Op1Const = dyn_cast<Constant>(Op1); | ||||
5273 | |||||
5274 | // If all operands are constant, constant fold the shuffle. This | ||||
5275 | // transformation depends on the value of the mask which is not known at | ||||
5276 | // compile time for scalable vectors | ||||
5277 | if (Op0Const && Op1Const) | ||||
5278 | return ConstantExpr::getShuffleVector(Op0Const, Op1Const, Mask); | ||||
5279 | |||||
5280 | // Canonicalization: if only one input vector is constant, it shall be the | ||||
5281 | // second one. This transformation depends on the value of the mask which | ||||
5282 | // is not known at compile time for scalable vectors | ||||
5283 | if (!Scalable && Op0Const && !Op1Const) { | ||||
5284 | std::swap(Op0, Op1); | ||||
5285 | ShuffleVectorInst::commuteShuffleMask(Indices, | ||||
5286 | InVecEltCount.getKnownMinValue()); | ||||
5287 | } | ||||
5288 | |||||
5289 | // A splat of an inserted scalar constant becomes a vector constant: | ||||
5290 | // shuf (inselt ?, C, IndexC), undef, <IndexC, IndexC...> --> <C, C...> | ||||
5291 | // NOTE: We may have commuted above, so analyze the updated Indices, not the | ||||
5292 | // original mask constant. | ||||
5293 | // NOTE: This transformation depends on the value of the mask which is not | ||||
5294 | // known at compile time for scalable vectors | ||||
5295 | Constant *C; | ||||
5296 | ConstantInt *IndexC; | ||||
5297 | if (!Scalable && match(Op0, m_InsertElt(m_Value(), m_Constant(C), | ||||
5298 | m_ConstantInt(IndexC)))) { | ||||
5299 | // Match a splat shuffle mask of the insert index allowing undef elements. | ||||
5300 | int InsertIndex = IndexC->getZExtValue(); | ||||
5301 | if (all_of(Indices, [InsertIndex](int MaskElt) { | ||||
5302 | return MaskElt == InsertIndex || MaskElt == -1; | ||||
5303 | })) { | ||||
5304 | assert(isa<UndefValue>(Op1) && "Expected undef operand 1 for splat")(static_cast <bool> (isa<UndefValue>(Op1) && "Expected undef operand 1 for splat") ? void (0) : __assert_fail ("isa<UndefValue>(Op1) && \"Expected undef operand 1 for splat\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 5304, __extension__ __PRETTY_FUNCTION__)); | ||||
5305 | |||||
5306 | // Shuffle mask undefs become undefined constant result elements. | ||||
5307 | SmallVector<Constant *, 16> VecC(MaskNumElts, C); | ||||
5308 | for (unsigned i = 0; i != MaskNumElts; ++i) | ||||
5309 | if (Indices[i] == -1) | ||||
5310 | VecC[i] = UndefValue::get(C->getType()); | ||||
5311 | return ConstantVector::get(VecC); | ||||
5312 | } | ||||
5313 | } | ||||
5314 | |||||
5315 | // A shuffle of a splat is always the splat itself. Legal if the shuffle's | ||||
5316 | // value type is same as the input vectors' type. | ||||
5317 | if (auto *OpShuf = dyn_cast<ShuffleVectorInst>(Op0)) | ||||
5318 | if (Q.isUndefValue(Op1) && RetTy == InVecTy && | ||||
5319 | all_equal(OpShuf->getShuffleMask())) | ||||
5320 | return Op0; | ||||
5321 | |||||
5322 | // All remaining transformation depend on the value of the mask, which is | ||||
5323 | // not known at compile time for scalable vectors. | ||||
5324 | if (Scalable) | ||||
5325 | return nullptr; | ||||
5326 | |||||
5327 | // Don't fold a shuffle with undef mask elements. This may get folded in a | ||||
5328 | // better way using demanded bits or other analysis. | ||||
5329 | // TODO: Should we allow this? | ||||
5330 | if (is_contained(Indices, -1)) | ||||
5331 | return nullptr; | ||||
5332 | |||||
5333 | // Check if every element of this shuffle can be mapped back to the | ||||
5334 | // corresponding element of a single root vector. If so, we don't need this | ||||
5335 | // shuffle. This handles simple identity shuffles as well as chains of | ||||
5336 | // shuffles that may widen/narrow and/or move elements across lanes and back. | ||||
5337 | Value *RootVec = nullptr; | ||||
5338 | for (unsigned i = 0; i != MaskNumElts; ++i) { | ||||
5339 | // Note that recursion is limited for each vector element, so if any element | ||||
5340 | // exceeds the limit, this will fail to simplify. | ||||
5341 | RootVec = | ||||
5342 | foldIdentityShuffles(i, Op0, Op1, Indices[i], RootVec, MaxRecurse); | ||||
5343 | |||||
5344 | // We can't replace a widening/narrowing shuffle with one of its operands. | ||||
5345 | if (!RootVec || RootVec->getType() != RetTy) | ||||
5346 | return nullptr; | ||||
5347 | } | ||||
5348 | return RootVec; | ||||
5349 | } | ||||
5350 | |||||
5351 | /// Given operands for a ShuffleVectorInst, fold the result or return null. | ||||
5352 | Value *llvm::simplifyShuffleVectorInst(Value *Op0, Value *Op1, | ||||
5353 | ArrayRef<int> Mask, Type *RetTy, | ||||
5354 | const SimplifyQuery &Q) { | ||||
5355 | return ::simplifyShuffleVectorInst(Op0, Op1, Mask, RetTy, Q, RecursionLimit); | ||||
5356 | } | ||||
5357 | |||||
5358 | static Constant *foldConstant(Instruction::UnaryOps Opcode, Value *&Op, | ||||
5359 | const SimplifyQuery &Q) { | ||||
5360 | if (auto *C = dyn_cast<Constant>(Op)) | ||||
5361 | return ConstantFoldUnaryOpOperand(Opcode, C, Q.DL); | ||||
5362 | return nullptr; | ||||
5363 | } | ||||
5364 | |||||
5365 | /// Given the operand for an FNeg, see if we can fold the result. If not, this | ||||
5366 | /// returns null. | ||||
5367 | static Value *simplifyFNegInst(Value *Op, FastMathFlags FMF, | ||||
5368 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
5369 | if (Constant *C = foldConstant(Instruction::FNeg, Op, Q)) | ||||
5370 | return C; | ||||
5371 | |||||
5372 | Value *X; | ||||
5373 | // fneg (fneg X) ==> X | ||||
5374 | if (match(Op, m_FNeg(m_Value(X)))) | ||||
5375 | return X; | ||||
5376 | |||||
5377 | return nullptr; | ||||
5378 | } | ||||
5379 | |||||
5380 | Value *llvm::simplifyFNegInst(Value *Op, FastMathFlags FMF, | ||||
5381 | const SimplifyQuery &Q) { | ||||
5382 | return ::simplifyFNegInst(Op, FMF, Q, RecursionLimit); | ||||
5383 | } | ||||
5384 | |||||
5385 | /// Try to propagate existing NaN values when possible. If not, replace the | ||||
5386 | /// constant or elements in the constant with a canonical NaN. | ||||
5387 | static Constant *propagateNaN(Constant *In) { | ||||
5388 | Type *Ty = In->getType(); | ||||
5389 | if (auto *VecTy = dyn_cast<FixedVectorType>(Ty)) { | ||||
5390 | unsigned NumElts = VecTy->getNumElements(); | ||||
5391 | SmallVector<Constant *, 32> NewC(NumElts); | ||||
5392 | for (unsigned i = 0; i != NumElts; ++i) { | ||||
5393 | Constant *EltC = In->getAggregateElement(i); | ||||
5394 | // Poison elements propagate. NaN propagates except signaling is quieted. | ||||
5395 | // Replace unknown or undef elements with canonical NaN. | ||||
5396 | if (EltC && isa<PoisonValue>(EltC)) | ||||
5397 | NewC[i] = EltC; | ||||
5398 | else if (EltC && EltC->isNaN()) | ||||
5399 | NewC[i] = ConstantFP::get( | ||||
5400 | EltC->getType(), cast<ConstantFP>(EltC)->getValue().makeQuiet()); | ||||
5401 | else | ||||
5402 | NewC[i] = ConstantFP::getNaN(VecTy->getElementType()); | ||||
5403 | } | ||||
5404 | return ConstantVector::get(NewC); | ||||
5405 | } | ||||
5406 | |||||
5407 | // If it is not a fixed vector, but not a simple NaN either, return a | ||||
5408 | // canonical NaN. | ||||
5409 | if (!In->isNaN()) | ||||
5410 | return ConstantFP::getNaN(Ty); | ||||
5411 | |||||
5412 | // Propagate an existing QNaN constant. If it is an SNaN, make it quiet, but | ||||
5413 | // preserve the sign/payload. | ||||
5414 | return ConstantFP::get(Ty, cast<ConstantFP>(In)->getValue().makeQuiet()); | ||||
5415 | } | ||||
5416 | |||||
5417 | /// Perform folds that are common to any floating-point operation. This implies | ||||
5418 | /// transforms based on poison/undef/NaN because the operation itself makes no | ||||
5419 | /// difference to the result. | ||||
5420 | static Constant *simplifyFPOp(ArrayRef<Value *> Ops, FastMathFlags FMF, | ||||
5421 | const SimplifyQuery &Q, | ||||
5422 | fp::ExceptionBehavior ExBehavior, | ||||
5423 | RoundingMode Rounding) { | ||||
5424 | // Poison is independent of anything else. It always propagates from an | ||||
5425 | // operand to a math result. | ||||
5426 | if (any_of(Ops, [](Value *V) { return match(V, m_Poison()); })) | ||||
5427 | return PoisonValue::get(Ops[0]->getType()); | ||||
5428 | |||||
5429 | for (Value *V : Ops) { | ||||
5430 | bool IsNan = match(V, m_NaN()); | ||||
5431 | bool IsInf = match(V, m_Inf()); | ||||
5432 | bool IsUndef = Q.isUndefValue(V); | ||||
5433 | |||||
5434 | // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand | ||||
5435 | // (an undef operand can be chosen to be Nan/Inf), then the result of | ||||
5436 | // this operation is poison. | ||||
5437 | if (FMF.noNaNs() && (IsNan || IsUndef)) | ||||
5438 | return PoisonValue::get(V->getType()); | ||||
5439 | if (FMF.noInfs() && (IsInf || IsUndef)) | ||||
5440 | return PoisonValue::get(V->getType()); | ||||
5441 | |||||
5442 | if (isDefaultFPEnvironment(ExBehavior, Rounding)) { | ||||
5443 | // Undef does not propagate because undef means that all bits can take on | ||||
5444 | // any value. If this is undef * NaN for example, then the result values | ||||
5445 | // (at least the exponent bits) are limited. Assume the undef is a | ||||
5446 | // canonical NaN and propagate that. | ||||
5447 | if (IsUndef) | ||||
5448 | return ConstantFP::getNaN(V->getType()); | ||||
5449 | if (IsNan) | ||||
5450 | return propagateNaN(cast<Constant>(V)); | ||||
5451 | } else if (ExBehavior != fp::ebStrict) { | ||||
5452 | if (IsNan) | ||||
5453 | return propagateNaN(cast<Constant>(V)); | ||||
5454 | } | ||||
5455 | } | ||||
5456 | return nullptr; | ||||
5457 | } | ||||
5458 | |||||
5459 | /// Given operands for an FAdd, see if we can fold the result. If not, this | ||||
5460 | /// returns null. | ||||
5461 | static Value * | ||||
5462 | simplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF, | ||||
5463 | const SimplifyQuery &Q, unsigned MaxRecurse, | ||||
5464 | fp::ExceptionBehavior ExBehavior = fp::ebIgnore, | ||||
5465 | RoundingMode Rounding = RoundingMode::NearestTiesToEven) { | ||||
5466 | if (isDefaultFPEnvironment(ExBehavior, Rounding)) | ||||
5467 | if (Constant *C = foldOrCommuteConstant(Instruction::FAdd, Op0, Op1, Q)) | ||||
5468 | return C; | ||||
5469 | |||||
5470 | if (Constant *C = simplifyFPOp({Op0, Op1}, FMF, Q, ExBehavior, Rounding)) | ||||
5471 | return C; | ||||
5472 | |||||
5473 | // fadd X, -0 ==> X | ||||
5474 | // With strict/constrained FP, we have these possible edge cases that do | ||||
5475 | // not simplify to Op0: | ||||
5476 | // fadd SNaN, -0.0 --> QNaN | ||||
5477 | // fadd +0.0, -0.0 --> -0.0 (but only with round toward negative) | ||||
5478 | if (canIgnoreSNaN(ExBehavior, FMF) && | ||||
5479 | (!canRoundingModeBe(Rounding, RoundingMode::TowardNegative) || | ||||
5480 | FMF.noSignedZeros())) | ||||
5481 | if (match(Op1, m_NegZeroFP())) | ||||
5482 | return Op0; | ||||
5483 | |||||
5484 | // fadd X, 0 ==> X, when we know X is not -0 | ||||
5485 | if (canIgnoreSNaN(ExBehavior, FMF)) | ||||
5486 | if (match(Op1, m_PosZeroFP()) && | ||||
5487 | (FMF.noSignedZeros() || CannotBeNegativeZero(Op0, Q.TLI))) | ||||
5488 | return Op0; | ||||
5489 | |||||
5490 | if (!isDefaultFPEnvironment(ExBehavior, Rounding)) | ||||
5491 | return nullptr; | ||||
5492 | |||||
5493 | if (FMF.noNaNs()) { | ||||
5494 | // With nnan: X + {+/-}Inf --> {+/-}Inf | ||||
5495 | if (match(Op1, m_Inf())) | ||||
5496 | return Op1; | ||||
5497 | |||||
5498 | // With nnan: -X + X --> 0.0 (and commuted variant) | ||||
5499 | // We don't have to explicitly exclude infinities (ninf): INF + -INF == NaN. | ||||
5500 | // Negative zeros are allowed because we always end up with positive zero: | ||||
5501 | // X = -0.0: (-0.0 - (-0.0)) + (-0.0) == ( 0.0) + (-0.0) == 0.0 | ||||
5502 | // X = -0.0: ( 0.0 - (-0.0)) + (-0.0) == ( 0.0) + (-0.0) == 0.0 | ||||
5503 | // X = 0.0: (-0.0 - ( 0.0)) + ( 0.0) == (-0.0) + ( 0.0) == 0.0 | ||||
5504 | // X = 0.0: ( 0.0 - ( 0.0)) + ( 0.0) == ( 0.0) + ( 0.0) == 0.0 | ||||
5505 | if (match(Op0, m_FSub(m_AnyZeroFP(), m_Specific(Op1))) || | ||||
5506 | match(Op1, m_FSub(m_AnyZeroFP(), m_Specific(Op0)))) | ||||
5507 | return ConstantFP::getNullValue(Op0->getType()); | ||||
5508 | |||||
5509 | if (match(Op0, m_FNeg(m_Specific(Op1))) || | ||||
5510 | match(Op1, m_FNeg(m_Specific(Op0)))) | ||||
5511 | return ConstantFP::getNullValue(Op0->getType()); | ||||
5512 | } | ||||
5513 | |||||
5514 | // (X - Y) + Y --> X | ||||
5515 | // Y + (X - Y) --> X | ||||
5516 | Value *X; | ||||
5517 | if (FMF.noSignedZeros() && FMF.allowReassoc() && | ||||
5518 | (match(Op0, m_FSub(m_Value(X), m_Specific(Op1))) || | ||||
5519 | match(Op1, m_FSub(m_Value(X), m_Specific(Op0))))) | ||||
5520 | return X; | ||||
5521 | |||||
5522 | return nullptr; | ||||
5523 | } | ||||
5524 | |||||
5525 | /// Given operands for an FSub, see if we can fold the result. If not, this | ||||
5526 | /// returns null. | ||||
5527 | static Value * | ||||
5528 | simplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF, | ||||
5529 | const SimplifyQuery &Q, unsigned MaxRecurse, | ||||
5530 | fp::ExceptionBehavior ExBehavior = fp::ebIgnore, | ||||
5531 | RoundingMode Rounding = RoundingMode::NearestTiesToEven) { | ||||
5532 | if (isDefaultFPEnvironment(ExBehavior, Rounding)) | ||||
5533 | if (Constant *C = foldOrCommuteConstant(Instruction::FSub, Op0, Op1, Q)) | ||||
5534 | return C; | ||||
5535 | |||||
5536 | if (Constant *C = simplifyFPOp({Op0, Op1}, FMF, Q, ExBehavior, Rounding)) | ||||
5537 | return C; | ||||
5538 | |||||
5539 | // fsub X, +0 ==> X | ||||
5540 | if (canIgnoreSNaN(ExBehavior, FMF) && | ||||
5541 | (!canRoundingModeBe(Rounding, RoundingMode::TowardNegative) || | ||||
5542 | FMF.noSignedZeros())) | ||||
5543 | if (match(Op1, m_PosZeroFP())) | ||||
5544 | return Op0; | ||||
5545 | |||||
5546 | // fsub X, -0 ==> X, when we know X is not -0 | ||||
5547 | if (canIgnoreSNaN(ExBehavior, FMF)) | ||||
5548 | if (match(Op1, m_NegZeroFP()) && | ||||
5549 | (FMF.noSignedZeros() || CannotBeNegativeZero(Op0, Q.TLI))) | ||||
5550 | return Op0; | ||||
5551 | |||||
5552 | // fsub -0.0, (fsub -0.0, X) ==> X | ||||
5553 | // fsub -0.0, (fneg X) ==> X | ||||
5554 | Value *X; | ||||
5555 | if (canIgnoreSNaN(ExBehavior, FMF)) | ||||
5556 | if (match(Op0, m_NegZeroFP()) && match(Op1, m_FNeg(m_Value(X)))) | ||||
5557 | return X; | ||||
5558 | |||||
5559 | // fsub 0.0, (fsub 0.0, X) ==> X if signed zeros are ignored. | ||||
5560 | // fsub 0.0, (fneg X) ==> X if signed zeros are ignored. | ||||
5561 | if (canIgnoreSNaN(ExBehavior, FMF)) | ||||
5562 | if (FMF.noSignedZeros() && match(Op0, m_AnyZeroFP()) && | ||||
5563 | (match(Op1, m_FSub(m_AnyZeroFP(), m_Value(X))) || | ||||
5564 | match(Op1, m_FNeg(m_Value(X))))) | ||||
5565 | return X; | ||||
5566 | |||||
5567 | if (!isDefaultFPEnvironment(ExBehavior, Rounding)) | ||||
5568 | return nullptr; | ||||
5569 | |||||
5570 | if (FMF.noNaNs()) { | ||||
5571 | // fsub nnan x, x ==> 0.0 | ||||
5572 | if (Op0 == Op1) | ||||
5573 | return Constant::getNullValue(Op0->getType()); | ||||
5574 | |||||
5575 | // With nnan: {+/-}Inf - X --> {+/-}Inf | ||||
5576 | if (match(Op0, m_Inf())) | ||||
5577 | return Op0; | ||||
5578 | |||||
5579 | // With nnan: X - {+/-}Inf --> {-/+}Inf | ||||
5580 | if (match(Op1, m_Inf())) | ||||
5581 | return foldConstant(Instruction::FNeg, Op1, Q); | ||||
5582 | } | ||||
5583 | |||||
5584 | // Y - (Y - X) --> X | ||||
5585 | // (X + Y) - Y --> X | ||||
5586 | if (FMF.noSignedZeros() && FMF.allowReassoc() && | ||||
5587 | (match(Op1, m_FSub(m_Specific(Op0), m_Value(X))) || | ||||
5588 | match(Op0, m_c_FAdd(m_Specific(Op1), m_Value(X))))) | ||||
5589 | return X; | ||||
5590 | |||||
5591 | return nullptr; | ||||
5592 | } | ||||
5593 | |||||
5594 | static Value *simplifyFMAFMul(Value *Op0, Value *Op1, FastMathFlags FMF, | ||||
5595 | const SimplifyQuery &Q, unsigned MaxRecurse, | ||||
5596 | fp::ExceptionBehavior ExBehavior, | ||||
5597 | RoundingMode Rounding) { | ||||
5598 | if (Constant *C = simplifyFPOp({Op0, Op1}, FMF, Q, ExBehavior, Rounding)) | ||||
5599 | return C; | ||||
5600 | |||||
5601 | if (!isDefaultFPEnvironment(ExBehavior, Rounding)) | ||||
5602 | return nullptr; | ||||
5603 | |||||
5604 | // Canonicalize special constants as operand 1. | ||||
5605 | if (match(Op0, m_FPOne()) || match(Op0, m_AnyZeroFP())) | ||||
5606 | std::swap(Op0, Op1); | ||||
5607 | |||||
5608 | // X * 1.0 --> X | ||||
5609 | if (match(Op1, m_FPOne())) | ||||
5610 | return Op0; | ||||
5611 | |||||
5612 | if (match(Op1, m_AnyZeroFP())) { | ||||
5613 | // X * 0.0 --> 0.0 (with nnan and nsz) | ||||
5614 | if (FMF.noNaNs() && FMF.noSignedZeros()) | ||||
5615 | return ConstantFP::getNullValue(Op0->getType()); | ||||
5616 | |||||
5617 | // +normal number * (-)0.0 --> (-)0.0 | ||||
5618 | if (isKnownNeverInfinity(Op0, Q.TLI) && isKnownNeverNaN(Op0, Q.TLI) && | ||||
5619 | SignBitMustBeZero(Op0, Q.TLI)) | ||||
5620 | return Op1; | ||||
5621 | } | ||||
5622 | |||||
5623 | // sqrt(X) * sqrt(X) --> X, if we can: | ||||
5624 | // 1. Remove the intermediate rounding (reassociate). | ||||
5625 | // 2. Ignore non-zero negative numbers because sqrt would produce NAN. | ||||
5626 | // 3. Ignore -0.0 because sqrt(-0.0) == -0.0, but -0.0 * -0.0 == 0.0. | ||||
5627 | Value *X; | ||||
5628 | if (Op0 == Op1 && match(Op0, m_Sqrt(m_Value(X))) && FMF.allowReassoc() && | ||||
5629 | FMF.noNaNs() && FMF.noSignedZeros()) | ||||
5630 | return X; | ||||
5631 | |||||
5632 | return nullptr; | ||||
5633 | } | ||||
5634 | |||||
5635 | /// Given the operands for an FMul, see if we can fold the result | ||||
5636 | static Value * | ||||
5637 | simplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF, | ||||
5638 | const SimplifyQuery &Q, unsigned MaxRecurse, | ||||
5639 | fp::ExceptionBehavior ExBehavior = fp::ebIgnore, | ||||
5640 | RoundingMode Rounding = RoundingMode::NearestTiesToEven) { | ||||
5641 | if (isDefaultFPEnvironment(ExBehavior, Rounding)) | ||||
5642 | if (Constant *C = foldOrCommuteConstant(Instruction::FMul, Op0, Op1, Q)) | ||||
5643 | return C; | ||||
5644 | |||||
5645 | // Now apply simplifications that do not require rounding. | ||||
5646 | return simplifyFMAFMul(Op0, Op1, FMF, Q, MaxRecurse, ExBehavior, Rounding); | ||||
5647 | } | ||||
5648 | |||||
5649 | Value *llvm::simplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF, | ||||
5650 | const SimplifyQuery &Q, | ||||
5651 | fp::ExceptionBehavior ExBehavior, | ||||
5652 | RoundingMode Rounding) { | ||||
5653 | return ::simplifyFAddInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, | ||||
5654 | Rounding); | ||||
5655 | } | ||||
5656 | |||||
5657 | Value *llvm::simplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF, | ||||
5658 | const SimplifyQuery &Q, | ||||
5659 | fp::ExceptionBehavior ExBehavior, | ||||
5660 | RoundingMode Rounding) { | ||||
5661 | return ::simplifyFSubInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, | ||||
5662 | Rounding); | ||||
5663 | } | ||||
5664 | |||||
5665 | Value *llvm::simplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF, | ||||
5666 | const SimplifyQuery &Q, | ||||
5667 | fp::ExceptionBehavior ExBehavior, | ||||
5668 | RoundingMode Rounding) { | ||||
5669 | return ::simplifyFMulInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, | ||||
5670 | Rounding); | ||||
5671 | } | ||||
5672 | |||||
5673 | Value *llvm::simplifyFMAFMul(Value *Op0, Value *Op1, FastMathFlags FMF, | ||||
5674 | const SimplifyQuery &Q, | ||||
5675 | fp::ExceptionBehavior ExBehavior, | ||||
5676 | RoundingMode Rounding) { | ||||
5677 | return ::simplifyFMAFMul(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, | ||||
5678 | Rounding); | ||||
5679 | } | ||||
5680 | |||||
5681 | static Value * | ||||
5682 | simplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF, | ||||
5683 | const SimplifyQuery &Q, unsigned, | ||||
5684 | fp::ExceptionBehavior ExBehavior = fp::ebIgnore, | ||||
5685 | RoundingMode Rounding = RoundingMode::NearestTiesToEven) { | ||||
5686 | if (isDefaultFPEnvironment(ExBehavior, Rounding)) | ||||
5687 | if (Constant *C = foldOrCommuteConstant(Instruction::FDiv, Op0, Op1, Q)) | ||||
5688 | return C; | ||||
5689 | |||||
5690 | if (Constant *C = simplifyFPOp({Op0, Op1}, FMF, Q, ExBehavior, Rounding)) | ||||
5691 | return C; | ||||
5692 | |||||
5693 | if (!isDefaultFPEnvironment(ExBehavior, Rounding)) | ||||
5694 | return nullptr; | ||||
5695 | |||||
5696 | // X / 1.0 -> X | ||||
5697 | if (match(Op1, m_FPOne())) | ||||
5698 | return Op0; | ||||
5699 | |||||
5700 | // 0 / X -> 0 | ||||
5701 | // Requires that NaNs are off (X could be zero) and signed zeroes are | ||||
5702 | // ignored (X could be positive or negative, so the output sign is unknown). | ||||
5703 | if (FMF.noNaNs() && FMF.noSignedZeros() && match(Op0, m_AnyZeroFP())) | ||||
5704 | return ConstantFP::getNullValue(Op0->getType()); | ||||
5705 | |||||
5706 | if (FMF.noNaNs()) { | ||||
5707 | // X / X -> 1.0 is legal when NaNs are ignored. | ||||
5708 | // We can ignore infinities because INF/INF is NaN. | ||||
5709 | if (Op0 == Op1) | ||||
5710 | return ConstantFP::get(Op0->getType(), 1.0); | ||||
5711 | |||||
5712 | // (X * Y) / Y --> X if we can reassociate to the above form. | ||||
5713 | Value *X; | ||||
5714 | if (FMF.allowReassoc() && match(Op0, m_c_FMul(m_Value(X), m_Specific(Op1)))) | ||||
5715 | return X; | ||||
5716 | |||||
5717 | // -X / X -> -1.0 and | ||||
5718 | // X / -X -> -1.0 are legal when NaNs are ignored. | ||||
5719 | // We can ignore signed zeros because +-0.0/+-0.0 is NaN and ignored. | ||||
5720 | if (match(Op0, m_FNegNSZ(m_Specific(Op1))) || | ||||
5721 | match(Op1, m_FNegNSZ(m_Specific(Op0)))) | ||||
5722 | return ConstantFP::get(Op0->getType(), -1.0); | ||||
5723 | |||||
5724 | // nnan ninf X / [-]0.0 -> poison | ||||
5725 | if (FMF.noInfs() && match(Op1, m_AnyZeroFP())) | ||||
5726 | return PoisonValue::get(Op1->getType()); | ||||
5727 | } | ||||
5728 | |||||
5729 | return nullptr; | ||||
5730 | } | ||||
5731 | |||||
5732 | Value *llvm::simplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF, | ||||
5733 | const SimplifyQuery &Q, | ||||
5734 | fp::ExceptionBehavior ExBehavior, | ||||
5735 | RoundingMode Rounding) { | ||||
5736 | return ::simplifyFDivInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, | ||||
5737 | Rounding); | ||||
5738 | } | ||||
5739 | |||||
5740 | static Value * | ||||
5741 | simplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF, | ||||
5742 | const SimplifyQuery &Q, unsigned, | ||||
5743 | fp::ExceptionBehavior ExBehavior = fp::ebIgnore, | ||||
5744 | RoundingMode Rounding = RoundingMode::NearestTiesToEven) { | ||||
5745 | if (isDefaultFPEnvironment(ExBehavior, Rounding)) | ||||
5746 | if (Constant *C = foldOrCommuteConstant(Instruction::FRem, Op0, Op1, Q)) | ||||
5747 | return C; | ||||
5748 | |||||
5749 | if (Constant *C = simplifyFPOp({Op0, Op1}, FMF, Q, ExBehavior, Rounding)) | ||||
5750 | return C; | ||||
5751 | |||||
5752 | if (!isDefaultFPEnvironment(ExBehavior, Rounding)) | ||||
5753 | return nullptr; | ||||
5754 | |||||
5755 | // Unlike fdiv, the result of frem always matches the sign of the dividend. | ||||
5756 | // The constant match may include undef elements in a vector, so return a full | ||||
5757 | // zero constant as the result. | ||||
5758 | if (FMF.noNaNs()) { | ||||
5759 | // +0 % X -> 0 | ||||
5760 | if (match(Op0, m_PosZeroFP())) | ||||
5761 | return ConstantFP::getNullValue(Op0->getType()); | ||||
5762 | // -0 % X -> -0 | ||||
5763 | if (match(Op0, m_NegZeroFP())) | ||||
5764 | return ConstantFP::getNegativeZero(Op0->getType()); | ||||
5765 | } | ||||
5766 | |||||
5767 | return nullptr; | ||||
5768 | } | ||||
5769 | |||||
5770 | Value *llvm::simplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF, | ||||
5771 | const SimplifyQuery &Q, | ||||
5772 | fp::ExceptionBehavior ExBehavior, | ||||
5773 | RoundingMode Rounding) { | ||||
5774 | return ::simplifyFRemInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, | ||||
5775 | Rounding); | ||||
5776 | } | ||||
5777 | |||||
5778 | //=== Helper functions for higher up the class hierarchy. | ||||
5779 | |||||
5780 | /// Given the operand for a UnaryOperator, see if we can fold the result. | ||||
5781 | /// If not, this returns null. | ||||
5782 | static Value *simplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q, | ||||
5783 | unsigned MaxRecurse) { | ||||
5784 | switch (Opcode) { | ||||
5785 | case Instruction::FNeg: | ||||
5786 | return simplifyFNegInst(Op, FastMathFlags(), Q, MaxRecurse); | ||||
5787 | default: | ||||
5788 | llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "llvm/lib/Analysis/InstructionSimplify.cpp" , 5788); | ||||
5789 | } | ||||
5790 | } | ||||
5791 | |||||
5792 | /// Given the operand for a UnaryOperator, see if we can fold the result. | ||||
5793 | /// If not, this returns null. | ||||
5794 | /// Try to use FastMathFlags when folding the result. | ||||
5795 | static Value *simplifyFPUnOp(unsigned Opcode, Value *Op, | ||||
5796 | const FastMathFlags &FMF, const SimplifyQuery &Q, | ||||
5797 | unsigned MaxRecurse) { | ||||
5798 | switch (Opcode) { | ||||
5799 | case Instruction::FNeg: | ||||
5800 | return simplifyFNegInst(Op, FMF, Q, MaxRecurse); | ||||
5801 | default: | ||||
5802 | return simplifyUnOp(Opcode, Op, Q, MaxRecurse); | ||||
5803 | } | ||||
5804 | } | ||||
5805 | |||||
5806 | Value *llvm::simplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q) { | ||||
5807 | return ::simplifyUnOp(Opcode, Op, Q, RecursionLimit); | ||||
5808 | } | ||||
5809 | |||||
5810 | Value *llvm::simplifyUnOp(unsigned Opcode, Value *Op, FastMathFlags FMF, | ||||
5811 | const SimplifyQuery &Q) { | ||||
5812 | return ::simplifyFPUnOp(Opcode, Op, FMF, Q, RecursionLimit); | ||||
5813 | } | ||||
5814 | |||||
5815 | /// Given operands for a BinaryOperator, see if we can fold the result. | ||||
5816 | /// If not, this returns null. | ||||
5817 | static Value *simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, | ||||
5818 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
5819 | switch (Opcode) { | ||||
5820 | case Instruction::Add: | ||||
5821 | return simplifyAddInst(LHS, RHS, /* IsNSW */ false, /* IsNUW */ false, Q, | ||||
5822 | MaxRecurse); | ||||
5823 | case Instruction::Sub: | ||||
5824 | return simplifySubInst(LHS, RHS, /* IsNSW */ false, /* IsNUW */ false, Q, | ||||
5825 | MaxRecurse); | ||||
5826 | case Instruction::Mul: | ||||
5827 | return simplifyMulInst(LHS, RHS, /* IsNSW */ false, /* IsNUW */ false, Q, | ||||
5828 | MaxRecurse); | ||||
5829 | case Instruction::SDiv: | ||||
5830 | return simplifySDivInst(LHS, RHS, /* IsExact */ false, Q, MaxRecurse); | ||||
5831 | case Instruction::UDiv: | ||||
5832 | return simplifyUDivInst(LHS, RHS, /* IsExact */ false, Q, MaxRecurse); | ||||
5833 | case Instruction::SRem: | ||||
5834 | return simplifySRemInst(LHS, RHS, Q, MaxRecurse); | ||||
5835 | case Instruction::URem: | ||||
5836 | return simplifyURemInst(LHS, RHS, Q, MaxRecurse); | ||||
5837 | case Instruction::Shl: | ||||
5838 | return simplifyShlInst(LHS, RHS, /* IsNSW */ false, /* IsNUW */ false, Q, | ||||
5839 | MaxRecurse); | ||||
5840 | case Instruction::LShr: | ||||
5841 | return simplifyLShrInst(LHS, RHS, /* IsExact */ false, Q, MaxRecurse); | ||||
5842 | case Instruction::AShr: | ||||
5843 | return simplifyAShrInst(LHS, RHS, /* IsExact */ false, Q, MaxRecurse); | ||||
5844 | case Instruction::And: | ||||
5845 | return simplifyAndInst(LHS, RHS, Q, MaxRecurse); | ||||
5846 | case Instruction::Or: | ||||
5847 | return simplifyOrInst(LHS, RHS, Q, MaxRecurse); | ||||
5848 | case Instruction::Xor: | ||||
5849 | return simplifyXorInst(LHS, RHS, Q, MaxRecurse); | ||||
5850 | case Instruction::FAdd: | ||||
5851 | return simplifyFAddInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); | ||||
5852 | case Instruction::FSub: | ||||
5853 | return simplifyFSubInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); | ||||
5854 | case Instruction::FMul: | ||||
5855 | return simplifyFMulInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); | ||||
5856 | case Instruction::FDiv: | ||||
5857 | return simplifyFDivInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); | ||||
5858 | case Instruction::FRem: | ||||
5859 | return simplifyFRemInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); | ||||
5860 | default: | ||||
5861 | llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "llvm/lib/Analysis/InstructionSimplify.cpp" , 5861); | ||||
5862 | } | ||||
5863 | } | ||||
5864 | |||||
5865 | /// Given operands for a BinaryOperator, see if we can fold the result. | ||||
5866 | /// If not, this returns null. | ||||
5867 | /// Try to use FastMathFlags when folding the result. | ||||
5868 | static Value *simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, | ||||
5869 | const FastMathFlags &FMF, const SimplifyQuery &Q, | ||||
5870 | unsigned MaxRecurse) { | ||||
5871 | switch (Opcode) { | ||||
5872 | case Instruction::FAdd: | ||||
5873 | return simplifyFAddInst(LHS, RHS, FMF, Q, MaxRecurse); | ||||
5874 | case Instruction::FSub: | ||||
5875 | return simplifyFSubInst(LHS, RHS, FMF, Q, MaxRecurse); | ||||
5876 | case Instruction::FMul: | ||||
5877 | return simplifyFMulInst(LHS, RHS, FMF, Q, MaxRecurse); | ||||
5878 | case Instruction::FDiv: | ||||
5879 | return simplifyFDivInst(LHS, RHS, FMF, Q, MaxRecurse); | ||||
5880 | default: | ||||
5881 | return simplifyBinOp(Opcode, LHS, RHS, Q, MaxRecurse); | ||||
5882 | } | ||||
5883 | } | ||||
5884 | |||||
5885 | Value *llvm::simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, | ||||
5886 | const SimplifyQuery &Q) { | ||||
5887 | return ::simplifyBinOp(Opcode, LHS, RHS, Q, RecursionLimit); | ||||
5888 | } | ||||
5889 | |||||
5890 | Value *llvm::simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, | ||||
5891 | FastMathFlags FMF, const SimplifyQuery &Q) { | ||||
5892 | return ::simplifyBinOp(Opcode, LHS, RHS, FMF, Q, RecursionLimit); | ||||
5893 | } | ||||
5894 | |||||
5895 | /// Given operands for a CmpInst, see if we can fold the result. | ||||
5896 | static Value *simplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, | ||||
5897 | const SimplifyQuery &Q, unsigned MaxRecurse) { | ||||
5898 | if (CmpInst::isIntPredicate((CmpInst::Predicate)Predicate)) | ||||
5899 | return simplifyICmpInst(Predicate, LHS, RHS, Q, MaxRecurse); | ||||
5900 | return simplifyFCmpInst(Predicate, LHS, RHS, FastMathFlags(), Q, MaxRecurse); | ||||
5901 | } | ||||
5902 | |||||
5903 | Value *llvm::simplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, | ||||
5904 | const SimplifyQuery &Q) { | ||||
5905 | return ::simplifyCmpInst(Predicate, LHS, RHS, Q, RecursionLimit); | ||||
5906 | } | ||||
5907 | |||||
5908 | static bool isIdempotent(Intrinsic::ID ID) { | ||||
5909 | switch (ID) { | ||||
5910 | default: | ||||
5911 | return false; | ||||
5912 | |||||
5913 | // Unary idempotent: f(f(x)) = f(x) | ||||
5914 | case Intrinsic::fabs: | ||||
5915 | case Intrinsic::floor: | ||||
5916 | case Intrinsic::ceil: | ||||
5917 | case Intrinsic::trunc: | ||||
5918 | case Intrinsic::rint: | ||||
5919 | case Intrinsic::nearbyint: | ||||
5920 | case Intrinsic::round: | ||||
5921 | case Intrinsic::roundeven: | ||||
5922 | case Intrinsic::canonicalize: | ||||
5923 | case Intrinsic::arithmetic_fence: | ||||
5924 | return true; | ||||
5925 | } | ||||
5926 | } | ||||
5927 | |||||
5928 | /// Return true if the intrinsic rounds a floating-point value to an integral | ||||
5929 | /// floating-point value (not an integer type). | ||||
5930 | static bool removesFPFraction(Intrinsic::ID ID) { | ||||
5931 | switch (ID) { | ||||
5932 | default: | ||||
5933 | return false; | ||||
5934 | |||||
5935 | case Intrinsic::floor: | ||||
5936 | case Intrinsic::ceil: | ||||
5937 | case Intrinsic::trunc: | ||||
5938 | case Intrinsic::rint: | ||||
5939 | case Intrinsic::nearbyint: | ||||
5940 | case Intrinsic::round: | ||||
5941 | case Intrinsic::roundeven: | ||||
5942 | return true; | ||||
5943 | } | ||||
5944 | } | ||||
5945 | |||||
5946 | static Value *simplifyRelativeLoad(Constant *Ptr, Constant *Offset, | ||||
5947 | const DataLayout &DL) { | ||||
5948 | GlobalValue *PtrSym; | ||||
5949 | APInt PtrOffset; | ||||
5950 | if (!IsConstantOffsetFromGlobal(Ptr, PtrSym, PtrOffset, DL)) | ||||
5951 | return nullptr; | ||||
5952 | |||||
5953 | Type *Int8PtrTy = Type::getInt8PtrTy(Ptr->getContext()); | ||||
5954 | Type *Int32Ty = Type::getInt32Ty(Ptr->getContext()); | ||||
5955 | Type *Int32PtrTy = Int32Ty->getPointerTo(); | ||||
5956 | Type *Int64Ty = Type::getInt64Ty(Ptr->getContext()); | ||||
5957 | |||||
5958 | auto *OffsetConstInt = dyn_cast<ConstantInt>(Offset); | ||||
5959 | if (!OffsetConstInt || OffsetConstInt->getType()->getBitWidth() > 64) | ||||
5960 | return nullptr; | ||||
5961 | |||||
5962 | uint64_t OffsetInt = OffsetConstInt->getSExtValue(); | ||||
5963 | if (OffsetInt % 4 != 0) | ||||
5964 | return nullptr; | ||||
5965 | |||||
5966 | Constant *C = ConstantExpr::getGetElementPtr( | ||||
5967 | Int32Ty, ConstantExpr::getBitCast(Ptr, Int32PtrTy), | ||||
5968 | ConstantInt::get(Int64Ty, OffsetInt / 4)); | ||||
5969 | Constant *Loaded = ConstantFoldLoadFromConstPtr(C, Int32Ty, DL); | ||||
5970 | if (!Loaded) | ||||
5971 | return nullptr; | ||||
5972 | |||||
5973 | auto *LoadedCE = dyn_cast<ConstantExpr>(Loaded); | ||||
5974 | if (!LoadedCE) | ||||
5975 | return nullptr; | ||||
5976 | |||||
5977 | if (LoadedCE->getOpcode() == Instruction::Trunc) { | ||||
5978 | LoadedCE = dyn_cast<ConstantExpr>(LoadedCE->getOperand(0)); | ||||
5979 | if (!LoadedCE) | ||||
5980 | return nullptr; | ||||
5981 | } | ||||
5982 | |||||
5983 | if (LoadedCE->getOpcode() != Instruction::Sub) | ||||
5984 | return nullptr; | ||||
5985 | |||||
5986 | auto *LoadedLHS = dyn_cast<ConstantExpr>(LoadedCE->getOperand(0)); | ||||
5987 | if (!LoadedLHS || LoadedLHS->getOpcode() != Instruction::PtrToInt) | ||||
5988 | return nullptr; | ||||
5989 | auto *LoadedLHSPtr = LoadedLHS->getOperand(0); | ||||
5990 | |||||
5991 | Constant *LoadedRHS = LoadedCE->getOperand(1); | ||||
5992 | GlobalValue *LoadedRHSSym; | ||||
5993 | APInt LoadedRHSOffset; | ||||
5994 | if (!IsConstantOffsetFromGlobal(LoadedRHS, LoadedRHSSym, LoadedRHSOffset, | ||||
5995 | DL) || | ||||
5996 | PtrSym != LoadedRHSSym || PtrOffset != LoadedRHSOffset) | ||||
5997 | return nullptr; | ||||
5998 | |||||
5999 | return ConstantExpr::getBitCast(LoadedLHSPtr, Int8PtrTy); | ||||
6000 | } | ||||
6001 | |||||
6002 | static Value *simplifyUnaryIntrinsic(Function *F, Value *Op0, | ||||
6003 | const SimplifyQuery &Q) { | ||||
6004 | // Idempotent functions return the same result when called repeatedly. | ||||
6005 | Intrinsic::ID IID = F->getIntrinsicID(); | ||||
6006 | if (isIdempotent(IID)) | ||||
6007 | if (auto *II = dyn_cast<IntrinsicInst>(Op0)) | ||||
6008 | if (II->getIntrinsicID() == IID) | ||||
6009 | return II; | ||||
6010 | |||||
6011 | if (removesFPFraction(IID)) { | ||||
6012 | // Converting from int or calling a rounding function always results in a | ||||
6013 | // finite integral number or infinity. For those inputs, rounding functions | ||||
6014 | // always return the same value, so the (2nd) rounding is eliminated. Ex: | ||||
6015 | // floor (sitofp x) -> sitofp x | ||||
6016 | // round (ceil x) -> ceil x | ||||
6017 | auto *II = dyn_cast<IntrinsicInst>(Op0); | ||||
6018 | if ((II && removesFPFraction(II->getIntrinsicID())) || | ||||
6019 | match(Op0, m_SIToFP(m_Value())) || match(Op0, m_UIToFP(m_Value()))) | ||||
6020 | return Op0; | ||||
6021 | } | ||||
6022 | |||||
6023 | Value *X; | ||||
6024 | switch (IID) { | ||||
6025 | case Intrinsic::fabs: | ||||
6026 | if (SignBitMustBeZero(Op0, Q.TLI)) | ||||
6027 | return Op0; | ||||
6028 | break; | ||||
6029 | case Intrinsic::bswap: | ||||
6030 | // bswap(bswap(x)) -> x | ||||
6031 | if (match(Op0, m_BSwap(m_Value(X)))) | ||||
6032 | return X; | ||||
6033 | break; | ||||
6034 | case Intrinsic::bitreverse: | ||||
6035 | // bitreverse(bitreverse(x)) -> x | ||||
6036 | if (match(Op0, m_BitReverse(m_Value(X)))) | ||||
6037 | return X; | ||||
6038 | break; | ||||
6039 | case Intrinsic::ctpop: { | ||||
6040 | // ctpop(X) -> 1 iff X is non-zero power of 2. | ||||
6041 | if (isKnownToBeAPowerOfTwo(Op0, Q.DL, /*OrZero*/ false, 0, Q.AC, Q.CxtI, | ||||
6042 | Q.DT)) | ||||
6043 | return ConstantInt::get(Op0->getType(), 1); | ||||
6044 | // If everything but the lowest bit is zero, that bit is the pop-count. Ex: | ||||
6045 | // ctpop(and X, 1) --> and X, 1 | ||||
6046 | unsigned BitWidth = Op0->getType()->getScalarSizeInBits(); | ||||
6047 | if (MaskedValueIsZero(Op0, APInt::getHighBitsSet(BitWidth, BitWidth - 1), | ||||
6048 | Q.DL, 0, Q.AC, Q.CxtI, Q.DT)) | ||||
6049 | return Op0; | ||||
6050 | break; | ||||
6051 | } | ||||
6052 | case Intrinsic::exp: | ||||
6053 | // exp(log(x)) -> x | ||||
6054 | if (Q.CxtI->hasAllowReassoc() && | ||||
6055 | match(Op0, m_Intrinsic<Intrinsic::log>(m_Value(X)))) | ||||
6056 | return X; | ||||
6057 | break; | ||||
6058 | case Intrinsic::exp2: | ||||
6059 | // exp2(log2(x)) -> x | ||||
6060 | if (Q.CxtI->hasAllowReassoc() && | ||||
6061 | match(Op0, m_Intrinsic<Intrinsic::log2>(m_Value(X)))) | ||||
6062 | return X; | ||||
6063 | break; | ||||
6064 | case Intrinsic::log: | ||||
6065 | // log(exp(x)) -> x | ||||
6066 | if (Q.CxtI->hasAllowReassoc() && | ||||
6067 | match(Op0, m_Intrinsic<Intrinsic::exp>(m_Value(X)))) | ||||
6068 | return X; | ||||
6069 | break; | ||||
6070 | case Intrinsic::log2: | ||||
6071 | // log2(exp2(x)) -> x | ||||
6072 | if (Q.CxtI->hasAllowReassoc() && | ||||
6073 | (match(Op0, m_Intrinsic<Intrinsic::exp2>(m_Value(X))) || | ||||
6074 | match(Op0, | ||||
6075 | m_Intrinsic<Intrinsic::pow>(m_SpecificFP(2.0), m_Value(X))))) | ||||
6076 | return X; | ||||
6077 | break; | ||||
6078 | case Intrinsic::log10: | ||||
6079 | // log10(pow(10.0, x)) -> x | ||||
6080 | if (Q.CxtI->hasAllowReassoc() && | ||||
6081 | match(Op0, m_Intrinsic<Intrinsic::pow>(m_SpecificFP(10.0), m_Value(X)))) | ||||
6082 | return X; | ||||
6083 | break; | ||||
6084 | case Intrinsic::experimental_vector_reverse: | ||||
6085 | // experimental.vector.reverse(experimental.vector.reverse(x)) -> x | ||||
6086 | if (match(Op0, m_VecReverse(m_Value(X)))) | ||||
6087 | return X; | ||||
6088 | // experimental.vector.reverse(splat(X)) -> splat(X) | ||||
6089 | if (isSplatValue(Op0)) | ||||
6090 | return Op0; | ||||
6091 | break; | ||||
6092 | default: | ||||
6093 | break; | ||||
6094 | } | ||||
6095 | |||||
6096 | return nullptr; | ||||
6097 | } | ||||
6098 | |||||
6099 | /// Given a min/max intrinsic, see if it can be removed based on having an | ||||
6100 | /// operand that is another min/max intrinsic with shared operand(s). The caller | ||||
6101 | /// is expected to swap the operand arguments to handle commutation. | ||||
6102 | static Value *foldMinMaxSharedOp(Intrinsic::ID IID, Value *Op0, Value *Op1) { | ||||
6103 | Value *X, *Y; | ||||
6104 | if (!match(Op0, m_MaxOrMin(m_Value(X), m_Value(Y)))) | ||||
6105 | return nullptr; | ||||
6106 | |||||
6107 | auto *MM0 = dyn_cast<IntrinsicInst>(Op0); | ||||
6108 | if (!MM0) | ||||
6109 | return nullptr; | ||||
6110 | Intrinsic::ID IID0 = MM0->getIntrinsicID(); | ||||
6111 | |||||
6112 | if (Op1 == X || Op1 == Y || | ||||
6113 | match(Op1, m_c_MaxOrMin(m_Specific(X), m_Specific(Y)))) { | ||||
6114 | // max (max X, Y), X --> max X, Y | ||||
6115 | if (IID0 == IID) | ||||
6116 | return MM0; | ||||
6117 | // max (min X, Y), X --> X | ||||
6118 | if (IID0 == getInverseMinMaxIntrinsic(IID)) | ||||
6119 | return Op1; | ||||
6120 | } | ||||
6121 | return nullptr; | ||||
6122 | } | ||||
6123 | |||||
6124 | static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1, | ||||
6125 | const SimplifyQuery &Q) { | ||||
6126 | Intrinsic::ID IID = F->getIntrinsicID(); | ||||
6127 | Type *ReturnType = F->getReturnType(); | ||||
6128 | unsigned BitWidth = ReturnType->getScalarSizeInBits(); | ||||
6129 | switch (IID) { | ||||
6130 | case Intrinsic::abs: | ||||
6131 | // abs(abs(x)) -> abs(x). We don't need to worry about the nsw arg here. | ||||
6132 | // It is always ok to pick the earlier abs. We'll just lose nsw if its only | ||||
6133 | // on the outer abs. | ||||
6134 | if (match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(), m_Value()))) | ||||
6135 | return Op0; | ||||
6136 | break; | ||||
6137 | |||||
6138 | case Intrinsic::cttz: { | ||||
6139 | Value *X; | ||||
6140 | if (match(Op0, m_Shl(m_One(), m_Value(X)))) | ||||
6141 | return X; | ||||
6142 | break; | ||||
6143 | } | ||||
6144 | case Intrinsic::ctlz: { | ||||
6145 | Value *X; | ||||
6146 | if (match(Op0, m_LShr(m_Negative(), m_Value(X)))) | ||||
6147 | return X; | ||||
6148 | if (match(Op0, m_AShr(m_Negative(), m_Value()))) | ||||
6149 | return Constant::getNullValue(ReturnType); | ||||
6150 | break; | ||||
6151 | } | ||||
6152 | case Intrinsic::smax: | ||||
6153 | case Intrinsic::smin: | ||||
6154 | case Intrinsic::umax: | ||||
6155 | case Intrinsic::umin: { | ||||
6156 | // If the arguments are the same, this is a no-op. | ||||
6157 | if (Op0 == Op1) | ||||
6158 | return Op0; | ||||
6159 | |||||
6160 | // Canonicalize immediate constant operand as Op1. | ||||
6161 | if (match(Op0, m_ImmConstant())) | ||||
6162 | std::swap(Op0, Op1); | ||||
6163 | |||||
6164 | // Assume undef is the limit value. | ||||
6165 | if (Q.isUndefValue(Op1)) | ||||
6166 | return ConstantInt::get( | ||||
6167 | ReturnType, MinMaxIntrinsic::getSaturationPoint(IID, BitWidth)); | ||||
6168 | |||||
6169 | const APInt *C; | ||||
6170 | if (match(Op1, m_APIntAllowUndef(C))) { | ||||
6171 | // Clamp to limit value. For example: | ||||
6172 | // umax(i8 %x, i8 255) --> 255 | ||||
6173 | if (*C == MinMaxIntrinsic::getSaturationPoint(IID, BitWidth)) | ||||
6174 | return ConstantInt::get(ReturnType, *C); | ||||
6175 | |||||
6176 | // If the constant op is the opposite of the limit value, the other must | ||||
6177 | // be larger/smaller or equal. For example: | ||||
6178 | // umin(i8 %x, i8 255) --> %x | ||||
6179 | if (*C == MinMaxIntrinsic::getSaturationPoint( | ||||
6180 | getInverseMinMaxIntrinsic(IID), BitWidth)) | ||||
6181 | return Op0; | ||||
6182 | |||||
6183 | // Remove nested call if constant operands allow it. Example: | ||||
6184 | // max (max X, 7), 5 -> max X, 7 | ||||
6185 | auto *MinMax0 = dyn_cast<IntrinsicInst>(Op0); | ||||
6186 | if (MinMax0 && MinMax0->getIntrinsicID() == IID) { | ||||
6187 | // TODO: loosen undef/splat restrictions for vector constants. | ||||
6188 | Value *M00 = MinMax0->getOperand(0), *M01 = MinMax0->getOperand(1); | ||||
6189 | const APInt *InnerC; | ||||
6190 | if ((match(M00, m_APInt(InnerC)) || match(M01, m_APInt(InnerC))) && | ||||
6191 | ICmpInst::compare(*InnerC, *C, | ||||
6192 | ICmpInst::getNonStrictPredicate( | ||||
6193 | MinMaxIntrinsic::getPredicate(IID)))) | ||||
6194 | return Op0; | ||||
6195 | } | ||||
6196 | } | ||||
6197 | |||||
6198 | if (Value *V = foldMinMaxSharedOp(IID, Op0, Op1)) | ||||
6199 | return V; | ||||
6200 | if (Value *V = foldMinMaxSharedOp(IID, Op1, Op0)) | ||||
6201 | return V; | ||||
6202 | |||||
6203 | ICmpInst::Predicate Pred = | ||||
6204 | ICmpInst::getNonStrictPredicate(MinMaxIntrinsic::getPredicate(IID)); | ||||
6205 | if (isICmpTrue(Pred, Op0, Op1, Q.getWithoutUndef(), RecursionLimit)) | ||||
6206 | return Op0; | ||||
6207 | if (isICmpTrue(Pred, Op1, Op0, Q.getWithoutUndef(), RecursionLimit)) | ||||
6208 | return Op1; | ||||
6209 | |||||
6210 | break; | ||||
6211 | } | ||||
6212 | case Intrinsic::usub_with_overflow: | ||||
6213 | case Intrinsic::ssub_with_overflow: | ||||
6214 | // X - X -> { 0, false } | ||||
6215 | // X - undef -> { 0, false } | ||||
6216 | // undef - X -> { 0, false } | ||||
6217 | if (Op0 == Op1 || Q.isUndefValue(Op0) || Q.isUndefValue(Op1)) | ||||
6218 | return Constant::getNullValue(ReturnType); | ||||
6219 | break; | ||||
6220 | case Intrinsic::uadd_with_overflow: | ||||
6221 | case Intrinsic::sadd_with_overflow: | ||||
6222 | // X + undef -> { -1, false } | ||||
6223 | // undef + x -> { -1, false } | ||||
6224 | if (Q.isUndefValue(Op0) || Q.isUndefValue(Op1)) { | ||||
6225 | return ConstantStruct::get( | ||||
6226 | cast<StructType>(ReturnType), | ||||
6227 | {Constant::getAllOnesValue(ReturnType->getStructElementType(0)), | ||||
6228 | Constant::getNullValue(ReturnType->getStructElementType(1))}); | ||||
6229 | } | ||||
6230 | break; | ||||
6231 | case Intrinsic::umul_with_overflow: | ||||
6232 | case Intrinsic::smul_with_overflow: | ||||
6233 | // 0 * X -> { 0, false } | ||||
6234 | // X * 0 -> { 0, false } | ||||
6235 | if (match(Op0, m_Zero()) || match(Op1, m_Zero())) | ||||
6236 | return Constant::getNullValue(ReturnType); | ||||
6237 | // undef * X -> { 0, false } | ||||
6238 | // X * undef -> { 0, false } | ||||
6239 | if (Q.isUndefValue(Op0) || Q.isUndefValue(Op1)) | ||||
6240 | return Constant::getNullValue(ReturnType); | ||||
6241 | break; | ||||
6242 | case Intrinsic::uadd_sat: | ||||
6243 | // sat(MAX + X) -> MAX | ||||
6244 | // sat(X + MAX) -> MAX | ||||
6245 | if (match(Op0, m_AllOnes()) || match(Op1, m_AllOnes())) | ||||
6246 | return Constant::getAllOnesValue(ReturnType); | ||||
6247 | [[fallthrough]]; | ||||
6248 | case Intrinsic::sadd_sat: | ||||
6249 | // sat(X + undef) -> -1 | ||||
6250 | // sat(undef + X) -> -1 | ||||
6251 | // For unsigned: Assume undef is MAX, thus we saturate to MAX (-1). | ||||
6252 | // For signed: Assume undef is ~X, in which case X + ~X = -1. | ||||
6253 | if (Q.isUndefValue(Op0) || Q.isUndefValue(Op1)) | ||||
6254 | return Constant::getAllOnesValue(ReturnType); | ||||
6255 | |||||
6256 | // X + 0 -> X | ||||
6257 | if (match(Op1, m_Zero())) | ||||
6258 | return Op0; | ||||
6259 | // 0 + X -> X | ||||
6260 | if (match(Op0, m_Zero())) | ||||
6261 | return Op1; | ||||
6262 | break; | ||||
6263 | case Intrinsic::usub_sat: | ||||
6264 | // sat(0 - X) -> 0, sat(X - MAX) -> 0 | ||||
6265 | if (match(Op0, m_Zero()) || match(Op1, m_AllOnes())) | ||||
6266 | return Constant::getNullValue(ReturnType); | ||||
6267 | [[fallthrough]]; | ||||
6268 | case Intrinsic::ssub_sat: | ||||
6269 | // X - X -> 0, X - undef -> 0, undef - X -> 0 | ||||
6270 | if (Op0 == Op1 || Q.isUndefValue(Op0) || Q.isUndefValue(Op1)) | ||||
6271 | return Constant::getNullValue(ReturnType); | ||||
6272 | // X - 0 -> X | ||||
6273 | if (match(Op1, m_Zero())) | ||||
6274 | return Op0; | ||||
6275 | break; | ||||
6276 | case Intrinsic::load_relative: | ||||
6277 | if (auto *C0 = dyn_cast<Constant>(Op0)) | ||||
6278 | if (auto *C1 = dyn_cast<Constant>(Op1)) | ||||
6279 | return simplifyRelativeLoad(C0, C1, Q.DL); | ||||
6280 | break; | ||||
6281 | case Intrinsic::powi: | ||||
6282 | if (auto *Power = dyn_cast<ConstantInt>(Op1)) { | ||||
6283 | // powi(x, 0) -> 1.0 | ||||
6284 | if (Power->isZero()) | ||||
6285 | return ConstantFP::get(Op0->getType(), 1.0); | ||||
6286 | // powi(x, 1) -> x | ||||
6287 | if (Power->isOne()) | ||||
6288 | return Op0; | ||||
6289 | } | ||||
6290 | break; | ||||
6291 | case Intrinsic::copysign: | ||||
6292 | // copysign X, X --> X | ||||
6293 | if (Op0 == Op1) | ||||
6294 | return Op0; | ||||
6295 | // copysign -X, X --> X | ||||
6296 | // copysign X, -X --> -X | ||||
6297 | if (match(Op0, m_FNeg(m_Specific(Op1))) || | ||||
6298 | match(Op1, m_FNeg(m_Specific(Op0)))) | ||||
6299 | return Op1; | ||||
6300 | break; | ||||
6301 | case Intrinsic::is_fpclass: { | ||||
6302 | if (isa<PoisonValue>(Op0)) | ||||
6303 | return PoisonValue::get(ReturnType); | ||||
6304 | |||||
6305 | uint64_t Mask = cast<ConstantInt>(Op1)->getZExtValue(); | ||||
6306 | // If all tests are made, it doesn't matter what the value is. | ||||
6307 | if ((Mask & fcAllFlags) == fcAllFlags) | ||||
6308 | return ConstantInt::get(ReturnType, true); | ||||
6309 | if ((Mask & fcAllFlags) == 0) | ||||
6310 | return ConstantInt::get(ReturnType, false); | ||||
6311 | if (Q.isUndefValue(Op0)) | ||||
6312 | return UndefValue::get(ReturnType); | ||||
6313 | break; | ||||
6314 | } | ||||
6315 | case Intrinsic::maxnum: | ||||
6316 | case Intrinsic::minnum: | ||||
6317 | case Intrinsic::maximum: | ||||
6318 | case Intrinsic::minimum: { | ||||
6319 | // If the arguments are the same, this is a no-op. | ||||
6320 | if (Op0 == Op1) | ||||
6321 | return Op0; | ||||
6322 | |||||
6323 | // Canonicalize constant operand as Op1. | ||||
6324 | if (isa<Constant>(Op0)) | ||||
6325 | std::swap(Op0, Op1); | ||||
6326 | |||||
6327 | // If an argument is undef, return the other argument. | ||||
6328 | if (Q.isUndefValue(Op1)) | ||||
6329 | return Op0; | ||||
6330 | |||||
6331 | bool PropagateNaN = IID == Intrinsic::minimum || IID == Intrinsic::maximum; | ||||
6332 | bool IsMin = IID == Intrinsic::minimum || IID == Intrinsic::minnum; | ||||
6333 | |||||
6334 | // minnum(X, nan) -> X | ||||
6335 | // maxnum(X, nan) -> X | ||||
6336 | // minimum(X, nan) -> nan | ||||
6337 | // maximum(X, nan) -> nan | ||||
6338 | if (match(Op1, m_NaN())) | ||||
6339 | return PropagateNaN ? propagateNaN(cast<Constant>(Op1)) : Op0; | ||||
6340 | |||||
6341 | // In the following folds, inf can be replaced with the largest finite | ||||
6342 | // float, if the ninf flag is set. | ||||
6343 | const APFloat *C; | ||||
6344 | if (match(Op1, m_APFloat(C)) && | ||||
6345 | (C->isInfinity() || (Q.CxtI->hasNoInfs() && C->isLargest()))) { | ||||
6346 | // minnum(X, -inf) -> -inf | ||||
6347 | // maxnum(X, +inf) -> +inf | ||||
6348 | // minimum(X, -inf) -> -inf if nnan | ||||
6349 | // maximum(X, +inf) -> +inf if nnan | ||||
6350 | if (C->isNegative() == IsMin && (!PropagateNaN || Q.CxtI->hasNoNaNs())) | ||||
6351 | return ConstantFP::get(ReturnType, *C); | ||||
6352 | |||||
6353 | // minnum(X, +inf) -> X if nnan | ||||
6354 | // maxnum(X, -inf) -> X if nnan | ||||
6355 | // minimum(X, +inf) -> X | ||||
6356 | // maximum(X, -inf) -> X | ||||
6357 | if (C->isNegative() != IsMin && (PropagateNaN || Q.CxtI->hasNoNaNs())) | ||||
6358 | return Op0; | ||||
6359 | } | ||||
6360 | |||||
6361 | // Min/max of the same operation with common operand: | ||||
6362 | // m(m(X, Y)), X --> m(X, Y) (4 commuted variants) | ||||
6363 | if (auto *M0 = dyn_cast<IntrinsicInst>(Op0)) | ||||
6364 | if (M0->getIntrinsicID() == IID && | ||||
6365 | (M0->getOperand(0) == Op1 || M0->getOperand(1) == Op1)) | ||||
6366 | return Op0; | ||||
6367 | if (auto *M1 = dyn_cast<IntrinsicInst>(Op1)) | ||||
6368 | if (M1->getIntrinsicID() == IID && | ||||
6369 | (M1->getOperand(0) == Op0 || M1->getOperand(1) == Op0)) | ||||
6370 | return Op1; | ||||
6371 | |||||
6372 | break; | ||||
6373 | } | ||||
6374 | case Intrinsic::vector_extract: { | ||||
6375 | Type *ReturnType = F->getReturnType(); | ||||
6376 | |||||
6377 | // (extract_vector (insert_vector _, X, 0), 0) -> X | ||||
6378 | unsigned IdxN = cast<ConstantInt>(Op1)->getZExtValue(); | ||||
6379 | Value *X = nullptr; | ||||
6380 | if (match(Op0, m_Intrinsic<Intrinsic::vector_insert>(m_Value(), m_Value(X), | ||||
6381 | m_Zero())) && | ||||
6382 | IdxN == 0 && X->getType() == ReturnType) | ||||
6383 | return X; | ||||
6384 | |||||
6385 | break; | ||||
6386 | } | ||||
6387 | default: | ||||
6388 | break; | ||||
6389 | } | ||||
6390 | |||||
6391 | return nullptr; | ||||
6392 | } | ||||
6393 | |||||
6394 | static Value *simplifyIntrinsic(CallBase *Call, const SimplifyQuery &Q) { | ||||
6395 | |||||
6396 | unsigned NumOperands = Call->arg_size(); | ||||
6397 | Function *F = cast<Function>(Call->getCalledFunction()); | ||||
6398 | Intrinsic::ID IID = F->getIntrinsicID(); | ||||
6399 | |||||
6400 | // Most of the intrinsics with no operands have some kind of side effect. | ||||
6401 | // Don't simplify. | ||||
6402 | if (!NumOperands) { | ||||
6403 | switch (IID) { | ||||
6404 | case Intrinsic::vscale: { | ||||
6405 | // Call may not be inserted into the IR yet at point of calling simplify. | ||||
6406 | if (!Call->getParent() || !Call->getParent()->getParent()) | ||||
6407 | return nullptr; | ||||
6408 | auto Attr = Call->getFunction()->getFnAttribute(Attribute::VScaleRange); | ||||
6409 | if (!Attr.isValid()) | ||||
6410 | return nullptr; | ||||
6411 | unsigned VScaleMin = Attr.getVScaleRangeMin(); | ||||
6412 | std::optional<unsigned> VScaleMax = Attr.getVScaleRangeMax(); | ||||
6413 | if (VScaleMax && VScaleMin == VScaleMax) | ||||
6414 | return ConstantInt::get(F->getReturnType(), VScaleMin); | ||||
6415 | return nullptr; | ||||
6416 | } | ||||
6417 | default: | ||||
6418 | return nullptr; | ||||
6419 | } | ||||
6420 | } | ||||
6421 | |||||
6422 | if (NumOperands == 1) | ||||
6423 | return simplifyUnaryIntrinsic(F, Call->getArgOperand(0), Q); | ||||
6424 | |||||
6425 | if (NumOperands == 2) | ||||
6426 | return simplifyBinaryIntrinsic(F, Call->getArgOperand(0), | ||||
6427 | Call->getArgOperand(1), Q); | ||||
6428 | |||||
6429 | // Handle intrinsics with 3 or more arguments. | ||||
6430 | switch (IID) { | ||||
6431 | case Intrinsic::masked_load: | ||||
6432 | case Intrinsic::masked_gather: { | ||||
6433 | Value *MaskArg = Call->getArgOperand(2); | ||||
6434 | Value *PassthruArg = Call->getArgOperand(3); | ||||
6435 | // If the mask is all zeros or undef, the "passthru" argument is the result. | ||||
6436 | if (maskIsAllZeroOrUndef(MaskArg)) | ||||
6437 | return PassthruArg; | ||||
6438 | return nullptr; | ||||
6439 | } | ||||
6440 | case Intrinsic::fshl: | ||||
6441 | case Intrinsic::fshr: { | ||||
6442 | Value *Op0 = Call->getArgOperand(0), *Op1 = Call->getArgOperand(1), | ||||
6443 | *ShAmtArg = Call->getArgOperand(2); | ||||
6444 | |||||
6445 | // If both operands are undef, the result is undef. | ||||
6446 | if (Q.isUndefValue(Op0) && Q.isUndefValue(Op1)) | ||||
6447 | return UndefValue::get(F->getReturnType()); | ||||
6448 | |||||
6449 | // If shift amount is undef, assume it is zero. | ||||
6450 | if (Q.isUndefValue(ShAmtArg)) | ||||
6451 | return Call->getArgOperand(IID == Intrinsic::fshl ? 0 : 1); | ||||
6452 | |||||
6453 | const APInt *ShAmtC; | ||||
6454 | if (match(ShAmtArg, m_APInt(ShAmtC))) { | ||||
6455 | // If there's effectively no shift, return the 1st arg or 2nd arg. | ||||
6456 | APInt BitWidth = APInt(ShAmtC->getBitWidth(), ShAmtC->getBitWidth()); | ||||
6457 | if (ShAmtC->urem(BitWidth).isZero()) | ||||
6458 | return Call->getArgOperand(IID == Intrinsic::fshl ? 0 : 1); | ||||
6459 | } | ||||
6460 | |||||
6461 | // Rotating zero by anything is zero. | ||||
6462 | if (match(Op0, m_Zero()) && match(Op1, m_Zero())) | ||||
6463 | return ConstantInt::getNullValue(F->getReturnType()); | ||||
6464 | |||||
6465 | // Rotating -1 by anything is -1. | ||||
6466 | if (match(Op0, m_AllOnes()) && match(Op1, m_AllOnes())) | ||||
6467 | return ConstantInt::getAllOnesValue(F->getReturnType()); | ||||
6468 | |||||
6469 | return nullptr; | ||||
6470 | } | ||||
6471 | case Intrinsic::experimental_constrained_fma: { | ||||
6472 | Value *Op0 = Call->getArgOperand(0); | ||||
6473 | Value *Op1 = Call->getArgOperand(1); | ||||
6474 | Value *Op2 = Call->getArgOperand(2); | ||||
6475 | auto *FPI = cast<ConstrainedFPIntrinsic>(Call); | ||||
6476 | if (Value *V = | ||||
6477 | simplifyFPOp({Op0, Op1, Op2}, {}, Q, *FPI->getExceptionBehavior(), | ||||
6478 | *FPI->getRoundingMode())) | ||||
6479 | return V; | ||||
6480 | return nullptr; | ||||
6481 | } | ||||
6482 | case Intrinsic::fma: | ||||
6483 | case Intrinsic::fmuladd: { | ||||
6484 | Value *Op0 = Call->getArgOperand(0); | ||||
6485 | Value *Op1 = Call->getArgOperand(1); | ||||
6486 | Value *Op2 = Call->getArgOperand(2); | ||||
6487 | if (Value *V = simplifyFPOp({Op0, Op1, Op2}, {}, Q, fp::ebIgnore, | ||||
6488 | RoundingMode::NearestTiesToEven)) | ||||
6489 | return V; | ||||
6490 | return nullptr; | ||||
6491 | } | ||||
6492 | case Intrinsic::smul_fix: | ||||
6493 | case Intrinsic::smul_fix_sat: { | ||||
6494 | Value *Op0 = Call->getArgOperand(0); | ||||
6495 | Value *Op1 = Call->getArgOperand(1); | ||||
6496 | Value *Op2 = Call->getArgOperand(2); | ||||
6497 | Type *ReturnType = F->getReturnType(); | ||||
6498 | |||||
6499 | // Canonicalize constant operand as Op1 (ConstantFolding handles the case | ||||
6500 | // when both Op0 and Op1 are constant so we do not care about that special | ||||
6501 | // case here). | ||||
6502 | if (isa<Constant>(Op0)) | ||||
6503 | std::swap(Op0, Op1); | ||||
6504 | |||||
6505 | // X * 0 -> 0 | ||||
6506 | if (match(Op1, m_Zero())) | ||||
6507 | return Constant::getNullValue(ReturnType); | ||||
6508 | |||||
6509 | // X * undef -> 0 | ||||
6510 | if (Q.isUndefValue(Op1)) | ||||
6511 | return Constant::getNullValue(ReturnType); | ||||
6512 | |||||
6513 | // X * (1 << Scale) -> X | ||||
6514 | APInt ScaledOne = | ||||
6515 | APInt::getOneBitSet(ReturnType->getScalarSizeInBits(), | ||||
6516 | cast<ConstantInt>(Op2)->getZExtValue()); | ||||
6517 | if (ScaledOne.isNonNegative() && match(Op1, m_SpecificInt(ScaledOne))) | ||||
6518 | return Op0; | ||||
6519 | |||||
6520 | return nullptr; | ||||
6521 | } | ||||
6522 | case Intrinsic::vector_insert: { | ||||
6523 | Value *Vec = Call->getArgOperand(0); | ||||
6524 | Value *SubVec = Call->getArgOperand(1); | ||||
6525 | Value *Idx = Call->getArgOperand(2); | ||||
6526 | Type *ReturnType = F->getReturnType(); | ||||
6527 | |||||
6528 | // (insert_vector Y, (extract_vector X, 0), 0) -> X | ||||
6529 | // where: Y is X, or Y is undef | ||||
6530 | unsigned IdxN = cast<ConstantInt>(Idx)->getZExtValue(); | ||||
6531 | Value *X = nullptr; | ||||
6532 | if (match(SubVec, | ||||
6533 | m_Intrinsic<Intrinsic::vector_extract>(m_Value(X), m_Zero())) && | ||||
6534 | (Q.isUndefValue(Vec) || Vec == X) && IdxN == 0 && | ||||
6535 | X->getType() == ReturnType) | ||||
| |||||
6536 | return X; | ||||
6537 | |||||
6538 | return nullptr; | ||||
6539 | } | ||||
6540 | case Intrinsic::experimental_constrained_fadd: { | ||||
6541 | auto *FPI = cast<ConstrainedFPIntrinsic>(Call); | ||||
6542 | return simplifyFAddInst( | ||||
6543 | FPI->getArgOperand(0), FPI->getArgOperand(1), FPI->getFastMathFlags(), | ||||
6544 | Q, *FPI->getExceptionBehavior(), *FPI->getRoundingMode()); | ||||
6545 | } | ||||
6546 | case Intrinsic::experimental_constrained_fsub: { | ||||
6547 | auto *FPI = cast<ConstrainedFPIntrinsic>(Call); | ||||
6548 | return simplifyFSubInst( | ||||
6549 | FPI->getArgOperand(0), FPI->getArgOperand(1), FPI->getFastMathFlags(), | ||||
6550 | Q, *FPI->getExceptionBehavior(), *FPI->getRoundingMode()); | ||||
6551 | } | ||||
6552 | case Intrinsic::experimental_constrained_fmul: { | ||||
6553 | auto *FPI = cast<ConstrainedFPIntrinsic>(Call); | ||||
6554 | return simplifyFMulInst( | ||||
6555 | FPI->getArgOperand(0), FPI->getArgOperand(1), FPI->getFastMathFlags(), | ||||
6556 | Q, *FPI->getExceptionBehavior(), *FPI->getRoundingMode()); | ||||
6557 | } | ||||
6558 | case Intrinsic::experimental_constrained_fdiv: { | ||||
6559 | auto *FPI = cast<ConstrainedFPIntrinsic>(Call); | ||||
6560 | return simplifyFDivInst( | ||||
6561 | FPI->getArgOperand(0), FPI->getArgOperand(1), FPI->getFastMathFlags(), | ||||
6562 | Q, *FPI->getExceptionBehavior(), *FPI->getRoundingMode()); | ||||
6563 | } | ||||
6564 | case Intrinsic::experimental_constrained_frem: { | ||||
6565 | auto *FPI = cast<ConstrainedFPIntrinsic>(Call); | ||||
6566 | return simplifyFRemInst( | ||||
6567 | FPI->getArgOperand(0), FPI->getArgOperand(1), FPI->getFastMathFlags(), | ||||
6568 | Q, *FPI->getExceptionBehavior(), *FPI->getRoundingMode()); | ||||
6569 | } | ||||
6570 | default: | ||||
6571 | return nullptr; | ||||
6572 | } | ||||
6573 | } | ||||
6574 | |||||
6575 | static Value *tryConstantFoldCall(CallBase *Call, const SimplifyQuery &Q) { | ||||
6576 | auto *F = dyn_cast<Function>(Call->getCalledOperand()); | ||||
6577 | if (!F || !canConstantFoldCallTo(Call, F)) | ||||
6578 | return nullptr; | ||||
6579 | |||||
6580 | SmallVector<Constant *, 4> ConstantArgs; | ||||
6581 | unsigned NumArgs = Call->arg_size(); | ||||
6582 | ConstantArgs.reserve(NumArgs); | ||||
6583 | for (auto &Arg : Call->args()) { | ||||
6584 | Constant *C = dyn_cast<Constant>(&Arg); | ||||
6585 | if (!C) { | ||||
6586 | if (isa<MetadataAsValue>(Arg.get())) | ||||
6587 | continue; | ||||
6588 | return nullptr; | ||||
6589 | } | ||||
6590 | ConstantArgs.push_back(C); | ||||
6591 | } | ||||
6592 | |||||
6593 | return ConstantFoldCall(Call, F, ConstantArgs, Q.TLI); | ||||
6594 | } | ||||
6595 | |||||
6596 | Value *llvm::simplifyCall(CallBase *Call, const SimplifyQuery &Q) { | ||||
6597 | // musttail calls can only be simplified if they are also DCEd. | ||||
6598 | // As we can't guarantee this here, don't simplify them. | ||||
6599 | if (Call->isMustTailCall()) | ||||
6600 | return nullptr; | ||||
6601 | |||||
6602 | // call undef -> poison | ||||
6603 | // call null -> poison | ||||
6604 | Value *Callee = Call->getCalledOperand(); | ||||
6605 | if (isa<UndefValue>(Callee) || isa<ConstantPointerNull>(Callee)) | ||||
6606 | return PoisonValue::get(Call->getType()); | ||||
6607 | |||||
6608 | if (Value *V = tryConstantFoldCall(Call, Q)) | ||||
6609 | return V; | ||||
6610 | |||||
6611 | auto *F = dyn_cast<Function>(Callee); | ||||
6612 | if (F && F->isIntrinsic()) | ||||
6613 | if (Value *Ret = simplifyIntrinsic(Call, Q)) | ||||
6614 | return Ret; | ||||
6615 | |||||
6616 | return nullptr; | ||||
6617 | } | ||||
6618 | |||||
6619 | Value *llvm::simplifyConstrainedFPCall(CallBase *Call, const SimplifyQuery &Q) { | ||||
6620 | assert(isa<ConstrainedFPIntrinsic>(Call))(static_cast <bool> (isa<ConstrainedFPIntrinsic>( Call)) ? void (0) : __assert_fail ("isa<ConstrainedFPIntrinsic>(Call)" , "llvm/lib/Analysis/InstructionSimplify.cpp", 6620, __extension__ __PRETTY_FUNCTION__)); | ||||
| |||||
6621 | if (Value *V
| ||||
6622 | return V; | ||||
6623 | if (Value *Ret = simplifyIntrinsic(Call, Q)) | ||||
6624 | return Ret; | ||||
6625 | return nullptr; | ||||
6626 | } | ||||
6627 | |||||
6628 | /// Given operands for a Freeze, see if we can fold the result. | ||||
6629 | static Value *simplifyFreezeInst(Value *Op0, const SimplifyQuery &Q) { | ||||
6630 | // Use a utility function defined in ValueTracking. | ||||
6631 | if (llvm::isGuaranteedNotToBeUndefOrPoison(Op0, Q.AC, Q.CxtI, Q.DT)) | ||||
6632 | return Op0; | ||||
6633 | // We have room for improvement. | ||||
6634 | return nullptr; | ||||
6635 | } | ||||
6636 | |||||
6637 | Value *llvm::simplifyFreezeInst(Value *Op0, const SimplifyQuery &Q) { | ||||
6638 | return ::simplifyFreezeInst(Op0, Q); | ||||
6639 | } | ||||
6640 | |||||
6641 | Value *llvm::simplifyLoadInst(LoadInst *LI, Value *PtrOp, | ||||
6642 | const SimplifyQuery &Q) { | ||||
6643 | if (LI->isVolatile()) | ||||
6644 | return nullptr; | ||||
6645 | |||||
6646 | if (auto *PtrOpC = dyn_cast<Constant>(PtrOp)) | ||||
6647 | return ConstantFoldLoadFromConstPtr(PtrOpC, LI->getType(), Q.DL); | ||||
6648 | |||||
6649 | // We can only fold the load if it is from a constant global with definitive | ||||
6650 | // initializer. Skip expensive logic if this is not the case. | ||||
6651 | auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(PtrOp)); | ||||
6652 | if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer()) | ||||
6653 | return nullptr; | ||||
6654 | |||||
6655 | // If GlobalVariable's initializer is uniform, then return the constant | ||||
6656 | // regardless of its offset. | ||||
6657 | if (Constant *C = | ||||
6658 | ConstantFoldLoadFromUniformValue(GV->getInitializer(), LI->getType())) | ||||
6659 | return C; | ||||
6660 | |||||
6661 | // Try to convert operand into a constant by stripping offsets while looking | ||||
6662 | // through invariant.group intrinsics. | ||||
6663 | APInt Offset(Q.DL.getIndexTypeSizeInBits(PtrOp->getType()), 0); | ||||
6664 | PtrOp = PtrOp->stripAndAccumulateConstantOffsets( | ||||
6665 | Q.DL, Offset, /* AllowNonInbounts */ true, | ||||
6666 | /* AllowInvariantGroup */ true); | ||||
6667 | if (PtrOp == GV) { | ||||
6668 | // Index size may have changed due to address space casts. | ||||
6669 | Offset = Offset.sextOrTrunc(Q.DL.getIndexTypeSizeInBits(PtrOp->getType())); | ||||
6670 | return ConstantFoldLoadFromConstPtr(GV, LI->getType(), Offset, Q.DL); | ||||
6671 | } | ||||
6672 | |||||
6673 | return nullptr; | ||||
6674 | } | ||||
6675 | |||||
6676 | /// See if we can compute a simplified version of this instruction. | ||||
6677 | /// If not, this returns null. | ||||
6678 | |||||
6679 | static Value *simplifyInstructionWithOperands(Instruction *I, | ||||
6680 | ArrayRef<Value *> NewOps, | ||||
6681 | const SimplifyQuery &SQ, | ||||
6682 | OptimizationRemarkEmitter *ORE) { | ||||
6683 | const SimplifyQuery Q = SQ.CxtI ? SQ : SQ.getWithInstruction(I); | ||||
6684 | |||||
6685 | switch (I->getOpcode()) { | ||||
6686 | default: | ||||
6687 | if (llvm::all_of(NewOps, [](Value *V) { return isa<Constant>(V); })) { | ||||
6688 | SmallVector<Constant *, 8> NewConstOps(NewOps.size()); | ||||
6689 | transform(NewOps, NewConstOps.begin(), | ||||
6690 | [](Value *V) { return cast<Constant>(V); }); | ||||
6691 | return ConstantFoldInstOperands(I, NewConstOps, Q.DL, Q.TLI); | ||||
6692 | } | ||||
6693 | return nullptr; | ||||
6694 | case Instruction::FNeg: | ||||
6695 | return simplifyFNegInst(NewOps[0], I->getFastMathFlags(), Q); | ||||
6696 | case Instruction::FAdd: | ||||
6697 | return simplifyFAddInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); | ||||
6698 | case Instruction::Add: | ||||
6699 | return simplifyAddInst(NewOps[0], NewOps[1], | ||||
6700 | Q.IIQ.hasNoSignedWrap(cast<BinaryOperator>(I)), | ||||
6701 | Q.IIQ.hasNoUnsignedWrap(cast<BinaryOperator>(I)), Q); | ||||
6702 | case Instruction::FSub: | ||||
6703 | return simplifyFSubInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); | ||||
6704 | case Instruction::Sub: | ||||
6705 | return simplifySubInst(NewOps[0], NewOps[1], | ||||
6706 | Q.IIQ.hasNoSignedWrap(cast<BinaryOperator>(I)), | ||||
6707 | Q.IIQ.hasNoUnsignedWrap(cast<BinaryOperator>(I)), Q); | ||||
6708 | case Instruction::FMul: | ||||
6709 | return simplifyFMulInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); | ||||
6710 | case Instruction::Mul: | ||||
6711 | return simplifyMulInst(NewOps[0], NewOps[1], | ||||
6712 | Q.IIQ.hasNoSignedWrap(cast<BinaryOperator>(I)), | ||||
6713 | Q.IIQ.hasNoUnsignedWrap(cast<BinaryOperator>(I)), Q); | ||||
6714 | case Instruction::SDiv: | ||||
6715 | return simplifySDivInst(NewOps[0], NewOps[1], | ||||
6716 | Q.IIQ.isExact(cast<BinaryOperator>(I)), Q); | ||||
6717 | case Instruction::UDiv: | ||||
6718 | return simplifyUDivInst(NewOps[0], NewOps[1], | ||||
6719 | Q.IIQ.isExact(cast<BinaryOperator>(I)), Q); | ||||
6720 | case Instruction::FDiv: | ||||
6721 | return simplifyFDivInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); | ||||
6722 | case Instruction::SRem: | ||||
6723 | return simplifySRemInst(NewOps[0], NewOps[1], Q); | ||||
6724 | case Instruction::URem: | ||||
6725 | return simplifyURemInst(NewOps[0], NewOps[1], Q); | ||||
6726 | case Instruction::FRem: | ||||
6727 | return simplifyFRemInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); | ||||
6728 | case Instruction::Shl: | ||||
6729 | return simplifyShlInst(NewOps[0], NewOps[1], | ||||
6730 | Q.IIQ.hasNoSignedWrap(cast<BinaryOperator>(I)), | ||||
6731 | Q.IIQ.hasNoUnsignedWrap(cast<BinaryOperator>(I)), Q); | ||||
6732 | case Instruction::LShr: | ||||
6733 | return simplifyLShrInst(NewOps[0], NewOps[1], | ||||
6734 | Q.IIQ.isExact(cast<BinaryOperator>(I)), Q); | ||||
6735 | case Instruction::AShr: | ||||
6736 | return simplifyAShrInst(NewOps[0], NewOps[1], | ||||
6737 | Q.IIQ.isExact(cast<BinaryOperator>(I)), Q); | ||||
6738 | case Instruction::And: | ||||
6739 | return simplifyAndInst(NewOps[0], NewOps[1], Q); | ||||
6740 | case Instruction::Or: | ||||
6741 | return simplifyOrInst(NewOps[0], NewOps[1], Q); | ||||
6742 | case Instruction::Xor: | ||||
6743 | return simplifyXorInst(NewOps[0], NewOps[1], Q); | ||||
6744 | case Instruction::ICmp: | ||||
6745 | return simplifyICmpInst(cast<ICmpInst>(I)->getPredicate(), NewOps[0], | ||||
6746 | NewOps[1], Q); | ||||
6747 | case Instruction::FCmp: | ||||
6748 | return simplifyFCmpInst(cast<FCmpInst>(I)->getPredicate(), NewOps[0], | ||||
6749 | NewOps[1], I->getFastMathFlags(), Q); | ||||
6750 | case Instruction::Select: | ||||
6751 | return simplifySelectInst(NewOps[0], NewOps[1], NewOps[2], Q); | ||||
6752 | break; | ||||
6753 | case Instruction::GetElementPtr: { | ||||
6754 | auto *GEPI = cast<GetElementPtrInst>(I); | ||||
6755 | return simplifyGEPInst(GEPI->getSourceElementType(), NewOps[0], | ||||
6756 | ArrayRef(NewOps).slice(1), GEPI->isInBounds(), Q); | ||||
6757 | } | ||||
6758 | case Instruction::InsertValue: { | ||||
6759 | InsertValueInst *IV = cast<InsertValueInst>(I); | ||||
6760 | return simplifyInsertValueInst(NewOps[0], NewOps[1], IV->getIndices(), Q); | ||||
6761 | } | ||||
6762 | case Instruction::InsertElement: | ||||
6763 | return simplifyInsertElementInst(NewOps[0], NewOps[1], NewOps[2], Q); | ||||
6764 | case Instruction::ExtractValue: { | ||||
6765 | auto *EVI = cast<ExtractValueInst>(I); | ||||
6766 | return simplifyExtractValueInst(NewOps[0], EVI->getIndices(), Q); | ||||
6767 | } | ||||
6768 | case Instruction::ExtractElement: | ||||
6769 | return simplifyExtractElementInst(NewOps[0], NewOps[1], Q); | ||||
6770 | case Instruction::ShuffleVector: { | ||||
6771 | auto *SVI = cast<ShuffleVectorInst>(I); | ||||
6772 | return simplifyShuffleVectorInst(NewOps[0], NewOps[1], | ||||
6773 | SVI->getShuffleMask(), SVI->getType(), Q); | ||||
6774 | } | ||||
6775 | case Instruction::PHI: | ||||
6776 | return simplifyPHINode(cast<PHINode>(I), NewOps, Q); | ||||
6777 | case Instruction::Call: | ||||
6778 | // TODO: Use NewOps | ||||
6779 | return simplifyCall(cast<CallInst>(I), Q); | ||||
6780 | case Instruction::Freeze: | ||||
6781 | return llvm::simplifyFreezeInst(NewOps[0], Q); | ||||
6782 | #define HANDLE_CAST_INST(num, opc, clas) case Instruction::opc: | ||||
6783 | #include "llvm/IR/Instruction.def" | ||||
6784 | #undef HANDLE_CAST_INST | ||||
6785 | return simplifyCastInst(I->getOpcode(), NewOps[0], I->getType(), Q); | ||||
6786 | case Instruction::Alloca: | ||||
6787 | // No simplifications for Alloca and it can't be constant folded. | ||||
6788 | return nullptr; | ||||
6789 | case Instruction::Load: | ||||
6790 | return simplifyLoadInst(cast<LoadInst>(I), NewOps[0], Q); | ||||
6791 | } | ||||
6792 | } | ||||
6793 | |||||
6794 | Value *llvm::simplifyInstructionWithOperands(Instruction *I, | ||||
6795 | ArrayRef<Value *> NewOps, | ||||
6796 | const SimplifyQuery &SQ, | ||||
6797 | OptimizationRemarkEmitter *ORE) { | ||||
6798 | assert(NewOps.size() == I->getNumOperands() &&(static_cast <bool> (NewOps.size() == I->getNumOperands () && "Number of operands should match the instruction!" ) ? void (0) : __assert_fail ("NewOps.size() == I->getNumOperands() && \"Number of operands should match the instruction!\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 6799, __extension__ __PRETTY_FUNCTION__)) | ||||
6799 | "Number of operands should match the instruction!")(static_cast <bool> (NewOps.size() == I->getNumOperands () && "Number of operands should match the instruction!" ) ? void (0) : __assert_fail ("NewOps.size() == I->getNumOperands() && \"Number of operands should match the instruction!\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 6799, __extension__ __PRETTY_FUNCTION__)); | ||||
6800 | return ::simplifyInstructionWithOperands(I, NewOps, SQ, ORE); | ||||
6801 | } | ||||
6802 | |||||
6803 | Value *llvm::simplifyInstruction(Instruction *I, const SimplifyQuery &SQ, | ||||
6804 | OptimizationRemarkEmitter *ORE) { | ||||
6805 | SmallVector<Value *, 8> Ops(I->operands()); | ||||
6806 | Value *Result = ::simplifyInstructionWithOperands(I, Ops, SQ, ORE); | ||||
6807 | |||||
6808 | /// If called on unreachable code, the instruction may simplify to itself. | ||||
6809 | /// Make life easier for users by detecting that case here, and returning a | ||||
6810 | /// safe value instead. | ||||
6811 | return Result == I ? UndefValue::get(I->getType()) : Result; | ||||
6812 | } | ||||
6813 | |||||
6814 | /// Implementation of recursive simplification through an instruction's | ||||
6815 | /// uses. | ||||
6816 | /// | ||||
6817 | /// This is the common implementation of the recursive simplification routines. | ||||
6818 | /// If we have a pre-simplified value in 'SimpleV', that is forcibly used to | ||||
6819 | /// replace the instruction 'I'. Otherwise, we simply add 'I' to the list of | ||||
6820 | /// instructions to process and attempt to simplify it using | ||||
6821 | /// InstructionSimplify. Recursively visited users which could not be | ||||
6822 | /// simplified themselves are to the optional UnsimplifiedUsers set for | ||||
6823 | /// further processing by the caller. | ||||
6824 | /// | ||||
6825 | /// This routine returns 'true' only when *it* simplifies something. The passed | ||||
6826 | /// in simplified value does not count toward this. | ||||
6827 | static bool replaceAndRecursivelySimplifyImpl( | ||||
6828 | Instruction *I, Value *SimpleV, const TargetLibraryInfo *TLI, | ||||
6829 | const DominatorTree *DT, AssumptionCache *AC, | ||||
6830 | SmallSetVector<Instruction *, 8> *UnsimplifiedUsers = nullptr) { | ||||
6831 | bool Simplified = false; | ||||
6832 | SmallSetVector<Instruction *, 8> Worklist; | ||||
6833 | const DataLayout &DL = I->getModule()->getDataLayout(); | ||||
6834 | |||||
6835 | // If we have an explicit value to collapse to, do that round of the | ||||
6836 | // simplification loop by hand initially. | ||||
6837 | if (SimpleV) { | ||||
6838 | for (User *U : I->users()) | ||||
6839 | if (U != I) | ||||
6840 | Worklist.insert(cast<Instruction>(U)); | ||||
6841 | |||||
6842 | // Replace the instruction with its simplified value. | ||||
6843 | I->replaceAllUsesWith(SimpleV); | ||||
6844 | |||||
6845 | // Gracefully handle edge cases where the instruction is not wired into any | ||||
6846 | // parent block. | ||||
6847 | if (I->getParent() && !I->isEHPad() && !I->isTerminator() && | ||||
6848 | !I->mayHaveSideEffects()) | ||||
6849 | I->eraseFromParent(); | ||||
6850 | } else { | ||||
6851 | Worklist.insert(I); | ||||
6852 | } | ||||
6853 | |||||
6854 | // Note that we must test the size on each iteration, the worklist can grow. | ||||
6855 | for (unsigned Idx = 0; Idx != Worklist.size(); ++Idx) { | ||||
6856 | I = Worklist[Idx]; | ||||
6857 | |||||
6858 | // See if this instruction simplifies. | ||||
6859 | SimpleV = simplifyInstruction(I, {DL, TLI, DT, AC}); | ||||
6860 | if (!SimpleV) { | ||||
6861 | if (UnsimplifiedUsers) | ||||
6862 | UnsimplifiedUsers->insert(I); | ||||
6863 | continue; | ||||
6864 | } | ||||
6865 | |||||
6866 | Simplified = true; | ||||
6867 | |||||
6868 | // Stash away all the uses of the old instruction so we can check them for | ||||
6869 | // recursive simplifications after a RAUW. This is cheaper than checking all | ||||
6870 | // uses of To on the recursive step in most cases. | ||||
6871 | for (User *U : I->users()) | ||||
6872 | Worklist.insert(cast<Instruction>(U)); | ||||
6873 | |||||
6874 | // Replace the instruction with its simplified value. | ||||
6875 | I->replaceAllUsesWith(SimpleV); | ||||
6876 | |||||
6877 | // Gracefully handle edge cases where the instruction is not wired into any | ||||
6878 | // parent block. | ||||
6879 | if (I->getParent() && !I->isEHPad() && !I->isTerminator() && | ||||
6880 | !I->mayHaveSideEffects()) | ||||
6881 | I->eraseFromParent(); | ||||
6882 | } | ||||
6883 | return Simplified; | ||||
6884 | } | ||||
6885 | |||||
6886 | bool llvm::replaceAndRecursivelySimplify( | ||||
6887 | Instruction *I, Value *SimpleV, const TargetLibraryInfo *TLI, | ||||
6888 | const DominatorTree *DT, AssumptionCache *AC, | ||||
6889 | SmallSetVector<Instruction *, 8> *UnsimplifiedUsers) { | ||||
6890 | assert(I != SimpleV && "replaceAndRecursivelySimplify(X,X) is not valid!")(static_cast <bool> (I != SimpleV && "replaceAndRecursivelySimplify(X,X) is not valid!" ) ? void (0) : __assert_fail ("I != SimpleV && \"replaceAndRecursivelySimplify(X,X) is not valid!\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 6890, __extension__ __PRETTY_FUNCTION__)); | ||||
6891 | assert(SimpleV && "Must provide a simplified value.")(static_cast <bool> (SimpleV && "Must provide a simplified value." ) ? void (0) : __assert_fail ("SimpleV && \"Must provide a simplified value.\"" , "llvm/lib/Analysis/InstructionSimplify.cpp", 6891, __extension__ __PRETTY_FUNCTION__)); | ||||
6892 | return replaceAndRecursivelySimplifyImpl(I, SimpleV, TLI, DT, AC, | ||||
6893 | UnsimplifiedUsers); | ||||
6894 | } | ||||
6895 | |||||
6896 | namespace llvm { | ||||
6897 | const SimplifyQuery getBestSimplifyQuery(Pass &P, Function &F) { | ||||
6898 | auto *DTWP = P.getAnalysisIfAvailable<DominatorTreeWrapperPass>(); | ||||
6899 | auto *DT = DTWP ? &DTWP->getDomTree() : nullptr; | ||||
6900 | auto *TLIWP = P.getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>(); | ||||
6901 | auto *TLI = TLIWP ? &TLIWP->getTLI(F) : nullptr; | ||||
6902 | auto *ACWP = P.getAnalysisIfAvailable<AssumptionCacheTracker>(); | ||||
6903 | auto *AC = ACWP ? &ACWP->getAssumptionCache(F) : nullptr; | ||||
6904 | return {F.getParent()->getDataLayout(), TLI, DT, AC}; | ||||
6905 | } | ||||
6906 | |||||
6907 | const SimplifyQuery getBestSimplifyQuery(LoopStandardAnalysisResults &AR, | ||||
6908 | const DataLayout &DL) { | ||||
6909 | return {DL, &AR.TLI, &AR.DT, &AR.AC}; | ||||
6910 | } | ||||
6911 | |||||
6912 | template <class T, class... TArgs> | ||||
6913 | const SimplifyQuery getBestSimplifyQuery(AnalysisManager<T, TArgs...> &AM, | ||||
6914 | Function &F) { | ||||
6915 | auto *DT = AM.template getCachedResult<DominatorTreeAnalysis>(F); | ||||
6916 | auto *TLI = AM.template getCachedResult<TargetLibraryAnalysis>(F); | ||||
6917 | auto *AC = AM.template getCachedResult<AssumptionAnalysis>(F); | ||||
6918 | return {F.getParent()->getDataLayout(), TLI, DT, AC}; | ||||
6919 | } | ||||
6920 | template const SimplifyQuery getBestSimplifyQuery(AnalysisManager<Function> &, | ||||
6921 | Function &); | ||||
6922 | } // namespace llvm | ||||
6923 | |||||
6924 | void InstSimplifyFolder::anchor() {} |
1 | //===- PatternMatch.h - Match on the LLVM IR --------------------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file provides a simple and efficient mechanism for performing general |
10 | // tree-based pattern matches on the LLVM IR. The power of these routines is |
11 | // that it allows you to write concise patterns that are expressive and easy to |
12 | // understand. The other major advantage of this is that it allows you to |
13 | // trivially capture/bind elements in the pattern to variables. For example, |
14 | // you can do something like this: |
15 | // |
16 | // Value *Exp = ... |
17 | // Value *X, *Y; ConstantInt *C1, *C2; // (X & C1) | (Y & C2) |
18 | // if (match(Exp, m_Or(m_And(m_Value(X), m_ConstantInt(C1)), |
19 | // m_And(m_Value(Y), m_ConstantInt(C2))))) { |
20 | // ... Pattern is matched and variables are bound ... |
21 | // } |
22 | // |
23 | // This is primarily useful to things like the instruction combiner, but can |
24 | // also be useful for static analysis tools or code generators. |
25 | // |
26 | //===----------------------------------------------------------------------===// |
27 | |
28 | #ifndef LLVM_IR_PATTERNMATCH_H |
29 | #define LLVM_IR_PATTERNMATCH_H |
30 | |
31 | #include "llvm/ADT/APFloat.h" |
32 | #include "llvm/ADT/APInt.h" |
33 | #include "llvm/IR/Constant.h" |
34 | #include "llvm/IR/Constants.h" |
35 | #include "llvm/IR/DataLayout.h" |
36 | #include "llvm/IR/InstrTypes.h" |
37 | #include "llvm/IR/Instruction.h" |
38 | #include "llvm/IR/Instructions.h" |
39 | #include "llvm/IR/IntrinsicInst.h" |
40 | #include "llvm/IR/Intrinsics.h" |
41 | #include "llvm/IR/Operator.h" |
42 | #include "llvm/IR/Value.h" |
43 | #include "llvm/Support/Casting.h" |
44 | #include <cstdint> |
45 | |
46 | namespace llvm { |
47 | namespace PatternMatch { |
48 | |
49 | template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) { |
50 | return const_cast<Pattern &>(P).match(V); |
51 | } |
52 | |
53 | template <typename Pattern> bool match(ArrayRef<int> Mask, const Pattern &P) { |
54 | return const_cast<Pattern &>(P).match(Mask); |
55 | } |
56 | |
57 | template <typename SubPattern_t> struct OneUse_match { |
58 | SubPattern_t SubPattern; |
59 | |
60 | OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {} |
61 | |
62 | template <typename OpTy> bool match(OpTy *V) { |
63 | return V->hasOneUse() && SubPattern.match(V); |
64 | } |
65 | }; |
66 | |
67 | template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) { |
68 | return SubPattern; |
69 | } |
70 | |
71 | template <typename Class> struct class_match { |
72 | template <typename ITy> bool match(ITy *V) { return isa<Class>(V); } |
73 | }; |
74 | |
75 | /// Match an arbitrary value and ignore it. |
76 | inline class_match<Value> m_Value() { return class_match<Value>(); } |
77 | |
78 | /// Match an arbitrary unary operation and ignore it. |
79 | inline class_match<UnaryOperator> m_UnOp() { |
80 | return class_match<UnaryOperator>(); |
81 | } |
82 | |
83 | /// Match an arbitrary binary operation and ignore it. |
84 | inline class_match<BinaryOperator> m_BinOp() { |
85 | return class_match<BinaryOperator>(); |
86 | } |
87 | |
88 | /// Matches any compare instruction and ignore it. |
89 | inline class_match<CmpInst> m_Cmp() { return class_match<CmpInst>(); } |
90 | |
91 | struct undef_match { |
92 | static bool check(const Value *V) { |
93 | if (isa<UndefValue>(V)) |
94 | return true; |
95 | |
96 | const auto *CA = dyn_cast<ConstantAggregate>(V); |
97 | if (!CA) |
98 | return false; |
99 | |
100 | SmallPtrSet<const ConstantAggregate *, 8> Seen; |
101 | SmallVector<const ConstantAggregate *, 8> Worklist; |
102 | |
103 | // Either UndefValue, PoisonValue, or an aggregate that only contains |
104 | // these is accepted by matcher. |
105 | // CheckValue returns false if CA cannot satisfy this constraint. |
106 | auto CheckValue = [&](const ConstantAggregate *CA) { |
107 | for (const Value *Op : CA->operand_values()) { |
108 | if (isa<UndefValue>(Op)) |
109 | continue; |
110 | |
111 | const auto *CA = dyn_cast<ConstantAggregate>(Op); |
112 | if (!CA) |
113 | return false; |
114 | if (Seen.insert(CA).second) |
115 | Worklist.emplace_back(CA); |
116 | } |
117 | |
118 | return true; |
119 | }; |
120 | |
121 | if (!CheckValue(CA)) |
122 | return false; |
123 | |
124 | while (!Worklist.empty()) { |
125 | if (!CheckValue(Worklist.pop_back_val())) |
126 | return false; |
127 | } |
128 | return true; |
129 | } |
130 | template <typename ITy> bool match(ITy *V) { return check(V); } |
131 | }; |
132 | |
133 | /// Match an arbitrary undef constant. This matches poison as well. |
134 | /// If this is an aggregate and contains a non-aggregate element that is |
135 | /// neither undef nor poison, the aggregate is not matched. |
136 | inline auto m_Undef() { return undef_match(); } |
137 | |
138 | /// Match an arbitrary poison constant. |
139 | inline class_match<PoisonValue> m_Poison() { |
140 | return class_match<PoisonValue>(); |
141 | } |
142 | |
143 | /// Match an arbitrary Constant and ignore it. |
144 | inline class_match<Constant> m_Constant() { return class_match<Constant>(); } |
145 | |
146 | /// Match an arbitrary ConstantInt and ignore it. |
147 | inline class_match<ConstantInt> m_ConstantInt() { |
148 | return class_match<ConstantInt>(); |
149 | } |
150 | |
151 | /// Match an arbitrary ConstantFP and ignore it. |
152 | inline class_match<ConstantFP> m_ConstantFP() { |
153 | return class_match<ConstantFP>(); |
154 | } |
155 | |
156 | struct constantexpr_match { |
157 | template <typename ITy> bool match(ITy *V) { |
158 | auto *C = dyn_cast<Constant>(V); |
159 | return C && (isa<ConstantExpr>(C) || C->containsConstantExpression()); |
160 | } |
161 | }; |
162 | |
163 | /// Match a constant expression or a constant that contains a constant |
164 | /// expression. |
165 | inline constantexpr_match m_ConstantExpr() { return constantexpr_match(); } |
166 | |
167 | /// Match an arbitrary basic block value and ignore it. |
168 | inline class_match<BasicBlock> m_BasicBlock() { |
169 | return class_match<BasicBlock>(); |
170 | } |
171 | |
172 | /// Inverting matcher |
173 | template <typename Ty> struct match_unless { |
174 | Ty M; |
175 | |
176 | match_unless(const Ty &Matcher) : M(Matcher) {} |
177 | |
178 | template <typename ITy> bool match(ITy *V) { return !M.match(V); } |
179 | }; |
180 | |
181 | /// Match if the inner matcher does *NOT* match. |
182 | template <typename Ty> inline match_unless<Ty> m_Unless(const Ty &M) { |
183 | return match_unless<Ty>(M); |
184 | } |
185 | |
186 | /// Matching combinators |
187 | template <typename LTy, typename RTy> struct match_combine_or { |
188 | LTy L; |
189 | RTy R; |
190 | |
191 | match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} |
192 | |
193 | template <typename ITy> bool match(ITy *V) { |
194 | if (L.match(V)) |
195 | return true; |
196 | if (R.match(V)) |
197 | return true; |
198 | return false; |
199 | } |
200 | }; |
201 | |
202 | template <typename LTy, typename RTy> struct match_combine_and { |
203 | LTy L; |
204 | RTy R; |
205 | |
206 | match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} |
207 | |
208 | template <typename ITy> bool match(ITy *V) { |
209 | if (L.match(V)) |
210 | if (R.match(V)) |
211 | return true; |
212 | return false; |
213 | } |
214 | }; |
215 | |
216 | /// Combine two pattern matchers matching L || R |
217 | template <typename LTy, typename RTy> |
218 | inline match_combine_or<LTy, RTy> m_CombineOr(const LTy &L, const RTy &R) { |
219 | return match_combine_or<LTy, RTy>(L, R); |
220 | } |
221 | |
222 | /// Combine two pattern matchers matching L && R |
223 | template <typename LTy, typename RTy> |
224 | inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) { |
225 | return match_combine_and<LTy, RTy>(L, R); |
226 | } |
227 | |
228 | struct apint_match { |
229 | const APInt *&Res; |
230 | bool AllowUndef; |
231 | |
232 | apint_match(const APInt *&Res, bool AllowUndef) |
233 | : Res(Res), AllowUndef(AllowUndef) {} |
234 | |
235 | template <typename ITy> bool match(ITy *V) { |
236 | if (auto *CI = dyn_cast<ConstantInt>(V)) { |
237 | Res = &CI->getValue(); |
238 | return true; |
239 | } |
240 | if (V->getType()->isVectorTy()) |
241 | if (const auto *C = dyn_cast<Constant>(V)) |
242 | if (auto *CI = |
243 | dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndef))) { |
244 | Res = &CI->getValue(); |
245 | return true; |
246 | } |
247 | return false; |
248 | } |
249 | }; |
250 | // Either constexpr if or renaming ConstantFP::getValueAPF to |
251 | // ConstantFP::getValue is needed to do it via single template |
252 | // function for both apint/apfloat. |
253 | struct apfloat_match { |
254 | const APFloat *&Res; |
255 | bool AllowUndef; |
256 | |
257 | apfloat_match(const APFloat *&Res, bool AllowUndef) |
258 | : Res(Res), AllowUndef(AllowUndef) {} |
259 | |
260 | template <typename ITy> bool match(ITy *V) { |
261 | if (auto *CI = dyn_cast<ConstantFP>(V)) { |
262 | Res = &CI->getValueAPF(); |
263 | return true; |
264 | } |
265 | if (V->getType()->isVectorTy()) |
266 | if (const auto *C = dyn_cast<Constant>(V)) |
267 | if (auto *CI = |
268 | dyn_cast_or_null<ConstantFP>(C->getSplatValue(AllowUndef))) { |
269 | Res = &CI->getValueAPF(); |
270 | return true; |
271 | } |
272 | return false; |
273 | } |
274 | }; |
275 | |
276 | /// Match a ConstantInt or splatted ConstantVector, binding the |
277 | /// specified pointer to the contained APInt. |
278 | inline apint_match m_APInt(const APInt *&Res) { |
279 | // Forbid undefs by default to maintain previous behavior. |
280 | return apint_match(Res, /* AllowUndef */ false); |
281 | } |
282 | |
283 | /// Match APInt while allowing undefs in splat vector constants. |
284 | inline apint_match m_APIntAllowUndef(const APInt *&Res) { |
285 | return apint_match(Res, /* AllowUndef */ true); |
286 | } |
287 | |
288 | /// Match APInt while forbidding undefs in splat vector constants. |
289 | inline apint_match m_APIntForbidUndef(const APInt *&Res) { |
290 | return apint_match(Res, /* AllowUndef */ false); |
291 | } |
292 | |
293 | /// Match a ConstantFP or splatted ConstantVector, binding the |
294 | /// specified pointer to the contained APFloat. |
295 | inline apfloat_match m_APFloat(const APFloat *&Res) { |
296 | // Forbid undefs by default to maintain previous behavior. |
297 | return apfloat_match(Res, /* AllowUndef */ false); |
298 | } |
299 | |
300 | /// Match APFloat while allowing undefs in splat vector constants. |
301 | inline apfloat_match m_APFloatAllowUndef(const APFloat *&Res) { |
302 | return apfloat_match(Res, /* AllowUndef */ true); |
303 | } |
304 | |
305 | /// Match APFloat while forbidding undefs in splat vector constants. |
306 | inline apfloat_match m_APFloatForbidUndef(const APFloat *&Res) { |
307 | return apfloat_match(Res, /* AllowUndef */ false); |
308 | } |
309 | |
310 | template <int64_t Val> struct constantint_match { |
311 | template <typename ITy> bool match(ITy *V) { |
312 | if (const auto *CI = dyn_cast<ConstantInt>(V)) { |
313 | const APInt &CIV = CI->getValue(); |
314 | if (Val >= 0) |
315 | return CIV == static_cast<uint64_t>(Val); |
316 | // If Val is negative, and CI is shorter than it, truncate to the right |
317 | // number of bits. If it is larger, then we have to sign extend. Just |
318 | // compare their negated values. |
319 | return -CIV == -Val; |
320 | } |
321 | return false; |
322 | } |
323 | }; |
324 | |
325 | /// Match a ConstantInt with a specific value. |
326 | template <int64_t Val> inline constantint_match<Val> m_ConstantInt() { |
327 | return constantint_match<Val>(); |
328 | } |
329 | |
330 | /// This helper class is used to match constant scalars, vector splats, |
331 | /// and fixed width vectors that satisfy a specified predicate. |
332 | /// For fixed width vector constants, undefined elements are ignored. |
333 | template <typename Predicate, typename ConstantVal> |
334 | struct cstval_pred_ty : public Predicate { |
335 | template <typename ITy> bool match(ITy *V) { |
336 | if (const auto *CV = dyn_cast<ConstantVal>(V)) |
337 | return this->isValue(CV->getValue()); |
338 | if (const auto *VTy = dyn_cast<VectorType>(V->getType())) { |
339 | if (const auto *C = dyn_cast<Constant>(V)) { |
340 | if (const auto *CV = dyn_cast_or_null<ConstantVal>(C->getSplatValue())) |
341 | return this->isValue(CV->getValue()); |
342 | |
343 | // Number of elements of a scalable vector unknown at compile time |
344 | auto *FVTy = dyn_cast<FixedVectorType>(VTy); |
345 | if (!FVTy) |
346 | return false; |
347 | |
348 | // Non-splat vector constant: check each element for a match. |
349 | unsigned NumElts = FVTy->getNumElements(); |
350 | assert(NumElts != 0 && "Constant vector with no elements?")(static_cast <bool> (NumElts != 0 && "Constant vector with no elements?" ) ? void (0) : __assert_fail ("NumElts != 0 && \"Constant vector with no elements?\"" , "llvm/include/llvm/IR/PatternMatch.h", 350, __extension__ __PRETTY_FUNCTION__ )); |
351 | bool HasNonUndefElements = false; |
352 | for (unsigned i = 0; i != NumElts; ++i) { |
353 | Constant *Elt = C->getAggregateElement(i); |
354 | if (!Elt) |
355 | return false; |
356 | if (isa<UndefValue>(Elt)) |
357 | continue; |
358 | auto *CV = dyn_cast<ConstantVal>(Elt); |
359 | if (!CV || !this->isValue(CV->getValue())) |
360 | return false; |
361 | HasNonUndefElements = true; |
362 | } |
363 | return HasNonUndefElements; |
364 | } |
365 | } |
366 | return false; |
367 | } |
368 | }; |
369 | |
370 | /// specialization of cstval_pred_ty for ConstantInt |
371 | template <typename Predicate> |
372 | using cst_pred_ty = cstval_pred_ty<Predicate, ConstantInt>; |
373 | |
374 | /// specialization of cstval_pred_ty for ConstantFP |
375 | template <typename Predicate> |
376 | using cstfp_pred_ty = cstval_pred_ty<Predicate, ConstantFP>; |
377 | |
378 | /// This helper class is used to match scalar and vector constants that |
379 | /// satisfy a specified predicate, and bind them to an APInt. |
380 | template <typename Predicate> struct api_pred_ty : public Predicate { |
381 | const APInt *&Res; |
382 | |
383 | api_pred_ty(const APInt *&R) : Res(R) {} |
384 | |
385 | template <typename ITy> bool match(ITy *V) { |
386 | if (const auto *CI = dyn_cast<ConstantInt>(V)) |
387 | if (this->isValue(CI->getValue())) { |
388 | Res = &CI->getValue(); |
389 | return true; |
390 | } |
391 | if (V->getType()->isVectorTy()) |
392 | if (const auto *C = dyn_cast<Constant>(V)) |
393 | if (auto *CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue())) |
394 | if (this->isValue(CI->getValue())) { |
395 | Res = &CI->getValue(); |
396 | return true; |
397 | } |
398 | |
399 | return false; |
400 | } |
401 | }; |
402 | |
403 | /// This helper class is used to match scalar and vector constants that |
404 | /// satisfy a specified predicate, and bind them to an APFloat. |
405 | /// Undefs are allowed in splat vector constants. |
406 | template <typename Predicate> struct apf_pred_ty : public Predicate { |
407 | const APFloat *&Res; |
408 | |
409 | apf_pred_ty(const APFloat *&R) : Res(R) {} |
410 | |
411 | template <typename ITy> bool match(ITy *V) { |
412 | if (const auto *CI = dyn_cast<ConstantFP>(V)) |
413 | if (this->isValue(CI->getValue())) { |
414 | Res = &CI->getValue(); |
415 | return true; |
416 | } |
417 | if (V->getType()->isVectorTy()) |
418 | if (const auto *C = dyn_cast<Constant>(V)) |
419 | if (auto *CI = dyn_cast_or_null<ConstantFP>( |
420 | C->getSplatValue(/* AllowUndef */ true))) |
421 | if (this->isValue(CI->getValue())) { |
422 | Res = &CI->getValue(); |
423 | return true; |
424 | } |
425 | |
426 | return false; |
427 | } |
428 | }; |
429 | |
430 | /////////////////////////////////////////////////////////////////////////////// |
431 | // |
432 | // Encapsulate constant value queries for use in templated predicate matchers. |
433 | // This allows checking if constants match using compound predicates and works |
434 | // with vector constants, possibly with relaxed constraints. For example, ignore |
435 | // undef values. |
436 | // |
437 | /////////////////////////////////////////////////////////////////////////////// |
438 | |
439 | struct is_any_apint { |
440 | bool isValue(const APInt &C) { return true; } |
441 | }; |
442 | /// Match an integer or vector with any integral constant. |
443 | /// For vectors, this includes constants with undefined elements. |
444 | inline cst_pred_ty<is_any_apint> m_AnyIntegralConstant() { |
445 | return cst_pred_ty<is_any_apint>(); |
446 | } |
447 | |
448 | struct is_all_ones { |
449 | bool isValue(const APInt &C) { return C.isAllOnes(); } |
450 | }; |
451 | /// Match an integer or vector with all bits set. |
452 | /// For vectors, this includes constants with undefined elements. |
453 | inline cst_pred_ty<is_all_ones> m_AllOnes() { |
454 | return cst_pred_ty<is_all_ones>(); |
455 | } |
456 | |
457 | struct is_maxsignedvalue { |
458 | bool isValue(const APInt &C) { return C.isMaxSignedValue(); } |
459 | }; |
460 | /// Match an integer or vector with values having all bits except for the high |
461 | /// bit set (0x7f...). |
462 | /// For vectors, this includes constants with undefined elements. |
463 | inline cst_pred_ty<is_maxsignedvalue> m_MaxSignedValue() { |
464 | return cst_pred_ty<is_maxsignedvalue>(); |
465 | } |
466 | inline api_pred_ty<is_maxsignedvalue> m_MaxSignedValue(const APInt *&V) { |
467 | return V; |
468 | } |
469 | |
470 | struct is_negative { |
471 | bool isValue(const APInt &C) { return C.isNegative(); } |
472 | }; |
473 | /// Match an integer or vector of negative values. |
474 | /// For vectors, this includes constants with undefined elements. |
475 | inline cst_pred_ty<is_negative> m_Negative() { |
476 | return cst_pred_ty<is_negative>(); |
477 | } |
478 | inline api_pred_ty<is_negative> m_Negative(const APInt *&V) { return V; } |
479 | |
480 | struct is_nonnegative { |
481 | bool isValue(const APInt &C) { return C.isNonNegative(); } |
482 | }; |
483 | /// Match an integer or vector of non-negative values. |
484 | /// For vectors, this includes constants with undefined elements. |
485 | inline cst_pred_ty<is_nonnegative> m_NonNegative() { |
486 | return cst_pred_ty<is_nonnegative>(); |
487 | } |
488 | inline api_pred_ty<is_nonnegative> m_NonNegative(const APInt *&V) { return V; } |
489 | |
490 | struct is_strictlypositive { |
491 | bool isValue(const APInt &C) { return C.isStrictlyPositive(); } |
492 | }; |
493 | /// Match an integer or vector of strictly positive values. |
494 | /// For vectors, this includes constants with undefined elements. |
495 | inline cst_pred_ty<is_strictlypositive> m_StrictlyPositive() { |
496 | return cst_pred_ty<is_strictlypositive>(); |
497 | } |
498 | inline api_pred_ty<is_strictlypositive> m_StrictlyPositive(const APInt *&V) { |
499 | return V; |
500 | } |
501 | |
502 | struct is_nonpositive { |
503 | bool isValue(const APInt &C) { return C.isNonPositive(); } |
504 | }; |
505 | /// Match an integer or vector of non-positive values. |
506 | /// For vectors, this includes constants with undefined elements. |
507 | inline cst_pred_ty<is_nonpositive> m_NonPositive() { |
508 | return cst_pred_ty<is_nonpositive>(); |
509 | } |
510 | inline api_pred_ty<is_nonpositive> m_NonPositive(const APInt *&V) { return V; } |
511 | |
512 | struct is_one { |
513 | bool isValue(const APInt &C) { return C.isOne(); } |
514 | }; |
515 | /// Match an integer 1 or a vector with all elements equal to 1. |
516 | /// For vectors, this includes constants with undefined elements. |
517 | inline cst_pred_ty<is_one> m_One() { return cst_pred_ty<is_one>(); } |
518 | |
519 | struct is_zero_int { |
520 | bool isValue(const APInt &C) { return C.isZero(); } |
521 | }; |
522 | /// Match an integer 0 or a vector with all elements equal to 0. |
523 | /// For vectors, this includes constants with undefined elements. |
524 | inline cst_pred_ty<is_zero_int> m_ZeroInt() { |
525 | return cst_pred_ty<is_zero_int>(); |
526 | } |
527 | |
528 | struct is_zero { |
529 | template <typename ITy> bool match(ITy *V) { |
530 | auto *C = dyn_cast<Constant>(V); |
531 | // FIXME: this should be able to do something for scalable vectors |
532 | return C && (C->isNullValue() || cst_pred_ty<is_zero_int>().match(C)); |
533 | } |
534 | }; |
535 | /// Match any null constant or a vector with all elements equal to 0. |
536 | /// For vectors, this includes constants with undefined elements. |
537 | inline is_zero m_Zero() { return is_zero(); } |
538 | |
539 | struct is_power2 { |
540 | bool isValue(const APInt &C) { return C.isPowerOf2(); } |
541 | }; |
542 | /// Match an integer or vector power-of-2. |
543 | /// For vectors, this includes constants with undefined elements. |
544 | inline cst_pred_ty<is_power2> m_Power2() { return cst_pred_ty<is_power2>(); } |
545 | inline api_pred_ty<is_power2> m_Power2(const APInt *&V) { return V; } |
546 | |
547 | struct is_negated_power2 { |
548 | bool isValue(const APInt &C) { return C.isNegatedPowerOf2(); } |
549 | }; |
550 | /// Match a integer or vector negated power-of-2. |
551 | /// For vectors, this includes constants with undefined elements. |
552 | inline cst_pred_ty<is_negated_power2> m_NegatedPower2() { |
553 | return cst_pred_ty<is_negated_power2>(); |
554 | } |
555 | inline api_pred_ty<is_negated_power2> m_NegatedPower2(const APInt *&V) { |
556 | return V; |
557 | } |
558 | |
559 | struct is_power2_or_zero { |
560 | bool isValue(const APInt &C) { return !C || C.isPowerOf2(); } |
561 | }; |
562 | /// Match an integer or vector of 0 or power-of-2 values. |
563 | /// For vectors, this includes constants with undefined elements. |
564 | inline cst_pred_ty<is_power2_or_zero> m_Power2OrZero() { |
565 | return cst_pred_ty<is_power2_or_zero>(); |
566 | } |
567 | inline api_pred_ty<is_power2_or_zero> m_Power2OrZero(const APInt *&V) { |
568 | return V; |
569 | } |
570 | |
571 | struct is_sign_mask { |
572 | bool isValue(const APInt &C) { return C.isSignMask(); } |
573 | }; |
574 | /// Match an integer or vector with only the sign bit(s) set. |
575 | /// For vectors, this includes constants with undefined elements. |
576 | inline cst_pred_ty<is_sign_mask> m_SignMask() { |
577 | return cst_pred_ty<is_sign_mask>(); |
578 | } |
579 | |
580 | struct is_lowbit_mask { |
581 | bool isValue(const APInt &C) { return C.isMask(); } |
582 | }; |
583 | /// Match an integer or vector with only the low bit(s) set. |
584 | /// For vectors, this includes constants with undefined elements. |
585 | inline cst_pred_ty<is_lowbit_mask> m_LowBitMask() { |
586 | return cst_pred_ty<is_lowbit_mask>(); |
587 | } |
588 | inline api_pred_ty<is_lowbit_mask> m_LowBitMask(const APInt *&V) { return V; } |
589 | |
590 | struct icmp_pred_with_threshold { |
591 | ICmpInst::Predicate Pred; |
592 | const APInt *Thr; |
593 | bool isValue(const APInt &C) { return ICmpInst::compare(C, *Thr, Pred); } |
594 | }; |
595 | /// Match an integer or vector with every element comparing 'pred' (eg/ne/...) |
596 | /// to Threshold. For vectors, this includes constants with undefined elements. |
597 | inline cst_pred_ty<icmp_pred_with_threshold> |
598 | m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold) { |
599 | cst_pred_ty<icmp_pred_with_threshold> P; |
600 | P.Pred = Predicate; |
601 | P.Thr = &Threshold; |
602 | return P; |
603 | } |
604 | |
605 | struct is_nan { |
606 | bool isValue(const APFloat &C) { return C.isNaN(); } |
607 | }; |
608 | /// Match an arbitrary NaN constant. This includes quiet and signalling nans. |
609 | /// For vectors, this includes constants with undefined elements. |
610 | inline cstfp_pred_ty<is_nan> m_NaN() { return cstfp_pred_ty<is_nan>(); } |
611 | |
612 | struct is_nonnan { |
613 | bool isValue(const APFloat &C) { return !C.isNaN(); } |
614 | }; |
615 | /// Match a non-NaN FP constant. |
616 | /// For vectors, this includes constants with undefined elements. |
617 | inline cstfp_pred_ty<is_nonnan> m_NonNaN() { |
618 | return cstfp_pred_ty<is_nonnan>(); |
619 | } |
620 | |
621 | struct is_inf { |
622 | bool isValue(const APFloat &C) { return C.isInfinity(); } |
623 | }; |
624 | /// Match a positive or negative infinity FP constant. |
625 | /// For vectors, this includes constants with undefined elements. |
626 | inline cstfp_pred_ty<is_inf> m_Inf() { return cstfp_pred_ty<is_inf>(); } |
627 | |
628 | struct is_noninf { |
629 | bool isValue(const APFloat &C) { return !C.isInfinity(); } |
630 | }; |
631 | /// Match a non-infinity FP constant, i.e. finite or NaN. |
632 | /// For vectors, this includes constants with undefined elements. |
633 | inline cstfp_pred_ty<is_noninf> m_NonInf() { |
634 | return cstfp_pred_ty<is_noninf>(); |
635 | } |
636 | |
637 | struct is_finite { |
638 | bool isValue(const APFloat &C) { return C.isFinite(); } |
639 | }; |
640 | /// Match a finite FP constant, i.e. not infinity or NaN. |
641 | /// For vectors, this includes constants with undefined elements. |
642 | inline cstfp_pred_ty<is_finite> m_Finite() { |
643 | return cstfp_pred_ty<is_finite>(); |
644 | } |
645 | inline apf_pred_ty<is_finite> m_Finite(const APFloat *&V) { return V; } |
646 | |
647 | struct is_finitenonzero { |
648 | bool isValue(const APFloat &C) { return C.isFiniteNonZero(); } |
649 | }; |
650 | /// Match a finite non-zero FP constant. |
651 | /// For vectors, this includes constants with undefined elements. |
652 | inline cstfp_pred_ty<is_finitenonzero> m_FiniteNonZero() { |
653 | return cstfp_pred_ty<is_finitenonzero>(); |
654 | } |
655 | inline apf_pred_ty<is_finitenonzero> m_FiniteNonZero(const APFloat *&V) { |
656 | return V; |
657 | } |
658 | |
659 | struct is_any_zero_fp { |
660 | bool isValue(const APFloat &C) { return C.isZero(); } |
661 | }; |
662 | /// Match a floating-point negative zero or positive zero. |
663 | /// For vectors, this includes constants with undefined elements. |
664 | inline cstfp_pred_ty<is_any_zero_fp> m_AnyZeroFP() { |
665 | return cstfp_pred_ty<is_any_zero_fp>(); |
666 | } |
667 | |
668 | struct is_pos_zero_fp { |
669 | bool isValue(const APFloat &C) { return C.isPosZero(); } |
670 | }; |
671 | /// Match a floating-point positive zero. |
672 | /// For vectors, this includes constants with undefined elements. |
673 | inline cstfp_pred_ty<is_pos_zero_fp> m_PosZeroFP() { |
674 | return cstfp_pred_ty<is_pos_zero_fp>(); |
675 | } |
676 | |
677 | struct is_neg_zero_fp { |
678 | bool isValue(const APFloat &C) { return C.isNegZero(); } |
679 | }; |
680 | /// Match a floating-point negative zero. |
681 | /// For vectors, this includes constants with undefined elements. |
682 | inline cstfp_pred_ty<is_neg_zero_fp> m_NegZeroFP() { |
683 | return cstfp_pred_ty<is_neg_zero_fp>(); |
684 | } |
685 | |
686 | struct is_non_zero_fp { |
687 | bool isValue(const APFloat &C) { return C.isNonZero(); } |
688 | }; |
689 | /// Match a floating-point non-zero. |
690 | /// For vectors, this includes constants with undefined elements. |
691 | inline cstfp_pred_ty<is_non_zero_fp> m_NonZeroFP() { |
692 | return cstfp_pred_ty<is_non_zero_fp>(); |
693 | } |
694 | |
695 | /////////////////////////////////////////////////////////////////////////////// |
696 | |
697 | template <typename Class> struct bind_ty { |
698 | Class *&VR; |
699 | |
700 | bind_ty(Class *&V) : VR(V) {} |
701 | |
702 | template <typename ITy> bool match(ITy *V) { |
703 | if (auto *CV = dyn_cast<Class>(V)) { |
704 | VR = CV; |
705 | return true; |
706 | } |
707 | return false; |
708 | } |
709 | }; |
710 | |
711 | /// Match a value, capturing it if we match. |
712 | inline bind_ty<Value> m_Value(Value *&V) { return V; } |
713 | inline bind_ty<const Value> m_Value(const Value *&V) { return V; } |
714 | |
715 | /// Match an instruction, capturing it if we match. |
716 | inline bind_ty<Instruction> m_Instruction(Instruction *&I) { return I; } |
717 | /// Match a unary operator, capturing it if we match. |
718 | inline bind_ty<UnaryOperator> m_UnOp(UnaryOperator *&I) { return I; } |
719 | /// Match a binary operator, capturing it if we match. |
720 | inline bind_ty<BinaryOperator> m_BinOp(BinaryOperator *&I) { return I; } |
721 | /// Match a with overflow intrinsic, capturing it if we match. |
722 | inline bind_ty<WithOverflowInst> m_WithOverflowInst(WithOverflowInst *&I) { |
723 | return I; |
724 | } |
725 | inline bind_ty<const WithOverflowInst> |
726 | m_WithOverflowInst(const WithOverflowInst *&I) { |
727 | return I; |
728 | } |
729 | |
730 | /// Match a Constant, capturing the value if we match. |
731 | inline bind_ty<Constant> m_Constant(Constant *&C) { return C; } |
732 | |
733 | /// Match a ConstantInt, capturing the value if we match. |
734 | inline bind_ty<ConstantInt> m_ConstantInt(ConstantInt *&CI) { return CI; } |
735 | |
736 | /// Match a ConstantFP, capturing the value if we match. |
737 | inline bind_ty<ConstantFP> m_ConstantFP(ConstantFP *&C) { return C; } |
738 | |
739 | /// Match a ConstantExpr, capturing the value if we match. |
740 | inline bind_ty<ConstantExpr> m_ConstantExpr(ConstantExpr *&C) { return C; } |
741 | |
742 | /// Match a basic block value, capturing it if we match. |
743 | inline bind_ty<BasicBlock> m_BasicBlock(BasicBlock *&V) { return V; } |
744 | inline bind_ty<const BasicBlock> m_BasicBlock(const BasicBlock *&V) { |
745 | return V; |
746 | } |
747 | |
748 | /// Match an arbitrary immediate Constant and ignore it. |
749 | inline match_combine_and<class_match<Constant>, |
750 | match_unless<constantexpr_match>> |
751 | m_ImmConstant() { |
752 | return m_CombineAnd(m_Constant(), m_Unless(m_ConstantExpr())); |
753 | } |
754 | |
755 | /// Match an immediate Constant, capturing the value if we match. |
756 | inline match_combine_and<bind_ty<Constant>, |
757 | match_unless<constantexpr_match>> |
758 | m_ImmConstant(Constant *&C) { |
759 | return m_CombineAnd(m_Constant(C), m_Unless(m_ConstantExpr())); |
760 | } |
761 | |
762 | /// Match a specified Value*. |
763 | struct specificval_ty { |
764 | const Value *Val; |
765 | |
766 | specificval_ty(const Value *V) : Val(V) {} |
767 | |
768 | template <typename ITy> bool match(ITy *V) { return V == Val; } |
769 | }; |
770 | |
771 | /// Match if we have a specific specified value. |
772 | inline specificval_ty m_Specific(const Value *V) { return V; } |
773 | |
774 | /// Stores a reference to the Value *, not the Value * itself, |
775 | /// thus can be used in commutative matchers. |
776 | template <typename Class> struct deferredval_ty { |
777 | Class *const &Val; |
778 | |
779 | deferredval_ty(Class *const &V) : Val(V) {} |
780 | |
781 | template <typename ITy> bool match(ITy *const V) { return V == Val; } |
782 | }; |
783 | |
784 | /// Like m_Specific(), but works if the specific value to match is determined |
785 | /// as part of the same match() expression. For example: |
786 | /// m_Add(m_Value(X), m_Specific(X)) is incorrect, because m_Specific() will |
787 | /// bind X before the pattern match starts. |
788 | /// m_Add(m_Value(X), m_Deferred(X)) is correct, and will check against |
789 | /// whichever value m_Value(X) populated. |
790 | inline deferredval_ty<Value> m_Deferred(Value *const &V) { return V; } |
791 | inline deferredval_ty<const Value> m_Deferred(const Value *const &V) { |
792 | return V; |
793 | } |
794 | |
795 | /// Match a specified floating point value or vector of all elements of |
796 | /// that value. |
797 | struct specific_fpval { |
798 | double Val; |
799 | |
800 | specific_fpval(double V) : Val(V) {} |
801 | |
802 | template <typename ITy> bool match(ITy *V) { |
803 | if (const auto *CFP = dyn_cast<ConstantFP>(V)) |
804 | return CFP->isExactlyValue(Val); |
805 | if (V->getType()->isVectorTy()) |
806 | if (const auto *C = dyn_cast<Constant>(V)) |
807 | if (auto *CFP = dyn_cast_or_null<ConstantFP>(C->getSplatValue())) |
808 | return CFP->isExactlyValue(Val); |
809 | return false; |
810 | } |
811 | }; |
812 | |
813 | /// Match a specific floating point value or vector with all elements |
814 | /// equal to the value. |
815 | inline specific_fpval m_SpecificFP(double V) { return specific_fpval(V); } |
816 | |
817 | /// Match a float 1.0 or vector with all elements equal to 1.0. |
818 | inline specific_fpval m_FPOne() { return m_SpecificFP(1.0); } |
819 | |
820 | struct bind_const_intval_ty { |
821 | uint64_t &VR; |
822 | |
823 | bind_const_intval_ty(uint64_t &V) : VR(V) {} |
824 | |
825 | template <typename ITy> bool match(ITy *V) { |
826 | if (const auto *CV = dyn_cast<ConstantInt>(V)) |
827 | if (CV->getValue().ule(UINT64_MAX(18446744073709551615UL))) { |
828 | VR = CV->getZExtValue(); |
829 | return true; |
830 | } |
831 | return false; |
832 | } |
833 | }; |
834 | |
835 | /// Match a specified integer value or vector of all elements of that |
836 | /// value. |
837 | template <bool AllowUndefs> struct specific_intval { |
838 | APInt Val; |
839 | |
840 | specific_intval(APInt V) : Val(std::move(V)) {} |
841 | |
842 | template <typename ITy> bool match(ITy *V) { |
843 | const auto *CI = dyn_cast<ConstantInt>(V); |
844 | if (!CI && V->getType()->isVectorTy()) |
845 | if (const auto *C = dyn_cast<Constant>(V)) |
846 | CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndefs)); |
847 | |
848 | return CI && APInt::isSameValue(CI->getValue(), Val); |
849 | } |
850 | }; |
851 | |
852 | /// Match a specific integer value or vector with all elements equal to |
853 | /// the value. |
854 | inline specific_intval<false> m_SpecificInt(APInt V) { |
855 | return specific_intval<false>(std::move(V)); |
856 | } |
857 | |
858 | inline specific_intval<false> m_SpecificInt(uint64_t V) { |
859 | return m_SpecificInt(APInt(64, V)); |
860 | } |
861 | |
862 | inline specific_intval<true> m_SpecificIntAllowUndef(APInt V) { |
863 | return specific_intval<true>(std::move(V)); |
864 | } |
865 | |
866 | inline specific_intval<true> m_SpecificIntAllowUndef(uint64_t V) { |
867 | return m_SpecificIntAllowUndef(APInt(64, V)); |
868 | } |
869 | |
870 | /// Match a ConstantInt and bind to its value. This does not match |
871 | /// ConstantInts wider than 64-bits. |
872 | inline bind_const_intval_ty m_ConstantInt(uint64_t &V) { return V; } |
873 | |
874 | /// Match a specified basic block value. |
875 | struct specific_bbval { |
876 | BasicBlock *Val; |
877 | |
878 | specific_bbval(BasicBlock *Val) : Val(Val) {} |
879 | |
880 | template <typename ITy> bool match(ITy *V) { |
881 | const auto *BB = dyn_cast<BasicBlock>(V); |
882 | return BB && BB == Val; |
883 | } |
884 | }; |
885 | |
886 | /// Match a specific basic block value. |
887 | inline specific_bbval m_SpecificBB(BasicBlock *BB) { |
888 | return specific_bbval(BB); |
889 | } |
890 | |
891 | /// A commutative-friendly version of m_Specific(). |
892 | inline deferredval_ty<BasicBlock> m_Deferred(BasicBlock *const &BB) { |
893 | return BB; |
894 | } |
895 | inline deferredval_ty<const BasicBlock> |
896 | m_Deferred(const BasicBlock *const &BB) { |
897 | return BB; |
898 | } |
899 | |
900 | //===----------------------------------------------------------------------===// |
901 | // Matcher for any binary operator. |
902 | // |
903 | template <typename LHS_t, typename RHS_t, bool Commutable = false> |
904 | struct AnyBinaryOp_match { |
905 | LHS_t L; |
906 | RHS_t R; |
907 | |
908 | // The evaluation order is always stable, regardless of Commutability. |
909 | // The LHS is always matched first. |
910 | AnyBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} |
911 | |
912 | template <typename OpTy> bool match(OpTy *V) { |
913 | if (auto *I = dyn_cast<BinaryOperator>(V)) |
914 | return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) || |
915 | (Commutable && L.match(I->getOperand(1)) && |
916 | R.match(I->getOperand(0))); |
917 | return false; |
918 | } |
919 | }; |
920 | |
921 | template <typename LHS, typename RHS> |
922 | inline AnyBinaryOp_match<LHS, RHS> m_BinOp(const LHS &L, const RHS &R) { |
923 | return AnyBinaryOp_match<LHS, RHS>(L, R); |
924 | } |
925 | |
926 | //===----------------------------------------------------------------------===// |
927 | // Matcher for any unary operator. |
928 | // TODO fuse unary, binary matcher into n-ary matcher |
929 | // |
930 | template <typename OP_t> struct AnyUnaryOp_match { |
931 | OP_t X; |
932 | |
933 | AnyUnaryOp_match(const OP_t &X) : X(X) {} |
934 | |
935 | template <typename OpTy> bool match(OpTy *V) { |
936 | if (auto *I = dyn_cast<UnaryOperator>(V)) |
937 | return X.match(I->getOperand(0)); |
938 | return false; |
939 | } |
940 | }; |
941 | |
942 | template <typename OP_t> inline AnyUnaryOp_match<OP_t> m_UnOp(const OP_t &X) { |
943 | return AnyUnaryOp_match<OP_t>(X); |
944 | } |
945 | |
946 | //===----------------------------------------------------------------------===// |
947 | // Matchers for specific binary operators. |
948 | // |
949 | |
950 | template <typename LHS_t, typename RHS_t, unsigned Opcode, |
951 | bool Commutable = false> |
952 | struct BinaryOp_match { |
953 | LHS_t L; |
954 | RHS_t R; |
955 | |
956 | // The evaluation order is always stable, regardless of Commutability. |
957 | // The LHS is always matched first. |
958 | BinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} |
959 | |
960 | template <typename OpTy> inline bool match(unsigned Opc, OpTy *V) { |
961 | if (V->getValueID() == Value::InstructionVal + Opc) { |
962 | auto *I = cast<BinaryOperator>(V); |
963 | return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) || |
964 | (Commutable && L.match(I->getOperand(1)) && |
965 | R.match(I->getOperand(0))); |
966 | } |
967 | if (auto *CE = dyn_cast<ConstantExpr>(V)) |
968 | return CE->getOpcode() == Opc && |
969 | ((L.match(CE->getOperand(0)) && R.match(CE->getOperand(1))) || |
970 | (Commutable && L.match(CE->getOperand(1)) && |
971 | R.match(CE->getOperand(0)))); |
972 | return false; |
973 | } |
974 | |
975 | template <typename OpTy> bool match(OpTy *V) { return match(Opcode, V); } |
976 | }; |
977 | |
978 | template <typename LHS, typename RHS> |
979 | inline BinaryOp_match<LHS, RHS, Instruction::Add> m_Add(const LHS &L, |
980 | const RHS &R) { |
981 | return BinaryOp_match<LHS, RHS, Instruction::Add>(L, R); |
982 | } |
983 | |
984 | template <typename LHS, typename RHS> |
985 | inline BinaryOp_match<LHS, RHS, Instruction::FAdd> m_FAdd(const LHS &L, |
986 | const RHS &R) { |
987 | return BinaryOp_match<LHS, RHS, Instruction::FAdd>(L, R); |
988 | } |
989 | |
990 | template <typename LHS, typename RHS> |
991 | inline BinaryOp_match<LHS, RHS, Instruction::Sub> m_Sub(const LHS &L, |
992 | const RHS &R) { |
993 | return BinaryOp_match<LHS, RHS, Instruction::Sub>(L, R); |
994 | } |
995 | |
996 | template <typename LHS, typename RHS> |
997 | inline BinaryOp_match<LHS, RHS, Instruction::FSub> m_FSub(const LHS &L, |
998 | const RHS &R) { |
999 | return BinaryOp_match<LHS, RHS, Instruction::FSub>(L, R); |
1000 | } |
1001 | |
1002 | template <typename Op_t> struct FNeg_match { |
1003 | Op_t X; |
1004 | |
1005 | FNeg_match(const Op_t &Op) : X(Op) {} |
1006 | template <typename OpTy> bool match(OpTy *V) { |
1007 | auto *FPMO = dyn_cast<FPMathOperator>(V); |
1008 | if (!FPMO) |
1009 | return false; |
1010 | |
1011 | if (FPMO->getOpcode() == Instruction::FNeg) |
1012 | return X.match(FPMO->getOperand(0)); |
1013 | |
1014 | if (FPMO->getOpcode() == Instruction::FSub) { |
1015 | if (FPMO->hasNoSignedZeros()) { |
1016 | // With 'nsz', any zero goes. |
1017 | if (!cstfp_pred_ty<is_any_zero_fp>().match(FPMO->getOperand(0))) |
1018 | return false; |
1019 | } else { |
1020 | // Without 'nsz', we need fsub -0.0, X exactly. |
1021 | if (!cstfp_pred_ty<is_neg_zero_fp>().match(FPMO->getOperand(0))) |
1022 | return false; |
1023 | } |
1024 | |
1025 | return X.match(FPMO->getOperand(1)); |
1026 | } |
1027 | |
1028 | return false; |
1029 | } |
1030 | }; |
1031 | |
1032 | /// Match 'fneg X' as 'fsub -0.0, X'. |
1033 | template <typename OpTy> inline FNeg_match<OpTy> m_FNeg(const OpTy &X) { |
1034 | return FNeg_match<OpTy>(X); |
1035 | } |
1036 | |
1037 | /// Match 'fneg X' as 'fsub +-0.0, X'. |
1038 | template <typename RHS> |
1039 | inline BinaryOp_match<cstfp_pred_ty<is_any_zero_fp>, RHS, Instruction::FSub> |
1040 | m_FNegNSZ(const RHS &X) { |
1041 | return m_FSub(m_AnyZeroFP(), X); |
1042 | } |
1043 | |
1044 | template <typename LHS, typename RHS> |
1045 | inline BinaryOp_match<LHS, RHS, Instruction::Mul> m_Mul(const LHS &L, |
1046 | const RHS &R) { |
1047 | return BinaryOp_match<LHS, RHS, Instruction::Mul>(L, R); |
1048 | } |
1049 | |
1050 | template <typename LHS, typename RHS> |
1051 | inline BinaryOp_match<LHS, RHS, Instruction::FMul> m_FMul(const LHS &L, |
1052 | const RHS &R) { |
1053 | return BinaryOp_match<LHS, RHS, Instruction::FMul>(L, R); |
1054 | } |
1055 | |
1056 | template <typename LHS, typename RHS> |
1057 | inline BinaryOp_match<LHS, RHS, Instruction::UDiv> m_UDiv(const LHS &L, |
1058 | const RHS &R) { |
1059 | return BinaryOp_match<LHS, RHS, Instruction::UDiv>(L, R); |
1060 | } |
1061 | |
1062 | template <typename LHS, typename RHS> |
1063 | inline BinaryOp_match<LHS, RHS, Instruction::SDiv> m_SDiv(const LHS &L, |
1064 | const RHS &R) { |
1065 | return BinaryOp_match<LHS, RHS, Instruction::SDiv>(L, R); |
1066 | } |
1067 | |
1068 | template <typename LHS, typename RHS> |
1069 | inline BinaryOp_match<LHS, RHS, Instruction::FDiv> m_FDiv(const LHS &L, |
1070 | const RHS &R) { |
1071 | return BinaryOp_match<LHS, RHS, Instruction::FDiv>(L, R); |
1072 | } |
1073 | |
1074 | template <typename LHS, typename RHS> |
1075 | inline BinaryOp_match<LHS, RHS, Instruction::URem> m_URem(const LHS &L, |
1076 | const RHS &R) { |
1077 | return BinaryOp_match<LHS, RHS, Instruction::URem>(L, R); |
1078 | } |
1079 | |
1080 | template <typename LHS, typename RHS> |
1081 | inline BinaryOp_match<LHS, RHS, Instruction::SRem> m_SRem(const LHS &L, |
1082 | const RHS &R) { |
1083 | return BinaryOp_match<LHS, RHS, Instruction::SRem>(L, R); |
1084 | } |
1085 | |
1086 | template <typename LHS, typename RHS> |
1087 | inline BinaryOp_match<LHS, RHS, Instruction::FRem> m_FRem(const LHS &L, |
1088 | const RHS &R) { |
1089 | return BinaryOp_match<LHS, RHS, Instruction::FRem>(L, R); |
1090 | } |
1091 | |
1092 | template <typename LHS, typename RHS> |
1093 | inline BinaryOp_match<LHS, RHS, Instruction::And> m_And(const LHS &L, |
1094 | const RHS &R) { |
1095 | return BinaryOp_match<LHS, RHS, Instruction::And>(L, R); |
1096 | } |
1097 | |
1098 | template <typename LHS, typename RHS> |
1099 | inline BinaryOp_match<LHS, RHS, Instruction::Or> m_Or(const LHS &L, |
1100 | const RHS &R) { |
1101 | return BinaryOp_match<LHS, RHS, Instruction::Or>(L, R); |
1102 | } |
1103 | |
1104 | template <typename LHS, typename RHS> |
1105 | inline BinaryOp_match<LHS, RHS, Instruction::Xor> m_Xor(const LHS &L, |
1106 | const RHS &R) { |
1107 | return BinaryOp_match<LHS, RHS, Instruction::Xor>(L, R); |
1108 | } |
1109 | |
1110 | template <typename LHS, typename RHS> |
1111 | inline BinaryOp_match<LHS, RHS, Instruction::Shl> m_Shl(const LHS &L, |
1112 | const RHS &R) { |
1113 | return BinaryOp_match<LHS, RHS, Instruction::Shl>(L, R); |
1114 | } |
1115 | |
1116 | template <typename LHS, typename RHS> |
1117 | inline BinaryOp_match<LHS, RHS, Instruction::LShr> m_LShr(const LHS &L, |
1118 | const RHS &R) { |
1119 | return BinaryOp_match<LHS, RHS, Instruction::LShr>(L, R); |
1120 | } |
1121 | |
1122 | template <typename LHS, typename RHS> |
1123 | inline BinaryOp_match<LHS, RHS, Instruction::AShr> m_AShr(const LHS &L, |
1124 | const RHS &R) { |
1125 | return BinaryOp_match<LHS, RHS, Instruction::AShr>(L, R); |
1126 | } |
1127 | |
1128 | template <typename LHS_t, typename RHS_t, unsigned Opcode, |
1129 | unsigned WrapFlags = 0> |
1130 | struct OverflowingBinaryOp_match { |
1131 | LHS_t L; |
1132 | RHS_t R; |
1133 | |
1134 | OverflowingBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) |
1135 | : L(LHS), R(RHS) {} |
1136 | |
1137 | template <typename OpTy> bool match(OpTy *V) { |
1138 | if (auto *Op = dyn_cast<OverflowingBinaryOperator>(V)) { |
1139 | if (Op->getOpcode() != Opcode) |
1140 | return false; |
1141 | if ((WrapFlags & OverflowingBinaryOperator::NoUnsignedWrap) && |
1142 | !Op->hasNoUnsignedWrap()) |
1143 | return false; |
1144 | if ((WrapFlags & OverflowingBinaryOperator::NoSignedWrap) && |
1145 | !Op->hasNoSignedWrap()) |
1146 | return false; |
1147 | return L.match(Op->getOperand(0)) && R.match(Op->getOperand(1)); |
1148 | } |
1149 | return false; |
1150 | } |
1151 | }; |
1152 | |
1153 | template <typename LHS, typename RHS> |
1154 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, |
1155 | OverflowingBinaryOperator::NoSignedWrap> |
1156 | m_NSWAdd(const LHS &L, const RHS &R) { |
1157 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, |
1158 | OverflowingBinaryOperator::NoSignedWrap>(L, |
1159 | R); |
1160 | } |
1161 | template <typename LHS, typename RHS> |
1162 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, |
1163 | OverflowingBinaryOperator::NoSignedWrap> |
1164 | m_NSWSub(const LHS &L, const RHS &R) { |
1165 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, |
1166 | OverflowingBinaryOperator::NoSignedWrap>(L, |
1167 | R); |
1168 | } |
1169 | template <typename LHS, typename RHS> |
1170 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, |
1171 | OverflowingBinaryOperator::NoSignedWrap> |
1172 | m_NSWMul(const LHS &L, const RHS &R) { |
1173 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, |
1174 | OverflowingBinaryOperator::NoSignedWrap>(L, |
1175 | R); |
1176 | } |
1177 | template <typename LHS, typename RHS> |
1178 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, |
1179 | OverflowingBinaryOperator::NoSignedWrap> |
1180 | m_NSWShl(const LHS &L, const RHS &R) { |
1181 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, |
1182 | OverflowingBinaryOperator::NoSignedWrap>(L, |
1183 | R); |
1184 | } |
1185 | |
1186 | template <typename LHS, typename RHS> |
1187 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, |
1188 | OverflowingBinaryOperator::NoUnsignedWrap> |
1189 | m_NUWAdd(const LHS &L, const RHS &R) { |
1190 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, |
1191 | OverflowingBinaryOperator::NoUnsignedWrap>( |
1192 | L, R); |
1193 | } |
1194 | template <typename LHS, typename RHS> |
1195 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, |
1196 | OverflowingBinaryOperator::NoUnsignedWrap> |
1197 | m_NUWSub(const LHS &L, const RHS &R) { |
1198 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, |
1199 | OverflowingBinaryOperator::NoUnsignedWrap>( |
1200 | L, R); |
1201 | } |
1202 | template <typename LHS, typename RHS> |
1203 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, |
1204 | OverflowingBinaryOperator::NoUnsignedWrap> |
1205 | m_NUWMul(const LHS &L, const RHS &R) { |
1206 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, |
1207 | OverflowingBinaryOperator::NoUnsignedWrap>( |
1208 | L, R); |
1209 | } |
1210 | template <typename LHS, typename RHS> |
1211 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, |
1212 | OverflowingBinaryOperator::NoUnsignedWrap> |
1213 | m_NUWShl(const LHS &L, const RHS &R) { |
1214 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, |
1215 | OverflowingBinaryOperator::NoUnsignedWrap>( |
1216 | L, R); |
1217 | } |
1218 | |
1219 | template <typename LHS_t, typename RHS_t, bool Commutable = false> |
1220 | struct SpecificBinaryOp_match |
1221 | : public BinaryOp_match<LHS_t, RHS_t, 0, Commutable> { |
1222 | unsigned Opcode; |
1223 | |
1224 | SpecificBinaryOp_match(unsigned Opcode, const LHS_t &LHS, const RHS_t &RHS) |
1225 | : BinaryOp_match<LHS_t, RHS_t, 0, Commutable>(LHS, RHS), Opcode(Opcode) {} |
1226 | |
1227 | template <typename OpTy> bool match(OpTy *V) { |
1228 | return BinaryOp_match<LHS_t, RHS_t, 0, Commutable>::match(Opcode, V); |
1229 | } |
1230 | }; |
1231 | |
1232 | /// Matches a specific opcode. |
1233 | template <typename LHS, typename RHS> |
1234 | inline SpecificBinaryOp_match<LHS, RHS> m_BinOp(unsigned Opcode, const LHS &L, |
1235 | const RHS &R) { |
1236 | return SpecificBinaryOp_match<LHS, RHS>(Opcode, L, R); |
1237 | } |
1238 | |
1239 | //===----------------------------------------------------------------------===// |
1240 | // Class that matches a group of binary opcodes. |
1241 | // |
1242 | template <typename LHS_t, typename RHS_t, typename Predicate> |
1243 | struct BinOpPred_match : Predicate { |
1244 | LHS_t L; |
1245 | RHS_t R; |
1246 | |
1247 | BinOpPred_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} |
1248 | |
1249 | template <typename OpTy> bool match(OpTy *V) { |
1250 | if (auto *I = dyn_cast<Instruction>(V)) |
1251 | return this->isOpType(I->getOpcode()) && L.match(I->getOperand(0)) && |
1252 | R.match(I->getOperand(1)); |
1253 | if (auto *CE = dyn_cast<ConstantExpr>(V)) |
1254 | return this->isOpType(CE->getOpcode()) && L.match(CE->getOperand(0)) && |
1255 | R.match(CE->getOperand(1)); |
1256 | return false; |
1257 | } |
1258 | }; |
1259 | |
1260 | struct is_shift_op { |
1261 | bool isOpType(unsigned Opcode) { return Instruction::isShift(Opcode); } |
1262 | }; |
1263 | |
1264 | struct is_right_shift_op { |
1265 | bool isOpType(unsigned Opcode) { |
1266 | return Opcode == Instruction::LShr || Opcode == Instruction::AShr; |
1267 | } |
1268 | }; |
1269 | |
1270 | struct is_logical_shift_op { |
1271 | bool isOpType(unsigned Opcode) { |
1272 | return Opcode == Instruction::LShr || Opcode == Instruction::Shl; |
1273 | } |
1274 | }; |
1275 | |
1276 | struct is_bitwiselogic_op { |
1277 | bool isOpType(unsigned Opcode) { |
1278 | return Instruction::isBitwiseLogicOp(Opcode); |
1279 | } |
1280 | }; |
1281 | |
1282 | struct is_idiv_op { |
1283 | bool isOpType(unsigned Opcode) { |
1284 | return Opcode == Instruction::SDiv || Opcode == Instruction::UDiv; |
1285 | } |
1286 | }; |
1287 | |
1288 | struct is_irem_op { |
1289 | bool isOpType(unsigned Opcode) { |
1290 | return Opcode == Instruction::SRem || Opcode == Instruction::URem; |
1291 | } |
1292 | }; |
1293 | |
1294 | /// Matches shift operations. |
1295 | template <typename LHS, typename RHS> |
1296 | inline BinOpPred_match<LHS, RHS, is_shift_op> m_Shift(const LHS &L, |
1297 | const RHS &R) { |
1298 | return BinOpPred_match<LHS, RHS, is_shift_op>(L, R); |
1299 | } |
1300 | |
1301 | /// Matches logical shift operations. |
1302 | template <typename LHS, typename RHS> |
1303 | inline BinOpPred_match<LHS, RHS, is_right_shift_op> m_Shr(const LHS &L, |
1304 | const RHS &R) { |
1305 | return BinOpPred_match<LHS, RHS, is_right_shift_op>(L, R); |
1306 | } |
1307 | |
1308 | /// Matches logical shift operations. |
1309 | template <typename LHS, typename RHS> |
1310 | inline BinOpPred_match<LHS, RHS, is_logical_shift_op> |
1311 | m_LogicalShift(const LHS &L, const RHS &R) { |
1312 | return BinOpPred_match<LHS, RHS, is_logical_shift_op>(L, R); |
1313 | } |
1314 | |
1315 | /// Matches bitwise logic operations. |
1316 | template <typename LHS, typename RHS> |
1317 | inline BinOpPred_match<LHS, RHS, is_bitwiselogic_op> |
1318 | m_BitwiseLogic(const LHS &L, const RHS &R) { |
1319 | return BinOpPred_match<LHS, RHS, is_bitwiselogic_op>(L, R); |
1320 | } |
1321 | |
1322 | /// Matches integer division operations. |
1323 | template <typename LHS, typename RHS> |
1324 | inline BinOpPred_match<LHS, RHS, is_idiv_op> m_IDiv(const LHS &L, |
1325 | const RHS &R) { |
1326 | return BinOpPred_match<LHS, RHS, is_idiv_op>(L, R); |
1327 | } |
1328 | |
1329 | /// Matches integer remainder operations. |
1330 | template <typename LHS, typename RHS> |
1331 | inline BinOpPred_match<LHS, RHS, is_irem_op> m_IRem(const LHS &L, |
1332 | const RHS &R) { |
1333 | return BinOpPred_match<LHS, RHS, is_irem_op>(L, R); |
1334 | } |
1335 | |
1336 | //===----------------------------------------------------------------------===// |
1337 | // Class that matches exact binary ops. |
1338 | // |
1339 | template <typename SubPattern_t> struct Exact_match { |
1340 | SubPattern_t SubPattern; |
1341 | |
1342 | Exact_match(const SubPattern_t &SP) : SubPattern(SP) {} |
1343 | |
1344 | template <typename OpTy> bool match(OpTy *V) { |
1345 | if (auto *PEO = dyn_cast<PossiblyExactOperator>(V)) |
1346 | return PEO->isExact() && SubPattern.match(V); |
1347 | return false; |
1348 | } |
1349 | }; |
1350 | |
1351 | template <typename T> inline Exact_match<T> m_Exact(const T &SubPattern) { |
1352 | return SubPattern; |
1353 | } |
1354 | |
1355 | //===----------------------------------------------------------------------===// |
1356 | // Matchers for CmpInst classes |
1357 | // |
1358 | |
1359 | template <typename LHS_t, typename RHS_t, typename Class, typename PredicateTy, |
1360 | bool Commutable = false> |
1361 | struct CmpClass_match { |
1362 | PredicateTy &Predicate; |
1363 | LHS_t L; |
1364 | RHS_t R; |
1365 | |
1366 | // The evaluation order is always stable, regardless of Commutability. |
1367 | // The LHS is always matched first. |
1368 | CmpClass_match(PredicateTy &Pred, const LHS_t &LHS, const RHS_t &RHS) |
1369 | : Predicate(Pred), L(LHS), R(RHS) {} |
1370 | |
1371 | template <typename OpTy> bool match(OpTy *V) { |
1372 | if (auto *I = dyn_cast<Class>(V)) { |
1373 | if (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) { |
1374 | Predicate = I->getPredicate(); |
1375 | return true; |
1376 | } else if (Commutable && L.match(I->getOperand(1)) && |
1377 | R.match(I->getOperand(0))) { |
1378 | Predicate = I->getSwappedPredicate(); |
1379 | return true; |
1380 | } |
1381 | } |
1382 | return false; |
1383 | } |
1384 | }; |
1385 | |
1386 | template <typename LHS, typename RHS> |
1387 | inline CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate> |
1388 | m_Cmp(CmpInst::Predicate &Pred, const LHS &L, const RHS &R) { |
1389 | return CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate>(Pred, L, R); |
1390 | } |
1391 | |
1392 | template <typename LHS, typename RHS> |
1393 | inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate> |
1394 | m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { |
1395 | return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate>(Pred, L, R); |
1396 | } |
1397 | |
1398 | template <typename LHS, typename RHS> |
1399 | inline CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate> |
1400 | m_FCmp(FCmpInst::Predicate &Pred, const LHS &L, const RHS &R) { |
1401 | return CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate>(Pred, L, R); |
1402 | } |
1403 | |
1404 | //===----------------------------------------------------------------------===// |
1405 | // Matchers for instructions with a given opcode and number of operands. |
1406 | // |
1407 | |
1408 | /// Matches instructions with Opcode and three operands. |
1409 | template <typename T0, unsigned Opcode> struct OneOps_match { |
1410 | T0 Op1; |
1411 | |
1412 | OneOps_match(const T0 &Op1) : Op1(Op1) {} |
1413 | |
1414 | template <typename OpTy> bool match(OpTy *V) { |
1415 | if (V->getValueID() == Value::InstructionVal + Opcode) { |
1416 | auto *I = cast<Instruction>(V); |
1417 | return Op1.match(I->getOperand(0)); |
1418 | } |
1419 | return false; |
1420 | } |
1421 | }; |
1422 | |
1423 | /// Matches instructions with Opcode and three operands. |
1424 | template <typename T0, typename T1, unsigned Opcode> struct TwoOps_match { |
1425 | T0 Op1; |
1426 | T1 Op2; |
1427 | |
1428 | TwoOps_match(const T0 &Op1, const T1 &Op2) : Op1(Op1), Op2(Op2) {} |
1429 | |
1430 | template <typename OpTy> bool match(OpTy *V) { |
1431 | if (V->getValueID() == Value::InstructionVal + Opcode) { |
1432 | auto *I = cast<Instruction>(V); |
1433 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)); |
1434 | } |
1435 | return false; |
1436 | } |
1437 | }; |
1438 | |
1439 | /// Matches instructions with Opcode and three operands. |
1440 | template <typename T0, typename T1, typename T2, unsigned Opcode> |
1441 | struct ThreeOps_match { |
1442 | T0 Op1; |
1443 | T1 Op2; |
1444 | T2 Op3; |
1445 | |
1446 | ThreeOps_match(const T0 &Op1, const T1 &Op2, const T2 &Op3) |
1447 | : Op1(Op1), Op2(Op2), Op3(Op3) {} |
1448 | |
1449 | template <typename OpTy> bool match(OpTy *V) { |
1450 | if (V->getValueID() == Value::InstructionVal + Opcode) { |
1451 | auto *I = cast<Instruction>(V); |
1452 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) && |
1453 | Op3.match(I->getOperand(2)); |
1454 | } |
1455 | return false; |
1456 | } |
1457 | }; |
1458 | |
1459 | /// Matches SelectInst. |
1460 | template <typename Cond, typename LHS, typename RHS> |
1461 | inline ThreeOps_match<Cond, LHS, RHS, Instruction::Select> |
1462 | m_Select(const Cond &C, const LHS &L, const RHS &R) { |
1463 | return ThreeOps_match<Cond, LHS, RHS, Instruction::Select>(C, L, R); |
1464 | } |
1465 | |
1466 | /// This matches a select of two constants, e.g.: |
1467 | /// m_SelectCst<-1, 0>(m_Value(V)) |
1468 | template <int64_t L, int64_t R, typename Cond> |
1469 | inline ThreeOps_match<Cond, constantint_match<L>, constantint_match<R>, |
1470 | Instruction::Select> |
1471 | m_SelectCst(const Cond &C) { |
1472 | return m_Select(C, m_ConstantInt<L>(), m_ConstantInt<R>()); |
1473 | } |
1474 | |
1475 | /// Matches FreezeInst. |
1476 | template <typename OpTy> |
1477 | inline OneOps_match<OpTy, Instruction::Freeze> m_Freeze(const OpTy &Op) { |
1478 | return OneOps_match<OpTy, Instruction::Freeze>(Op); |
1479 | } |
1480 | |
1481 | /// Matches InsertElementInst. |
1482 | template <typename Val_t, typename Elt_t, typename Idx_t> |
1483 | inline ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement> |
1484 | m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx) { |
1485 | return ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement>( |
1486 | Val, Elt, Idx); |
1487 | } |
1488 | |
1489 | /// Matches ExtractElementInst. |
1490 | template <typename Val_t, typename Idx_t> |
1491 | inline TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement> |
1492 | m_ExtractElt(const Val_t &Val, const Idx_t &Idx) { |
1493 | return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx); |
1494 | } |
1495 | |
1496 | /// Matches shuffle. |
1497 | template <typename T0, typename T1, typename T2> struct Shuffle_match { |
1498 | T0 Op1; |
1499 | T1 Op2; |
1500 | T2 Mask; |
1501 | |
1502 | Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask) |
1503 | : Op1(Op1), Op2(Op2), Mask(Mask) {} |
1504 | |
1505 | template <typename OpTy> bool match(OpTy *V) { |
1506 | if (auto *I = dyn_cast<ShuffleVectorInst>(V)) { |
1507 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) && |
1508 | Mask.match(I->getShuffleMask()); |
1509 | } |
1510 | return false; |
1511 | } |
1512 | }; |
1513 | |
1514 | struct m_Mask { |
1515 | ArrayRef<int> &MaskRef; |
1516 | m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {} |
1517 | bool match(ArrayRef<int> Mask) { |
1518 | MaskRef = Mask; |
1519 | return true; |
1520 | } |
1521 | }; |
1522 | |
1523 | struct m_ZeroMask { |
1524 | bool match(ArrayRef<int> Mask) { |
1525 | return all_of(Mask, [](int Elem) { return Elem == 0 || Elem == -1; }); |
1526 | } |
1527 | }; |
1528 | |
1529 | struct m_SpecificMask { |
1530 | ArrayRef<int> &MaskRef; |
1531 | m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {} |
1532 | bool match(ArrayRef<int> Mask) { return MaskRef == Mask; } |
1533 | }; |
1534 | |
1535 | struct m_SplatOrUndefMask { |
1536 | int &SplatIndex; |
1537 | m_SplatOrUndefMask(int &SplatIndex) : SplatIndex(SplatIndex) {} |
1538 | bool match(ArrayRef<int> Mask) { |
1539 | const auto *First = find_if(Mask, [](int Elem) { return Elem != -1; }); |
1540 | if (First == Mask.end()) |
1541 | return false; |
1542 | SplatIndex = *First; |
1543 | return all_of(Mask, |
1544 | [First](int Elem) { return Elem == *First || Elem == -1; }); |
1545 | } |
1546 | }; |
1547 | |
1548 | /// Matches ShuffleVectorInst independently of mask value. |
1549 | template <typename V1_t, typename V2_t> |
1550 | inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector> |
1551 | m_Shuffle(const V1_t &v1, const V2_t &v2) { |
1552 | return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2); |
1553 | } |
1554 | |
1555 | template <typename V1_t, typename V2_t, typename Mask_t> |
1556 | inline Shuffle_match<V1_t, V2_t, Mask_t> |
1557 | m_Shuffle(const V1_t &v1, const V2_t &v2, const Mask_t &mask) { |
1558 | return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask); |
1559 | } |
1560 | |
1561 | /// Matches LoadInst. |
1562 | template <typename OpTy> |
1563 | inline OneOps_match<OpTy, Instruction::Load> m_Load(const OpTy &Op) { |
1564 | return OneOps_match<OpTy, Instruction::Load>(Op); |
1565 | } |
1566 | |
1567 | /// Matches StoreInst. |
1568 | template <typename ValueOpTy, typename PointerOpTy> |
1569 | inline TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store> |
1570 | m_Store(const ValueOpTy &ValueOp, const PointerOpTy &PointerOp) { |
1571 | return TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store>(ValueOp, |
1572 | PointerOp); |
1573 | } |
1574 | |
1575 | //===----------------------------------------------------------------------===// |
1576 | // Matchers for CastInst classes |
1577 | // |
1578 | |
1579 | template <typename Op_t, unsigned Opcode> struct CastClass_match { |
1580 | Op_t Op; |
1581 | |
1582 | CastClass_match(const Op_t &OpMatch) : Op(OpMatch) {} |
1583 | |
1584 | template <typename OpTy> bool match(OpTy *V) { |
1585 | if (auto *O = dyn_cast<Operator>(V)) |
1586 | return O->getOpcode() == Opcode && Op.match(O->getOperand(0)); |
1587 | return false; |
1588 | } |
1589 | }; |
1590 | |
1591 | /// Matches BitCast. |
1592 | template <typename OpTy> |
1593 | inline CastClass_match<OpTy, Instruction::BitCast> m_BitCast(const OpTy &Op) { |
1594 | return CastClass_match<OpTy, Instruction::BitCast>(Op); |
1595 | } |
1596 | |
1597 | /// Matches PtrToInt. |
1598 | template <typename OpTy> |
1599 | inline CastClass_match<OpTy, Instruction::PtrToInt> m_PtrToInt(const OpTy &Op) { |
1600 | return CastClass_match<OpTy, Instruction::PtrToInt>(Op); |
1601 | } |
1602 | |
1603 | /// Matches IntToPtr. |
1604 | template <typename OpTy> |
1605 | inline CastClass_match<OpTy, Instruction::IntToPtr> m_IntToPtr(const OpTy &Op) { |
1606 | return CastClass_match<OpTy, Instruction::IntToPtr>(Op); |
1607 | } |
1608 | |
1609 | /// Matches Trunc. |
1610 | template <typename OpTy> |
1611 | inline CastClass_match<OpTy, Instruction::Trunc> m_Trunc(const OpTy &Op) { |
1612 | return CastClass_match<OpTy, Instruction::Trunc>(Op); |
1613 | } |
1614 | |
1615 | template <typename OpTy> |
1616 | inline match_combine_or<CastClass_match<OpTy, Instruction::Trunc>, OpTy> |
1617 | m_TruncOrSelf(const OpTy &Op) { |
1618 | return m_CombineOr(m_Trunc(Op), Op); |
1619 | } |
1620 | |
1621 | /// Matches SExt. |
1622 | template <typename OpTy> |
1623 | inline CastClass_match<OpTy, Instruction::SExt> m_SExt(const OpTy &Op) { |
1624 | return CastClass_match<OpTy, Instruction::SExt>(Op); |
1625 | } |
1626 | |
1627 | /// Matches ZExt. |
1628 | template <typename OpTy> |
1629 | inline CastClass_match<OpTy, Instruction::ZExt> m_ZExt(const OpTy &Op) { |
1630 | return CastClass_match<OpTy, Instruction::ZExt>(Op); |
1631 | } |
1632 | |
1633 | template <typename OpTy> |
1634 | inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, OpTy> |
1635 | m_ZExtOrSelf(const OpTy &Op) { |
1636 | return m_CombineOr(m_ZExt(Op), Op); |
1637 | } |
1638 | |
1639 | template <typename OpTy> |
1640 | inline match_combine_or<CastClass_match<OpTy, Instruction::SExt>, OpTy> |
1641 | m_SExtOrSelf(const OpTy &Op) { |
1642 | return m_CombineOr(m_SExt(Op), Op); |
1643 | } |
1644 | |
1645 | template <typename OpTy> |
1646 | inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, |
1647 | CastClass_match<OpTy, Instruction::SExt>> |
1648 | m_ZExtOrSExt(const OpTy &Op) { |
1649 | return m_CombineOr(m_ZExt(Op), m_SExt(Op)); |
1650 | } |
1651 | |
1652 | template <typename OpTy> |
1653 | inline match_combine_or< |
1654 | match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, |
1655 | CastClass_match<OpTy, Instruction::SExt>>, |
1656 | OpTy> |
1657 | m_ZExtOrSExtOrSelf(const OpTy &Op) { |
1658 | return m_CombineOr(m_ZExtOrSExt(Op), Op); |
1659 | } |
1660 | |
1661 | template <typename OpTy> |
1662 | inline CastClass_match<OpTy, Instruction::UIToFP> m_UIToFP(const OpTy &Op) { |
1663 | return CastClass_match<OpTy, Instruction::UIToFP>(Op); |
1664 | } |
1665 | |
1666 | template <typename OpTy> |
1667 | inline CastClass_match<OpTy, Instruction::SIToFP> m_SIToFP(const OpTy &Op) { |
1668 | return CastClass_match<OpTy, Instruction::SIToFP>(Op); |
1669 | } |
1670 | |
1671 | template <typename OpTy> |
1672 | inline CastClass_match<OpTy, Instruction::FPToUI> m_FPToUI(const OpTy &Op) { |
1673 | return CastClass_match<OpTy, Instruction::FPToUI>(Op); |
1674 | } |
1675 | |
1676 | template <typename OpTy> |
1677 | inline CastClass_match<OpTy, Instruction::FPToSI> m_FPToSI(const OpTy &Op) { |
1678 | return CastClass_match<OpTy, Instruction::FPToSI>(Op); |
1679 | } |
1680 | |
1681 | template <typename OpTy> |
1682 | inline CastClass_match<OpTy, Instruction::FPTrunc> m_FPTrunc(const OpTy &Op) { |
1683 | return CastClass_match<OpTy, Instruction::FPTrunc>(Op); |
1684 | } |
1685 | |
1686 | template <typename OpTy> |
1687 | inline CastClass_match<OpTy, Instruction::FPExt> m_FPExt(const OpTy &Op) { |
1688 | return CastClass_match<OpTy, Instruction::FPExt>(Op); |
1689 | } |
1690 | |
1691 | //===----------------------------------------------------------------------===// |
1692 | // Matchers for control flow. |
1693 | // |
1694 | |
1695 | struct br_match { |
1696 | BasicBlock *&Succ; |
1697 | |
1698 | br_match(BasicBlock *&Succ) : Succ(Succ) {} |
1699 | |
1700 | template <typename OpTy> bool match(OpTy *V) { |
1701 | if (auto *BI = dyn_cast<BranchInst>(V)) |
1702 | if (BI->isUnconditional()) { |
1703 | Succ = BI->getSuccessor(0); |
1704 | return true; |
1705 | } |
1706 | return false; |
1707 | } |
1708 | }; |
1709 | |
1710 | inline br_match m_UnconditionalBr(BasicBlock *&Succ) { return br_match(Succ); } |
1711 | |
1712 | template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> |
1713 | struct brc_match { |
1714 | Cond_t Cond; |
1715 | TrueBlock_t T; |
1716 | FalseBlock_t F; |
1717 | |
1718 | brc_match(const Cond_t &C, const TrueBlock_t &t, const FalseBlock_t &f) |
1719 | : Cond(C), T(t), F(f) {} |
1720 | |
1721 | template <typename OpTy> bool match(OpTy *V) { |
1722 | if (auto *BI = dyn_cast<BranchInst>(V)) |
1723 | if (BI->isConditional() && Cond.match(BI->getCondition())) |
1724 | return T.match(BI->getSuccessor(0)) && F.match(BI->getSuccessor(1)); |
1725 | return false; |
1726 | } |
1727 | }; |
1728 | |
1729 | template <typename Cond_t> |
1730 | inline brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>> |
1731 | m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) { |
1732 | return brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>( |
1733 | C, m_BasicBlock(T), m_BasicBlock(F)); |
1734 | } |
1735 | |
1736 | template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> |
1737 | inline brc_match<Cond_t, TrueBlock_t, FalseBlock_t> |
1738 | m_Br(const Cond_t &C, const TrueBlock_t &T, const FalseBlock_t &F) { |
1739 | return brc_match<Cond_t, TrueBlock_t, FalseBlock_t>(C, T, F); |
1740 | } |
1741 | |
1742 | //===----------------------------------------------------------------------===// |
1743 | // Matchers for max/min idioms, eg: "select (sgt x, y), x, y" -> smax(x,y). |
1744 | // |
1745 | |
1746 | template <typename CmpInst_t, typename LHS_t, typename RHS_t, typename Pred_t, |
1747 | bool Commutable = false> |
1748 | struct MaxMin_match { |
1749 | using PredType = Pred_t; |
1750 | LHS_t L; |
1751 | RHS_t R; |
1752 | |
1753 | // The evaluation order is always stable, regardless of Commutability. |
1754 | // The LHS is always matched first. |
1755 | MaxMin_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} |
1756 | |
1757 | template <typename OpTy> bool match(OpTy *V) { |
1758 | if (auto *II = dyn_cast<IntrinsicInst>(V)) { |
1759 | Intrinsic::ID IID = II->getIntrinsicID(); |
1760 | if ((IID == Intrinsic::smax && Pred_t::match(ICmpInst::ICMP_SGT)) || |
1761 | (IID == Intrinsic::smin && Pred_t::match(ICmpInst::ICMP_SLT)) || |
1762 | (IID == Intrinsic::umax && Pred_t::match(ICmpInst::ICMP_UGT)) || |
1763 | (IID == Intrinsic::umin && Pred_t::match(ICmpInst::ICMP_ULT))) { |
1764 | Value *LHS = II->getOperand(0), *RHS = II->getOperand(1); |
1765 | return (L.match(LHS) && R.match(RHS)) || |
1766 | (Commutable && L.match(RHS) && R.match(LHS)); |
1767 | } |
1768 | } |
1769 | // Look for "(x pred y) ? x : y" or "(x pred y) ? y : x". |
1770 | auto *SI = dyn_cast<SelectInst>(V); |
1771 | if (!SI) |
1772 | return false; |
1773 | auto *Cmp = dyn_cast<CmpInst_t>(SI->getCondition()); |
1774 | if (!Cmp) |
1775 | return false; |
1776 | // At this point we have a select conditioned on a comparison. Check that |
1777 | // it is the values returned by the select that are being compared. |
1778 | auto *TrueVal = SI->getTrueValue(); |
1779 | auto *FalseVal = SI->getFalseValue(); |
1780 | auto *LHS = Cmp->getOperand(0); |
1781 | auto *RHS = Cmp->getOperand(1); |
1782 | if ((TrueVal != LHS || FalseVal != RHS) && |
1783 | (TrueVal != RHS || FalseVal != LHS)) |
1784 | return false; |
1785 | typename CmpInst_t::Predicate Pred = |
1786 | LHS == TrueVal ? Cmp->getPredicate() : Cmp->getInversePredicate(); |
1787 | // Does "(x pred y) ? x : y" represent the desired max/min operation? |
1788 | if (!Pred_t::match(Pred)) |
1789 | return false; |
1790 | // It does! Bind the operands. |
1791 | return (L.match(LHS) && R.match(RHS)) || |
1792 | (Commutable && L.match(RHS) && R.match(LHS)); |
1793 | } |
1794 | }; |
1795 | |
1796 | /// Helper class for identifying signed max predicates. |
1797 | struct smax_pred_ty { |
1798 | static bool match(ICmpInst::Predicate Pred) { |
1799 | return Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SGE; |
1800 | } |
1801 | }; |
1802 | |
1803 | /// Helper class for identifying signed min predicates. |
1804 | struct smin_pred_ty { |
1805 | static bool match(ICmpInst::Predicate Pred) { |
1806 | return Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SLE; |
1807 | } |
1808 | }; |
1809 | |
1810 | /// Helper class for identifying unsigned max predicates. |
1811 | struct umax_pred_ty { |
1812 | static bool match(ICmpInst::Predicate Pred) { |
1813 | return Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_UGE; |
1814 | } |
1815 | }; |
1816 | |
1817 | /// Helper class for identifying unsigned min predicates. |
1818 | struct umin_pred_ty { |
1819 | static bool match(ICmpInst::Predicate Pred) { |
1820 | return Pred == CmpInst::ICMP_ULT || Pred == CmpInst::ICMP_ULE; |
1821 | } |
1822 | }; |
1823 | |
1824 | /// Helper class for identifying ordered max predicates. |
1825 | struct ofmax_pred_ty { |
1826 | static bool match(FCmpInst::Predicate Pred) { |
1827 | return Pred == CmpInst::FCMP_OGT || Pred == CmpInst::FCMP_OGE; |
1828 | } |
1829 | }; |
1830 | |
1831 | /// Helper class for identifying ordered min predicates. |
1832 | struct ofmin_pred_ty { |
1833 | static bool match(FCmpInst::Predicate Pred) { |
1834 | return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE; |
1835 | } |
1836 | }; |
1837 | |
1838 | /// Helper class for identifying unordered max predicates. |
1839 | struct ufmax_pred_ty { |
1840 | static bool match(FCmpInst::Predicate Pred) { |
1841 | return Pred == CmpInst::FCMP_UGT || Pred == CmpInst::FCMP_UGE; |
1842 | } |
1843 | }; |
1844 | |
1845 | /// Helper class for identifying unordered min predicates. |
1846 | struct ufmin_pred_ty { |
1847 | static bool match(FCmpInst::Predicate Pred) { |
1848 | return Pred == CmpInst::FCMP_ULT || Pred == CmpInst::FCMP_ULE; |
1849 | } |
1850 | }; |
1851 | |
1852 | template <typename LHS, typename RHS> |
1853 | inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty> m_SMax(const LHS &L, |
1854 | const RHS &R) { |
1855 | return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>(L, R); |
1856 | } |
1857 | |
1858 | template <typename LHS, typename RHS> |
1859 | inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty> m_SMin(const LHS &L, |
1860 | const RHS &R) { |
1861 | return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>(L, R); |
1862 | } |
1863 | |
1864 | template <typename LHS, typename RHS> |
1865 | inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty> m_UMax(const LHS &L, |
1866 | const RHS &R) { |
1867 | return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>(L, R); |
1868 | } |
1869 | |
1870 | template <typename LHS, typename RHS> |
1871 | inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty> m_UMin(const LHS &L, |
1872 | const RHS &R) { |
1873 | return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>(L, R); |
1874 | } |
1875 | |
1876 | template <typename LHS, typename RHS> |
1877 | inline match_combine_or< |
1878 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>, |
1879 | MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>>, |
1880 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>, |
1881 | MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>>> |
1882 | m_MaxOrMin(const LHS &L, const RHS &R) { |
1883 | return m_CombineOr(m_CombineOr(m_SMax(L, R), m_SMin(L, R)), |
1884 | m_CombineOr(m_UMax(L, R), m_UMin(L, R))); |
1885 | } |
1886 | |
1887 | /// Match an 'ordered' floating point maximum function. |
1888 | /// Floating point has one special value 'NaN'. Therefore, there is no total |
1889 | /// order. However, if we can ignore the 'NaN' value (for example, because of a |
1890 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum' |
1891 | /// semantics. In the presence of 'NaN' we have to preserve the original |
1892 | /// select(fcmp(ogt/ge, L, R), L, R) semantics matched by this predicate. |
1893 | /// |
1894 | /// max(L, R) iff L and R are not NaN |
1895 | /// m_OrdFMax(L, R) = R iff L or R are NaN |
1896 | template <typename LHS, typename RHS> |
1897 | inline MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty> m_OrdFMax(const LHS &L, |
1898 | const RHS &R) { |
1899 | return MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty>(L, R); |
1900 | } |
1901 | |
1902 | /// Match an 'ordered' floating point minimum function. |
1903 | /// Floating point has one special value 'NaN'. Therefore, there is no total |
1904 | /// order. However, if we can ignore the 'NaN' value (for example, because of a |
1905 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum' |
1906 | /// semantics. In the presence of 'NaN' we have to preserve the original |
1907 | /// select(fcmp(olt/le, L, R), L, R) semantics matched by this predicate. |
1908 | /// |
1909 | /// min(L, R) iff L and R are not NaN |
1910 | /// m_OrdFMin(L, R) = R iff L or R are NaN |
1911 | template <typename LHS, typename RHS> |
1912 | inline MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty> m_OrdFMin(const LHS &L, |
1913 | const RHS &R) { |
1914 | return MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty>(L, R); |
1915 | } |
1916 | |
1917 | /// Match an 'unordered' floating point maximum function. |
1918 | /// Floating point has one special value 'NaN'. Therefore, there is no total |
1919 | /// order. However, if we can ignore the 'NaN' value (for example, because of a |
1920 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum' |
1921 | /// semantics. In the presence of 'NaN' we have to preserve the original |
1922 | /// select(fcmp(ugt/ge, L, R), L, R) semantics matched by this predicate. |
1923 | /// |
1924 | /// max(L, R) iff L and R are not NaN |
1925 | /// m_UnordFMax(L, R) = L iff L or R are NaN |
1926 | template <typename LHS, typename RHS> |
1927 | inline MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty> |
1928 | m_UnordFMax(const LHS &L, const RHS &R) { |
1929 | return MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>(L, R); |
1930 | } |
1931 | |
1932 | /// Match an 'unordered' floating point minimum function. |
1933 | /// Floating point has one special value 'NaN'. Therefore, there is no total |
1934 | /// order. However, if we can ignore the 'NaN' value (for example, because of a |
1935 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum' |
1936 | /// semantics. In the presence of 'NaN' we have to preserve the original |
1937 | /// select(fcmp(ult/le, L, R), L, R) semantics matched by this predicate. |
1938 | /// |
1939 | /// min(L, R) iff L and R are not NaN |
1940 | /// m_UnordFMin(L, R) = L iff L or R are NaN |
1941 | template <typename LHS, typename RHS> |
1942 | inline MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty> |
1943 | m_UnordFMin(const LHS &L, const RHS &R) { |
1944 | return MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>(L, R); |
1945 | } |
1946 | |
1947 | //===----------------------------------------------------------------------===// |
1948 | // Matchers for overflow check patterns: e.g. (a + b) u< a, (a ^ -1) <u b |
1949 | // Note that S might be matched to other instructions than AddInst. |
1950 | // |
1951 | |
1952 | template <typename LHS_t, typename RHS_t, typename Sum_t> |
1953 | struct UAddWithOverflow_match { |
1954 | LHS_t L; |
1955 | RHS_t R; |
1956 | Sum_t S; |
1957 | |
1958 | UAddWithOverflow_match(const LHS_t &L, const RHS_t &R, const Sum_t &S) |
1959 | : L(L), R(R), S(S) {} |
1960 | |
1961 | template <typename OpTy> bool match(OpTy *V) { |
1962 | Value *ICmpLHS, *ICmpRHS; |
1963 | ICmpInst::Predicate Pred; |
1964 | if (!m_ICmp(Pred, m_Value(ICmpLHS), m_Value(ICmpRHS)).match(V)) |
1965 | return false; |
1966 | |
1967 | Value *AddLHS, *AddRHS; |
1968 | auto AddExpr = m_Add(m_Value(AddLHS), m_Value(AddRHS)); |
1969 | |
1970 | // (a + b) u< a, (a + b) u< b |
1971 | if (Pred == ICmpInst::ICMP_ULT) |
1972 | if (AddExpr.match(ICmpLHS) && (ICmpRHS == AddLHS || ICmpRHS == AddRHS)) |
1973 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS); |
1974 | |
1975 | // a >u (a + b), b >u (a + b) |
1976 | if (Pred == ICmpInst::ICMP_UGT) |
1977 | if (AddExpr.match(ICmpRHS) && (ICmpLHS == AddLHS || ICmpLHS == AddRHS)) |
1978 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); |
1979 | |
1980 | Value *Op1; |
1981 | auto XorExpr = m_OneUse(m_Xor(m_Value(Op1), m_AllOnes())); |
1982 | // (a ^ -1) <u b |
1983 | if (Pred == ICmpInst::ICMP_ULT) { |
1984 | if (XorExpr.match(ICmpLHS)) |
1985 | return L.match(Op1) && R.match(ICmpRHS) && S.match(ICmpLHS); |
1986 | } |
1987 | // b > u (a ^ -1) |
1988 | if (Pred == ICmpInst::ICMP_UGT) { |
1989 | if (XorExpr.match(ICmpRHS)) |
1990 | return L.match(Op1) && R.match(ICmpLHS) && S.match(ICmpRHS); |
1991 | } |
1992 | |
1993 | // Match special-case for increment-by-1. |
1994 | if (Pred == ICmpInst::ICMP_EQ) { |
1995 | // (a + 1) == 0 |
1996 | // (1 + a) == 0 |
1997 | if (AddExpr.match(ICmpLHS) && m_ZeroInt().match(ICmpRHS) && |
1998 | (m_One().match(AddLHS) || m_One().match(AddRHS))) |
1999 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS); |
2000 | // 0 == (a + 1) |
2001 | // 0 == (1 + a) |
2002 | if (m_ZeroInt().match(ICmpLHS) && AddExpr.match(ICmpRHS) && |
2003 | (m_One().match(AddLHS) || m_One().match(AddRHS))) |
2004 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); |
2005 | } |
2006 | |
2007 | return false; |
2008 | } |
2009 | }; |
2010 | |
2011 | /// Match an icmp instruction checking for unsigned overflow on addition. |
2012 | /// |
2013 | /// S is matched to the addition whose result is being checked for overflow, and |
2014 | /// L and R are matched to the LHS and RHS of S. |
2015 | template <typename LHS_t, typename RHS_t, typename Sum_t> |
2016 | UAddWithOverflow_match<LHS_t, RHS_t, Sum_t> |
2017 | m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S) { |
2018 | return UAddWithOverflow_match<LHS_t, RHS_t, Sum_t>(L, R, S); |
2019 | } |
2020 | |
2021 | template <typename Opnd_t> struct Argument_match { |
2022 | unsigned OpI; |
2023 | Opnd_t Val; |
2024 | |
2025 | Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {} |
2026 | |
2027 | template <typename OpTy> bool match(OpTy *V) { |
2028 | // FIXME: Should likely be switched to use `CallBase`. |
2029 | if (const auto *CI = dyn_cast<CallInst>(V)) |
2030 | return Val.match(CI->getArgOperand(OpI)); |
2031 | return false; |
2032 | } |
2033 | }; |
2034 | |
2035 | /// Match an argument. |
2036 | template <unsigned OpI, typename Opnd_t> |
2037 | inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) { |
2038 | return Argument_match<Opnd_t>(OpI, Op); |
2039 | } |
2040 | |
2041 | /// Intrinsic matchers. |
2042 | struct IntrinsicID_match { |
2043 | unsigned ID; |
2044 | |
2045 | IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {} |
2046 | |
2047 | template <typename OpTy> bool match(OpTy *V) { |
2048 | if (const auto *CI = dyn_cast<CallInst>(V)) |
2049 | if (const auto *F = CI->getCalledFunction()) |
2050 | return F->getIntrinsicID() == ID; |
2051 | return false; |
2052 | } |
2053 | }; |
2054 | |
2055 | /// Intrinsic matches are combinations of ID matchers, and argument |
2056 | /// matchers. Higher arity matcher are defined recursively in terms of and-ing |
2057 | /// them with lower arity matchers. Here's some convenient typedefs for up to |
2058 | /// several arguments, and more can be added as needed |
2059 | template <typename T0 = void, typename T1 = void, typename T2 = void, |
2060 | typename T3 = void, typename T4 = void, typename T5 = void, |
2061 | typename T6 = void, typename T7 = void, typename T8 = void, |
2062 | typename T9 = void, typename T10 = void> |
2063 | struct m_Intrinsic_Ty; |
2064 | template <typename T0> struct m_Intrinsic_Ty<T0> { |
2065 | using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>; |
2066 | }; |
2067 | template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> { |
2068 | using Ty = |
2069 | match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>; |
2070 | }; |
2071 | template <typename T0, typename T1, typename T2> |
2072 | struct m_Intrinsic_Ty<T0, T1, T2> { |
2073 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty, |
2074 | Argument_match<T2>>; |
2075 | }; |
2076 | template <typename T0, typename T1, typename T2, typename T3> |
2077 | struct m_Intrinsic_Ty<T0, T1, T2, T3> { |
2078 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty, |
2079 | Argument_match<T3>>; |
2080 | }; |
2081 | |
2082 | template <typename T0, typename T1, typename T2, typename T3, typename T4> |
2083 | struct m_Intrinsic_Ty<T0, T1, T2, T3, T4> { |
2084 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty, |
2085 | Argument_match<T4>>; |
2086 | }; |
2087 | |
2088 | template <typename T0, typename T1, typename T2, typename T3, typename T4, |
2089 | typename T5> |
2090 | struct m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5> { |
2091 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty, |
2092 | Argument_match<T5>>; |
2093 | }; |
2094 | |
2095 | /// Match intrinsic calls like this: |
2096 | /// m_Intrinsic<Intrinsic::fabs>(m_Value(X)) |
2097 | template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() { |
2098 | return IntrinsicID_match(IntrID); |
2099 | } |
2100 | |
2101 | /// Matches MaskedLoad Intrinsic. |
2102 | template <typename Opnd0, typename Opnd1, typename Opnd2, typename Opnd3> |
2103 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2, Opnd3>::Ty |
2104 | m_MaskedLoad(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2, |
2105 | const Opnd3 &Op3) { |
2106 | return m_Intrinsic<Intrinsic::masked_load>(Op0, Op1, Op2, Op3); |
2107 | } |
2108 | |
2109 | /// Matches MaskedGather Intrinsic. |
2110 | template <typename Opnd0, typename Opnd1, typename Opnd2, typename Opnd3> |
2111 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2, Opnd3>::Ty |
2112 | m_MaskedGather(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2, |
2113 | const Opnd3 &Op3) { |
2114 | return m_Intrinsic<Intrinsic::masked_gather>(Op0, Op1, Op2, Op3); |
2115 | } |
2116 | |
2117 | template <Intrinsic::ID IntrID, typename T0> |
2118 | inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) { |
2119 | return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0)); |
2120 | } |
2121 | |
2122 | template <Intrinsic::ID IntrID, typename T0, typename T1> |
2123 | inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0, |
2124 | const T1 &Op1) { |
2125 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1)); |
2126 | } |
2127 | |
2128 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2> |
2129 | inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty |
2130 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) { |
2131 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2)); |
2132 | } |
2133 | |
2134 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, |
2135 | typename T3> |
2136 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty |
2137 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) { |
2138 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3)); |
2139 | } |
2140 | |
2141 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, |
2142 | typename T3, typename T4> |
2143 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty |
2144 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3, |
2145 | const T4 &Op4) { |
2146 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3), |
2147 | m_Argument<4>(Op4)); |
2148 | } |
2149 | |
2150 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, |
2151 | typename T3, typename T4, typename T5> |
2152 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5>::Ty |
2153 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3, |
2154 | const T4 &Op4, const T5 &Op5) { |
2155 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3, Op4), |
2156 | m_Argument<5>(Op5)); |
2157 | } |
2158 | |
2159 | // Helper intrinsic matching specializations. |
2160 | template <typename Opnd0> |
2161 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BitReverse(const Opnd0 &Op0) { |
2162 | return m_Intrinsic<Intrinsic::bitreverse>(Op0); |
2163 | } |
2164 | |
2165 | template <typename Opnd0> |
2166 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BSwap(const Opnd0 &Op0) { |
2167 | return m_Intrinsic<Intrinsic::bswap>(Op0); |
2168 | } |
2169 | |
2170 | template <typename Opnd0> |
2171 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FAbs(const Opnd0 &Op0) { |
2172 | return m_Intrinsic<Intrinsic::fabs>(Op0); |
2173 | } |
2174 | |
2175 | template <typename Opnd0> |
2176 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FCanonicalize(const Opnd0 &Op0) { |
2177 | return m_Intrinsic<Intrinsic::canonicalize>(Op0); |
2178 | } |
2179 | |
2180 | template <typename Opnd0, typename Opnd1> |
2181 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMin(const Opnd0 &Op0, |
2182 | const Opnd1 &Op1) { |
2183 | return m_Intrinsic<Intrinsic::minnum>(Op0, Op1); |
2184 | } |
2185 | |
2186 | template <typename Opnd0, typename Opnd1> |
2187 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMax(const Opnd0 &Op0, |
2188 | const Opnd1 &Op1) { |
2189 | return m_Intrinsic<Intrinsic::maxnum>(Op0, Op1); |
2190 | } |
2191 | |
2192 | template <typename Opnd0, typename Opnd1, typename Opnd2> |
2193 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty |
2194 | m_FShl(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) { |
2195 | return m_Intrinsic<Intrinsic::fshl>(Op0, Op1, Op2); |
2196 | } |
2197 | |
2198 | template <typename Opnd0, typename Opnd1, typename Opnd2> |
2199 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty |
2200 | m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) { |
2201 | return m_Intrinsic<Intrinsic::fshr>(Op0, Op1, Op2); |
2202 | } |
2203 | |
2204 | template <typename Opnd0> |
2205 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_Sqrt(const Opnd0 &Op0) { |
2206 | return m_Intrinsic<Intrinsic::sqrt>(Op0); |
2207 | } |
2208 | |
2209 | template <typename Opnd0, typename Opnd1> |
2210 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_CopySign(const Opnd0 &Op0, |
2211 | const Opnd1 &Op1) { |
2212 | return m_Intrinsic<Intrinsic::copysign>(Op0, Op1); |
2213 | } |
2214 | |
2215 | template <typename Opnd0> |
2216 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_VecReverse(const Opnd0 &Op0) { |
2217 | return m_Intrinsic<Intrinsic::experimental_vector_reverse>(Op0); |
2218 | } |
2219 | |
2220 | //===----------------------------------------------------------------------===// |
2221 | // Matchers for two-operands operators with the operators in either order |
2222 | // |
2223 | |
2224 | /// Matches a BinaryOperator with LHS and RHS in either order. |
2225 | template <typename LHS, typename RHS> |
2226 | inline AnyBinaryOp_match<LHS, RHS, true> m_c_BinOp(const LHS &L, const RHS &R) { |
2227 | return AnyBinaryOp_match<LHS, RHS, true>(L, R); |
2228 | } |
2229 | |
2230 | /// Matches an ICmp with a predicate over LHS and RHS in either order. |
2231 | /// Swaps the predicate if operands are commuted. |
2232 | template <typename LHS, typename RHS> |
2233 | inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true> |
2234 | m_c_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { |
2235 | return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true>(Pred, L, |
2236 | R); |
2237 | } |
2238 | |
2239 | /// Matches a specific opcode with LHS and RHS in either order. |
2240 | template <typename LHS, typename RHS> |
2241 | inline SpecificBinaryOp_match<LHS, RHS, true> |
2242 | m_c_BinOp(unsigned Opcode, const LHS &L, const RHS &R) { |
2243 | return SpecificBinaryOp_match<LHS, RHS, true>(Opcode, L, R); |
2244 | } |
2245 | |
2246 | /// Matches a Add with LHS and RHS in either order. |
2247 | template <typename LHS, typename RHS> |
2248 | inline BinaryOp_match<LHS, RHS, Instruction::Add, true> m_c_Add(const LHS &L, |
2249 | const RHS &R) { |
2250 | return BinaryOp_match<LHS, RHS, Instruction::Add, true>(L, R); |
2251 | } |
2252 | |
2253 | /// Matches a Mul with LHS and RHS in either order. |
2254 | template <typename LHS, typename RHS> |
2255 | inline BinaryOp_match<LHS, RHS, Instruction::Mul, true> m_c_Mul(const LHS &L, |
2256 | const RHS &R) { |
2257 | return BinaryOp_match<LHS, RHS, Instruction::Mul, true>(L, R); |
2258 | } |
2259 | |
2260 | /// Matches an And with LHS and RHS in either order. |
2261 | template <typename LHS, typename RHS> |
2262 | inline BinaryOp_match<LHS, RHS, Instruction::And, true> m_c_And(const LHS &L, |
2263 | const RHS &R) { |
2264 | return BinaryOp_match<LHS, RHS, Instruction::And, true>(L, R); |
2265 | } |
2266 | |
2267 | /// Matches an Or with LHS and RHS in either order. |
2268 | template <typename LHS, typename RHS> |
2269 | inline BinaryOp_match<LHS, RHS, Instruction::Or, true> m_c_Or(const LHS &L, |
2270 | const RHS &R) { |
2271 | return BinaryOp_match<LHS, RHS, Instruction::Or, true>(L, R); |
2272 | } |
2273 | |
2274 | /// Matches an Xor with LHS and RHS in either order. |
2275 | template <typename LHS, typename RHS> |
2276 | inline BinaryOp_match<LHS, RHS, Instruction::Xor, true> m_c_Xor(const LHS &L, |
2277 | const RHS &R) { |
2278 | return BinaryOp_match<LHS, RHS, Instruction::Xor, true>(L, R); |
2279 | } |
2280 | |
2281 | /// Matches a 'Neg' as 'sub 0, V'. |
2282 | template <typename ValTy> |
2283 | inline BinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, Instruction::Sub> |
2284 | m_Neg(const ValTy &V) { |
2285 | return m_Sub(m_ZeroInt(), V); |
2286 | } |
2287 | |
2288 | /// Matches a 'Neg' as 'sub nsw 0, V'. |
2289 | template <typename ValTy> |
2290 | inline OverflowingBinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, |
2291 | Instruction::Sub, |
2292 | OverflowingBinaryOperator::NoSignedWrap> |
2293 | m_NSWNeg(const ValTy &V) { |
2294 | return m_NSWSub(m_ZeroInt(), V); |
2295 | } |
2296 | |
2297 | /// Matches a 'Not' as 'xor V, -1' or 'xor -1, V'. |
2298 | /// NOTE: we first match the 'Not' (by matching '-1'), |
2299 | /// and only then match the inner matcher! |
2300 | template <typename ValTy> |
2301 | inline BinaryOp_match<cst_pred_ty<is_all_ones>, ValTy, Instruction::Xor, true> |
2302 | m_Not(const ValTy &V) { |
2303 | return m_c_Xor(m_AllOnes(), V); |
2304 | } |
2305 | |
2306 | template <typename ValTy> struct NotForbidUndef_match { |
2307 | ValTy Val; |
2308 | NotForbidUndef_match(const ValTy &V) : Val(V) {} |
2309 | |
2310 | template <typename OpTy> bool match(OpTy *V) { |
2311 | // We do not use m_c_Xor because that could match an arbitrary APInt that is |
2312 | // not -1 as C and then fail to match the other operand if it is -1. |
2313 | // This code should still work even when both operands are constants. |
2314 | Value *X; |
2315 | const APInt *C; |
2316 | if (m_Xor(m_Value(X), m_APIntForbidUndef(C)).match(V) && C->isAllOnes()) |
2317 | return Val.match(X); |
2318 | if (m_Xor(m_APIntForbidUndef(C), m_Value(X)).match(V) && C->isAllOnes()) |
2319 | return Val.match(X); |
2320 | return false; |
2321 | } |
2322 | }; |
2323 | |
2324 | /// Matches a bitwise 'not' as 'xor V, -1' or 'xor -1, V'. For vectors, the |
2325 | /// constant value must be composed of only -1 scalar elements. |
2326 | template <typename ValTy> |
2327 | inline NotForbidUndef_match<ValTy> m_NotForbidUndef(const ValTy &V) { |
2328 | return NotForbidUndef_match<ValTy>(V); |
2329 | } |
2330 | |
2331 | /// Matches an SMin with LHS and RHS in either order. |
2332 | template <typename LHS, typename RHS> |
2333 | inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true> |
2334 | m_c_SMin(const LHS &L, const RHS &R) { |
2335 | return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>(L, R); |
2336 | } |
2337 | /// Matches an SMax with LHS and RHS in either order. |
2338 | template <typename LHS, typename RHS> |
2339 | inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true> |
2340 | m_c_SMax(const LHS &L, const RHS &R) { |
2341 | return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>(L, R); |
2342 | } |
2343 | /// Matches a UMin with LHS and RHS in either order. |
2344 | template <typename LHS, typename RHS> |
2345 | inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true> |
2346 | m_c_UMin(const LHS &L, const RHS &R) { |
2347 | return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>(L, R); |
2348 | } |
2349 | /// Matches a UMax with LHS and RHS in either order. |
2350 | template <typename LHS, typename RHS> |
2351 | inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true> |
2352 | m_c_UMax(const LHS &L, const RHS &R) { |
2353 | return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>(L, R); |
2354 | } |
2355 | |
2356 | template <typename LHS, typename RHS> |
2357 | inline match_combine_or< |
2358 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>, |
2359 | MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>>, |
2360 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>, |
2361 | MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>>> |
2362 | m_c_MaxOrMin(const LHS &L, const RHS &R) { |
2363 | return m_CombineOr(m_CombineOr(m_c_SMax(L, R), m_c_SMin(L, R)), |
2364 | m_CombineOr(m_c_UMax(L, R), m_c_UMin(L, R))); |
2365 | } |
2366 | |
2367 | /// Matches FAdd with LHS and RHS in either order. |
2368 | template <typename LHS, typename RHS> |
2369 | inline BinaryOp_match<LHS, RHS, Instruction::FAdd, true> |
2370 | m_c_FAdd(const LHS &L, const RHS &R) { |
2371 | return BinaryOp_match<LHS, RHS, Instruction::FAdd, true>(L, R); |
2372 | } |
2373 | |
2374 | /// Matches FMul with LHS and RHS in either order. |
2375 | template <typename LHS, typename RHS> |
2376 | inline BinaryOp_match<LHS, RHS, Instruction::FMul, true> |
2377 | m_c_FMul(const LHS &L, const RHS &R) { |
2378 | return BinaryOp_match<LHS, RHS, Instruction::FMul, true>(L, R); |
2379 | } |
2380 | |
2381 | template <typename Opnd_t> struct Signum_match { |
2382 | Opnd_t Val; |
2383 | Signum_match(const Opnd_t &V) : Val(V) {} |
2384 | |
2385 | template <typename OpTy> bool match(OpTy *V) { |
2386 | unsigned TypeSize = V->getType()->getScalarSizeInBits(); |
2387 | if (TypeSize == 0) |
2388 | return false; |
2389 | |
2390 | unsigned ShiftWidth = TypeSize - 1; |
2391 | Value *OpL = nullptr, *OpR = nullptr; |
2392 | |
2393 | // This is the representation of signum we match: |
2394 | // |
2395 | // signum(x) == (x >> 63) | (-x >>u 63) |
2396 | // |
2397 | // An i1 value is its own signum, so it's correct to match |
2398 | // |
2399 | // signum(x) == (x >> 0) | (-x >>u 0) |
2400 | // |
2401 | // for i1 values. |
2402 | |
2403 | auto LHS = m_AShr(m_Value(OpL), m_SpecificInt(ShiftWidth)); |
2404 | auto RHS = m_LShr(m_Neg(m_Value(OpR)), m_SpecificInt(ShiftWidth)); |
2405 | auto Signum = m_Or(LHS, RHS); |
2406 | |
2407 | return Signum.match(V) && OpL == OpR && Val.match(OpL); |
2408 | } |
2409 | }; |
2410 | |
2411 | /// Matches a signum pattern. |
2412 | /// |
2413 | /// signum(x) = |
2414 | /// x > 0 -> 1 |
2415 | /// x == 0 -> 0 |
2416 | /// x < 0 -> -1 |
2417 | template <typename Val_t> inline Signum_match<Val_t> m_Signum(const Val_t &V) { |
2418 | return Signum_match<Val_t>(V); |
2419 | } |
2420 | |
2421 | template <int Ind, typename Opnd_t> struct ExtractValue_match { |
2422 | Opnd_t Val; |
2423 | ExtractValue_match(const Opnd_t &V) : Val(V) {} |
2424 | |
2425 | template <typename OpTy> bool match(OpTy *V) { |
2426 | if (auto *I = dyn_cast<ExtractValueInst>(V)) { |
2427 | // If Ind is -1, don't inspect indices |
2428 | if (Ind != -1 && |
2429 | !(I->getNumIndices() == 1 && I->getIndices()[0] == (unsigned)Ind)) |
2430 | return false; |
2431 | return Val.match(I->getAggregateOperand()); |
2432 | } |
2433 | return false; |
2434 | } |
2435 | }; |
2436 | |
2437 | /// Match a single index ExtractValue instruction. |
2438 | /// For example m_ExtractValue<1>(...) |
2439 | template <int Ind, typename Val_t> |
2440 | inline ExtractValue_match<Ind, Val_t> m_ExtractValue(const Val_t &V) { |
2441 | return ExtractValue_match<Ind, Val_t>(V); |
2442 | } |
2443 | |
2444 | /// Match an ExtractValue instruction with any index. |
2445 | /// For example m_ExtractValue(...) |
2446 | template <typename Val_t> |
2447 | inline ExtractValue_match<-1, Val_t> m_ExtractValue(const Val_t &V) { |
2448 | return ExtractValue_match<-1, Val_t>(V); |
2449 | } |
2450 | |
2451 | /// Matcher for a single index InsertValue instruction. |
2452 | template <int Ind, typename T0, typename T1> struct InsertValue_match { |
2453 | T0 Op0; |
2454 | T1 Op1; |
2455 | |
2456 | InsertValue_match(const T0 &Op0, const T1 &Op1) : Op0(Op0), Op1(Op1) {} |
2457 | |
2458 | template <typename OpTy> bool match(OpTy *V) { |
2459 | if (auto *I = dyn_cast<InsertValueInst>(V)) { |
2460 | return Op0.match(I->getOperand(0)) && Op1.match(I->getOperand(1)) && |
2461 | I->getNumIndices() == 1 && Ind == I->getIndices()[0]; |
2462 | } |
2463 | return false; |
2464 | } |
2465 | }; |
2466 | |
2467 | /// Matches a single index InsertValue instruction. |
2468 | template <int Ind, typename Val_t, typename Elt_t> |
2469 | inline InsertValue_match<Ind, Val_t, Elt_t> m_InsertValue(const Val_t &Val, |
2470 | const Elt_t &Elt) { |
2471 | return InsertValue_match<Ind, Val_t, Elt_t>(Val, Elt); |
2472 | } |
2473 | |
2474 | /// Matches patterns for `vscale`. This can either be a call to `llvm.vscale` or |
2475 | /// the constant expression |
2476 | /// `ptrtoint(gep <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1>` |
2477 | /// under the right conditions determined by DataLayout. |
2478 | struct VScaleVal_match { |
2479 | template <typename ITy> bool match(ITy *V) { |
2480 | if (m_Intrinsic<Intrinsic::vscale>().match(V)) |
2481 | return true; |
2482 | |
2483 | Value *Ptr; |
2484 | if (m_PtrToInt(m_Value(Ptr)).match(V)) { |
2485 | if (auto *GEP = dyn_cast<GEPOperator>(Ptr)) { |
2486 | auto *DerefTy = |
2487 | dyn_cast<ScalableVectorType>(GEP->getSourceElementType()); |
2488 | if (GEP->getNumIndices() == 1 && DerefTy && |
2489 | DerefTy->getElementType()->isIntegerTy(8) && |
2490 | m_Zero().match(GEP->getPointerOperand()) && |
2491 | m_SpecificInt(1).match(GEP->idx_begin()->get())) |
2492 | return true; |
2493 | } |
2494 | } |
2495 | |
2496 | return false; |
2497 | } |
2498 | }; |
2499 | |
2500 | inline VScaleVal_match m_VScale() { |
2501 | return VScaleVal_match(); |
2502 | } |
2503 | |
2504 | template <typename LHS, typename RHS, unsigned Opcode, bool Commutable = false> |
2505 | struct LogicalOp_match { |
2506 | LHS L; |
2507 | RHS R; |
2508 | |
2509 | LogicalOp_match(const LHS &L, const RHS &R) : L(L), R(R) {} |
2510 | |
2511 | template <typename T> bool match(T *V) { |
2512 | auto *I = dyn_cast<Instruction>(V); |
2513 | if (!I || !I->getType()->isIntOrIntVectorTy(1)) |
2514 | return false; |
2515 | |
2516 | if (I->getOpcode() == Opcode) { |
2517 | auto *Op0 = I->getOperand(0); |
2518 | auto *Op1 = I->getOperand(1); |
2519 | return (L.match(Op0) && R.match(Op1)) || |
2520 | (Commutable && L.match(Op1) && R.match(Op0)); |
2521 | } |
2522 | |
2523 | if (auto *Select = dyn_cast<SelectInst>(I)) { |
2524 | auto *Cond = Select->getCondition(); |
2525 | auto *TVal = Select->getTrueValue(); |
2526 | auto *FVal = Select->getFalseValue(); |
2527 | |
2528 | // Don't match a scalar select of bool vectors. |
2529 | // Transforms expect a single type for operands if this matches. |
2530 | if (Cond->getType() != Select->getType()) |
2531 | return false; |
2532 | |
2533 | if (Opcode == Instruction::And) { |
2534 | auto *C = dyn_cast<Constant>(FVal); |
2535 | if (C && C->isNullValue()) |
2536 | return (L.match(Cond) && R.match(TVal)) || |
2537 | (Commutable && L.match(TVal) && R.match(Cond)); |
2538 | } else { |
2539 | assert(Opcode == Instruction::Or)(static_cast <bool> (Opcode == Instruction::Or) ? void ( 0) : __assert_fail ("Opcode == Instruction::Or", "llvm/include/llvm/IR/PatternMatch.h" , 2539, __extension__ __PRETTY_FUNCTION__)); |
2540 | auto *C = dyn_cast<Constant>(TVal); |
2541 | if (C && C->isOneValue()) |
2542 | return (L.match(Cond) && R.match(FVal)) || |
2543 | (Commutable && L.match(FVal) && R.match(Cond)); |
2544 | } |
2545 | } |
2546 | |
2547 | return false; |
2548 | } |
2549 | }; |
2550 | |
2551 | /// Matches L && R either in the form of L & R or L ? R : false. |
2552 | /// Note that the latter form is poison-blocking. |
2553 | template <typename LHS, typename RHS> |
2554 | inline LogicalOp_match<LHS, RHS, Instruction::And> m_LogicalAnd(const LHS &L, |
2555 | const RHS &R) { |
2556 | return LogicalOp_match<LHS, RHS, Instruction::And>(L, R); |
2557 | } |
2558 | |
2559 | /// Matches L && R where L and R are arbitrary values. |
2560 | inline auto m_LogicalAnd() { return m_LogicalAnd(m_Value(), m_Value()); } |
2561 | |
2562 | /// Matches L && R with LHS and RHS in either order. |
2563 | template <typename LHS, typename RHS> |
2564 | inline LogicalOp_match<LHS, RHS, Instruction::And, true> |
2565 | m_c_LogicalAnd(const LHS &L, const RHS &R) { |
2566 | return LogicalOp_match<LHS, RHS, Instruction::And, true>(L, R); |
2567 | } |
2568 | |
2569 | /// Matches L || R either in the form of L | R or L ? true : R. |
2570 | /// Note that the latter form is poison-blocking. |
2571 | template <typename LHS, typename RHS> |
2572 | inline LogicalOp_match<LHS, RHS, Instruction::Or> m_LogicalOr(const LHS &L, |
2573 | const RHS &R) { |
2574 | return LogicalOp_match<LHS, RHS, Instruction::Or>(L, R); |
2575 | } |
2576 | |
2577 | /// Matches L || R where L and R are arbitrary values. |
2578 | inline auto m_LogicalOr() { return m_LogicalOr(m_Value(), m_Value()); } |
2579 | |
2580 | /// Matches L || R with LHS and RHS in either order. |
2581 | template <typename LHS, typename RHS> |
2582 | inline LogicalOp_match<LHS, RHS, Instruction::Or, true> |
2583 | m_c_LogicalOr(const LHS &L, const RHS &R) { |
2584 | return LogicalOp_match<LHS, RHS, Instruction::Or, true>(L, R); |
2585 | } |
2586 | |
2587 | /// Matches either L && R or L || R, |
2588 | /// either one being in the either binary or logical form. |
2589 | /// Note that the latter form is poison-blocking. |
2590 | template <typename LHS, typename RHS, bool Commutable = false> |
2591 | inline auto m_LogicalOp(const LHS &L, const RHS &R) { |
2592 | return m_CombineOr( |
2593 | LogicalOp_match<LHS, RHS, Instruction::And, Commutable>(L, R), |
2594 | LogicalOp_match<LHS, RHS, Instruction::Or, Commutable>(L, R)); |
2595 | } |
2596 | |
2597 | /// Matches either L && R or L || R where L and R are arbitrary values. |
2598 | inline auto m_LogicalOp() { return m_LogicalOp(m_Value(), m_Value()); } |
2599 | |
2600 | /// Matches either L && R or L || R with LHS and RHS in either order. |
2601 | template <typename LHS, typename RHS> |
2602 | inline auto m_c_LogicalOp(const LHS &L, const RHS &R) { |
2603 | return m_LogicalOp<LHS, RHS, /*Commutable=*/true>(L, R); |
2604 | } |
2605 | |
2606 | } // end namespace PatternMatch |
2607 | } // end namespace llvm |
2608 | |
2609 | #endif // LLVM_IR_PATTERNMATCH_H |