File: | llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp |
Warning: | line 3650, column 7 Called C++ object pointer is uninitialized |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- InstCombineCompares.cpp --------------------------------------------===// | ||||||||
2 | // | ||||||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||||
6 | // | ||||||||
7 | //===----------------------------------------------------------------------===// | ||||||||
8 | // | ||||||||
9 | // This file implements the visitICmp and visitFCmp functions. | ||||||||
10 | // | ||||||||
11 | //===----------------------------------------------------------------------===// | ||||||||
12 | |||||||||
13 | #include "InstCombineInternal.h" | ||||||||
14 | #include "llvm/ADT/APSInt.h" | ||||||||
15 | #include "llvm/ADT/SetVector.h" | ||||||||
16 | #include "llvm/ADT/Statistic.h" | ||||||||
17 | #include "llvm/Analysis/ConstantFolding.h" | ||||||||
18 | #include "llvm/Analysis/InstructionSimplify.h" | ||||||||
19 | #include "llvm/Analysis/TargetLibraryInfo.h" | ||||||||
20 | #include "llvm/IR/ConstantRange.h" | ||||||||
21 | #include "llvm/IR/DataLayout.h" | ||||||||
22 | #include "llvm/IR/GetElementPtrTypeIterator.h" | ||||||||
23 | #include "llvm/IR/IntrinsicInst.h" | ||||||||
24 | #include "llvm/IR/PatternMatch.h" | ||||||||
25 | #include "llvm/Support/Debug.h" | ||||||||
26 | #include "llvm/Support/KnownBits.h" | ||||||||
27 | #include "llvm/Transforms/InstCombine/InstCombiner.h" | ||||||||
28 | |||||||||
29 | using namespace llvm; | ||||||||
30 | using namespace PatternMatch; | ||||||||
31 | |||||||||
32 | #define DEBUG_TYPE"instcombine" "instcombine" | ||||||||
33 | |||||||||
34 | // How many times is a select replaced by one of its operands? | ||||||||
35 | STATISTIC(NumSel, "Number of select opts")static llvm::Statistic NumSel = {"instcombine", "NumSel", "Number of select opts" }; | ||||||||
36 | |||||||||
37 | |||||||||
38 | /// Compute Result = In1+In2, returning true if the result overflowed for this | ||||||||
39 | /// type. | ||||||||
40 | static bool addWithOverflow(APInt &Result, const APInt &In1, | ||||||||
41 | const APInt &In2, bool IsSigned = false) { | ||||||||
42 | bool Overflow; | ||||||||
43 | if (IsSigned) | ||||||||
44 | Result = In1.sadd_ov(In2, Overflow); | ||||||||
45 | else | ||||||||
46 | Result = In1.uadd_ov(In2, Overflow); | ||||||||
47 | |||||||||
48 | return Overflow; | ||||||||
49 | } | ||||||||
50 | |||||||||
51 | /// Compute Result = In1-In2, returning true if the result overflowed for this | ||||||||
52 | /// type. | ||||||||
53 | static bool subWithOverflow(APInt &Result, const APInt &In1, | ||||||||
54 | const APInt &In2, bool IsSigned = false) { | ||||||||
55 | bool Overflow; | ||||||||
56 | if (IsSigned) | ||||||||
57 | Result = In1.ssub_ov(In2, Overflow); | ||||||||
58 | else | ||||||||
59 | Result = In1.usub_ov(In2, Overflow); | ||||||||
60 | |||||||||
61 | return Overflow; | ||||||||
62 | } | ||||||||
63 | |||||||||
64 | /// Given an icmp instruction, return true if any use of this comparison is a | ||||||||
65 | /// branch on sign bit comparison. | ||||||||
66 | static bool hasBranchUse(ICmpInst &I) { | ||||||||
67 | for (auto *U : I.users()) | ||||||||
68 | if (isa<BranchInst>(U)) | ||||||||
69 | return true; | ||||||||
70 | return false; | ||||||||
71 | } | ||||||||
72 | |||||||||
73 | /// Returns true if the exploded icmp can be expressed as a signed comparison | ||||||||
74 | /// to zero and updates the predicate accordingly. | ||||||||
75 | /// The signedness of the comparison is preserved. | ||||||||
76 | /// TODO: Refactor with decomposeBitTestICmp()? | ||||||||
77 | static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) { | ||||||||
78 | if (!ICmpInst::isSigned(Pred)) | ||||||||
79 | return false; | ||||||||
80 | |||||||||
81 | if (C.isNullValue()) | ||||||||
82 | return ICmpInst::isRelational(Pred); | ||||||||
83 | |||||||||
84 | if (C.isOneValue()) { | ||||||||
85 | if (Pred == ICmpInst::ICMP_SLT) { | ||||||||
86 | Pred = ICmpInst::ICMP_SLE; | ||||||||
87 | return true; | ||||||||
88 | } | ||||||||
89 | } else if (C.isAllOnesValue()) { | ||||||||
90 | if (Pred == ICmpInst::ICMP_SGT) { | ||||||||
91 | Pred = ICmpInst::ICMP_SGE; | ||||||||
92 | return true; | ||||||||
93 | } | ||||||||
94 | } | ||||||||
95 | |||||||||
96 | return false; | ||||||||
97 | } | ||||||||
98 | |||||||||
99 | /// This is called when we see this pattern: | ||||||||
100 | /// cmp pred (load (gep GV, ...)), cmpcst | ||||||||
101 | /// where GV is a global variable with a constant initializer. Try to simplify | ||||||||
102 | /// this into some simple computation that does not need the load. For example | ||||||||
103 | /// we can optimize "icmp eq (load (gep "foo", 0, i)), 0" into "icmp eq i, 3". | ||||||||
104 | /// | ||||||||
105 | /// If AndCst is non-null, then the loaded value is masked with that constant | ||||||||
106 | /// before doing the comparison. This handles cases like "A[i]&4 == 0". | ||||||||
107 | Instruction * | ||||||||
108 | InstCombinerImpl::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, | ||||||||
109 | GlobalVariable *GV, CmpInst &ICI, | ||||||||
110 | ConstantInt *AndCst) { | ||||||||
111 | Constant *Init = GV->getInitializer(); | ||||||||
112 | if (!isa<ConstantArray>(Init) && !isa<ConstantDataArray>(Init)) | ||||||||
113 | return nullptr; | ||||||||
114 | |||||||||
115 | uint64_t ArrayElementCount = Init->getType()->getArrayNumElements(); | ||||||||
116 | // Don't blow up on huge arrays. | ||||||||
117 | if (ArrayElementCount > MaxArraySizeForCombine) | ||||||||
118 | return nullptr; | ||||||||
119 | |||||||||
120 | // There are many forms of this optimization we can handle, for now, just do | ||||||||
121 | // the simple index into a single-dimensional array. | ||||||||
122 | // | ||||||||
123 | // Require: GEP GV, 0, i {{, constant indices}} | ||||||||
124 | if (GEP->getNumOperands() < 3 || | ||||||||
125 | !isa<ConstantInt>(GEP->getOperand(1)) || | ||||||||
126 | !cast<ConstantInt>(GEP->getOperand(1))->isZero() || | ||||||||
127 | isa<Constant>(GEP->getOperand(2))) | ||||||||
128 | return nullptr; | ||||||||
129 | |||||||||
130 | // Check that indices after the variable are constants and in-range for the | ||||||||
131 | // type they index. Collect the indices. This is typically for arrays of | ||||||||
132 | // structs. | ||||||||
133 | SmallVector<unsigned, 4> LaterIndices; | ||||||||
134 | |||||||||
135 | Type *EltTy = Init->getType()->getArrayElementType(); | ||||||||
136 | for (unsigned i = 3, e = GEP->getNumOperands(); i != e; ++i) { | ||||||||
137 | ConstantInt *Idx = dyn_cast<ConstantInt>(GEP->getOperand(i)); | ||||||||
138 | if (!Idx) return nullptr; // Variable index. | ||||||||
139 | |||||||||
140 | uint64_t IdxVal = Idx->getZExtValue(); | ||||||||
141 | if ((unsigned)IdxVal != IdxVal) return nullptr; // Too large array index. | ||||||||
142 | |||||||||
143 | if (StructType *STy = dyn_cast<StructType>(EltTy)) | ||||||||
144 | EltTy = STy->getElementType(IdxVal); | ||||||||
145 | else if (ArrayType *ATy = dyn_cast<ArrayType>(EltTy)) { | ||||||||
146 | if (IdxVal >= ATy->getNumElements()) return nullptr; | ||||||||
147 | EltTy = ATy->getElementType(); | ||||||||
148 | } else { | ||||||||
149 | return nullptr; // Unknown type. | ||||||||
150 | } | ||||||||
151 | |||||||||
152 | LaterIndices.push_back(IdxVal); | ||||||||
153 | } | ||||||||
154 | |||||||||
155 | enum { Overdefined = -3, Undefined = -2 }; | ||||||||
156 | |||||||||
157 | // Variables for our state machines. | ||||||||
158 | |||||||||
159 | // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form | ||||||||
160 | // "i == 47 | i == 87", where 47 is the first index the condition is true for, | ||||||||
161 | // and 87 is the second (and last) index. FirstTrueElement is -2 when | ||||||||
162 | // undefined, otherwise set to the first true element. SecondTrueElement is | ||||||||
163 | // -2 when undefined, -3 when overdefined and >= 0 when that index is true. | ||||||||
164 | int FirstTrueElement = Undefined, SecondTrueElement = Undefined; | ||||||||
165 | |||||||||
166 | // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the | ||||||||
167 | // form "i != 47 & i != 87". Same state transitions as for true elements. | ||||||||
168 | int FirstFalseElement = Undefined, SecondFalseElement = Undefined; | ||||||||
169 | |||||||||
170 | /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these | ||||||||
171 | /// define a state machine that triggers for ranges of values that the index | ||||||||
172 | /// is true or false for. This triggers on things like "abbbbc"[i] == 'b'. | ||||||||
173 | /// This is -2 when undefined, -3 when overdefined, and otherwise the last | ||||||||
174 | /// index in the range (inclusive). We use -2 for undefined here because we | ||||||||
175 | /// use relative comparisons and don't want 0-1 to match -1. | ||||||||
176 | int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined; | ||||||||
177 | |||||||||
178 | // MagicBitvector - This is a magic bitvector where we set a bit if the | ||||||||
179 | // comparison is true for element 'i'. If there are 64 elements or less in | ||||||||
180 | // the array, this will fully represent all the comparison results. | ||||||||
181 | uint64_t MagicBitvector = 0; | ||||||||
182 | |||||||||
183 | // Scan the array and see if one of our patterns matches. | ||||||||
184 | Constant *CompareRHS = cast<Constant>(ICI.getOperand(1)); | ||||||||
185 | for (unsigned i = 0, e = ArrayElementCount; i != e; ++i) { | ||||||||
186 | Constant *Elt = Init->getAggregateElement(i); | ||||||||
187 | if (!Elt) return nullptr; | ||||||||
188 | |||||||||
189 | // If this is indexing an array of structures, get the structure element. | ||||||||
190 | if (!LaterIndices.empty()) | ||||||||
191 | Elt = ConstantExpr::getExtractValue(Elt, LaterIndices); | ||||||||
192 | |||||||||
193 | // If the element is masked, handle it. | ||||||||
194 | if (AndCst) Elt = ConstantExpr::getAnd(Elt, AndCst); | ||||||||
195 | |||||||||
196 | // Find out if the comparison would be true or false for the i'th element. | ||||||||
197 | Constant *C = ConstantFoldCompareInstOperands(ICI.getPredicate(), Elt, | ||||||||
198 | CompareRHS, DL, &TLI); | ||||||||
199 | // If the result is undef for this element, ignore it. | ||||||||
200 | if (isa<UndefValue>(C)) { | ||||||||
201 | // Extend range state machines to cover this element in case there is an | ||||||||
202 | // undef in the middle of the range. | ||||||||
203 | if (TrueRangeEnd == (int)i-1) | ||||||||
204 | TrueRangeEnd = i; | ||||||||
205 | if (FalseRangeEnd == (int)i-1) | ||||||||
206 | FalseRangeEnd = i; | ||||||||
207 | continue; | ||||||||
208 | } | ||||||||
209 | |||||||||
210 | // If we can't compute the result for any of the elements, we have to give | ||||||||
211 | // up evaluating the entire conditional. | ||||||||
212 | if (!isa<ConstantInt>(C)) return nullptr; | ||||||||
213 | |||||||||
214 | // Otherwise, we know if the comparison is true or false for this element, | ||||||||
215 | // update our state machines. | ||||||||
216 | bool IsTrueForElt = !cast<ConstantInt>(C)->isZero(); | ||||||||
217 | |||||||||
218 | // State machine for single/double/range index comparison. | ||||||||
219 | if (IsTrueForElt) { | ||||||||
220 | // Update the TrueElement state machine. | ||||||||
221 | if (FirstTrueElement == Undefined) | ||||||||
222 | FirstTrueElement = TrueRangeEnd = i; // First true element. | ||||||||
223 | else { | ||||||||
224 | // Update double-compare state machine. | ||||||||
225 | if (SecondTrueElement == Undefined) | ||||||||
226 | SecondTrueElement = i; | ||||||||
227 | else | ||||||||
228 | SecondTrueElement = Overdefined; | ||||||||
229 | |||||||||
230 | // Update range state machine. | ||||||||
231 | if (TrueRangeEnd == (int)i-1) | ||||||||
232 | TrueRangeEnd = i; | ||||||||
233 | else | ||||||||
234 | TrueRangeEnd = Overdefined; | ||||||||
235 | } | ||||||||
236 | } else { | ||||||||
237 | // Update the FalseElement state machine. | ||||||||
238 | if (FirstFalseElement == Undefined) | ||||||||
239 | FirstFalseElement = FalseRangeEnd = i; // First false element. | ||||||||
240 | else { | ||||||||
241 | // Update double-compare state machine. | ||||||||
242 | if (SecondFalseElement == Undefined) | ||||||||
243 | SecondFalseElement = i; | ||||||||
244 | else | ||||||||
245 | SecondFalseElement = Overdefined; | ||||||||
246 | |||||||||
247 | // Update range state machine. | ||||||||
248 | if (FalseRangeEnd == (int)i-1) | ||||||||
249 | FalseRangeEnd = i; | ||||||||
250 | else | ||||||||
251 | FalseRangeEnd = Overdefined; | ||||||||
252 | } | ||||||||
253 | } | ||||||||
254 | |||||||||
255 | // If this element is in range, update our magic bitvector. | ||||||||
256 | if (i < 64 && IsTrueForElt) | ||||||||
257 | MagicBitvector |= 1ULL << i; | ||||||||
258 | |||||||||
259 | // If all of our states become overdefined, bail out early. Since the | ||||||||
260 | // predicate is expensive, only check it every 8 elements. This is only | ||||||||
261 | // really useful for really huge arrays. | ||||||||
262 | if ((i & 8) == 0 && i >= 64 && SecondTrueElement == Overdefined && | ||||||||
263 | SecondFalseElement == Overdefined && TrueRangeEnd == Overdefined && | ||||||||
264 | FalseRangeEnd == Overdefined) | ||||||||
265 | return nullptr; | ||||||||
266 | } | ||||||||
267 | |||||||||
268 | // Now that we've scanned the entire array, emit our new comparison(s). We | ||||||||
269 | // order the state machines in complexity of the generated code. | ||||||||
270 | Value *Idx = GEP->getOperand(2); | ||||||||
271 | |||||||||
272 | // If the index is larger than the pointer size of the target, truncate the | ||||||||
273 | // index down like the GEP would do implicitly. We don't have to do this for | ||||||||
274 | // an inbounds GEP because the index can't be out of range. | ||||||||
275 | if (!GEP->isInBounds()) { | ||||||||
276 | Type *IntPtrTy = DL.getIntPtrType(GEP->getType()); | ||||||||
277 | unsigned PtrSize = IntPtrTy->getIntegerBitWidth(); | ||||||||
278 | if (Idx->getType()->getPrimitiveSizeInBits().getFixedSize() > PtrSize) | ||||||||
279 | Idx = Builder.CreateTrunc(Idx, IntPtrTy); | ||||||||
280 | } | ||||||||
281 | |||||||||
282 | // If inbounds keyword is not present, Idx * ElementSize can overflow. | ||||||||
283 | // Let's assume that ElementSize is 2 and the wanted value is at offset 0. | ||||||||
284 | // Then, there are two possible values for Idx to match offset 0: | ||||||||
285 | // 0x00..00, 0x80..00. | ||||||||
286 | // Emitting 'icmp eq Idx, 0' isn't correct in this case because the | ||||||||
287 | // comparison is false if Idx was 0x80..00. | ||||||||
288 | // We need to erase the highest countTrailingZeros(ElementSize) bits of Idx. | ||||||||
289 | unsigned ElementSize = | ||||||||
290 | DL.getTypeAllocSize(Init->getType()->getArrayElementType()); | ||||||||
291 | auto MaskIdx = [&](Value* Idx){ | ||||||||
292 | if (!GEP->isInBounds() && countTrailingZeros(ElementSize) != 0) { | ||||||||
293 | Value *Mask = ConstantInt::get(Idx->getType(), -1); | ||||||||
294 | Mask = Builder.CreateLShr(Mask, countTrailingZeros(ElementSize)); | ||||||||
295 | Idx = Builder.CreateAnd(Idx, Mask); | ||||||||
296 | } | ||||||||
297 | return Idx; | ||||||||
298 | }; | ||||||||
299 | |||||||||
300 | // If the comparison is only true for one or two elements, emit direct | ||||||||
301 | // comparisons. | ||||||||
302 | if (SecondTrueElement != Overdefined) { | ||||||||
303 | Idx = MaskIdx(Idx); | ||||||||
304 | // None true -> false. | ||||||||
305 | if (FirstTrueElement == Undefined) | ||||||||
306 | return replaceInstUsesWith(ICI, Builder.getFalse()); | ||||||||
307 | |||||||||
308 | Value *FirstTrueIdx = ConstantInt::get(Idx->getType(), FirstTrueElement); | ||||||||
309 | |||||||||
310 | // True for one element -> 'i == 47'. | ||||||||
311 | if (SecondTrueElement == Undefined) | ||||||||
312 | return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx); | ||||||||
313 | |||||||||
314 | // True for two elements -> 'i == 47 | i == 72'. | ||||||||
315 | Value *C1 = Builder.CreateICmpEQ(Idx, FirstTrueIdx); | ||||||||
316 | Value *SecondTrueIdx = ConstantInt::get(Idx->getType(), SecondTrueElement); | ||||||||
317 | Value *C2 = Builder.CreateICmpEQ(Idx, SecondTrueIdx); | ||||||||
318 | return BinaryOperator::CreateOr(C1, C2); | ||||||||
319 | } | ||||||||
320 | |||||||||
321 | // If the comparison is only false for one or two elements, emit direct | ||||||||
322 | // comparisons. | ||||||||
323 | if (SecondFalseElement != Overdefined) { | ||||||||
324 | Idx = MaskIdx(Idx); | ||||||||
325 | // None false -> true. | ||||||||
326 | if (FirstFalseElement == Undefined) | ||||||||
327 | return replaceInstUsesWith(ICI, Builder.getTrue()); | ||||||||
328 | |||||||||
329 | Value *FirstFalseIdx = ConstantInt::get(Idx->getType(), FirstFalseElement); | ||||||||
330 | |||||||||
331 | // False for one element -> 'i != 47'. | ||||||||
332 | if (SecondFalseElement == Undefined) | ||||||||
333 | return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx); | ||||||||
334 | |||||||||
335 | // False for two elements -> 'i != 47 & i != 72'. | ||||||||
336 | Value *C1 = Builder.CreateICmpNE(Idx, FirstFalseIdx); | ||||||||
337 | Value *SecondFalseIdx = ConstantInt::get(Idx->getType(),SecondFalseElement); | ||||||||
338 | Value *C2 = Builder.CreateICmpNE(Idx, SecondFalseIdx); | ||||||||
339 | return BinaryOperator::CreateAnd(C1, C2); | ||||||||
340 | } | ||||||||
341 | |||||||||
342 | // If the comparison can be replaced with a range comparison for the elements | ||||||||
343 | // where it is true, emit the range check. | ||||||||
344 | if (TrueRangeEnd != Overdefined) { | ||||||||
345 | assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare")(static_cast <bool> (TrueRangeEnd != FirstTrueElement && "Should emit single compare") ? void (0) : __assert_fail ("TrueRangeEnd != FirstTrueElement && \"Should emit single compare\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 345, __extension__ __PRETTY_FUNCTION__)); | ||||||||
346 | Idx = MaskIdx(Idx); | ||||||||
347 | |||||||||
348 | // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1). | ||||||||
349 | if (FirstTrueElement) { | ||||||||
350 | Value *Offs = ConstantInt::get(Idx->getType(), -FirstTrueElement); | ||||||||
351 | Idx = Builder.CreateAdd(Idx, Offs); | ||||||||
352 | } | ||||||||
353 | |||||||||
354 | Value *End = ConstantInt::get(Idx->getType(), | ||||||||
355 | TrueRangeEnd-FirstTrueElement+1); | ||||||||
356 | return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End); | ||||||||
357 | } | ||||||||
358 | |||||||||
359 | // False range check. | ||||||||
360 | if (FalseRangeEnd != Overdefined) { | ||||||||
361 | assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare")(static_cast <bool> (FalseRangeEnd != FirstFalseElement && "Should emit single compare") ? void (0) : __assert_fail ("FalseRangeEnd != FirstFalseElement && \"Should emit single compare\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 361, __extension__ __PRETTY_FUNCTION__)); | ||||||||
362 | Idx = MaskIdx(Idx); | ||||||||
363 | // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse). | ||||||||
364 | if (FirstFalseElement) { | ||||||||
365 | Value *Offs = ConstantInt::get(Idx->getType(), -FirstFalseElement); | ||||||||
366 | Idx = Builder.CreateAdd(Idx, Offs); | ||||||||
367 | } | ||||||||
368 | |||||||||
369 | Value *End = ConstantInt::get(Idx->getType(), | ||||||||
370 | FalseRangeEnd-FirstFalseElement); | ||||||||
371 | return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End); | ||||||||
372 | } | ||||||||
373 | |||||||||
374 | // If a magic bitvector captures the entire comparison state | ||||||||
375 | // of this load, replace it with computation that does: | ||||||||
376 | // ((magic_cst >> i) & 1) != 0 | ||||||||
377 | { | ||||||||
378 | Type *Ty = nullptr; | ||||||||
379 | |||||||||
380 | // Look for an appropriate type: | ||||||||
381 | // - The type of Idx if the magic fits | ||||||||
382 | // - The smallest fitting legal type | ||||||||
383 | if (ArrayElementCount <= Idx->getType()->getIntegerBitWidth()) | ||||||||
384 | Ty = Idx->getType(); | ||||||||
385 | else | ||||||||
386 | Ty = DL.getSmallestLegalIntType(Init->getContext(), ArrayElementCount); | ||||||||
387 | |||||||||
388 | if (Ty) { | ||||||||
389 | Idx = MaskIdx(Idx); | ||||||||
390 | Value *V = Builder.CreateIntCast(Idx, Ty, false); | ||||||||
391 | V = Builder.CreateLShr(ConstantInt::get(Ty, MagicBitvector), V); | ||||||||
392 | V = Builder.CreateAnd(ConstantInt::get(Ty, 1), V); | ||||||||
393 | return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, 0)); | ||||||||
394 | } | ||||||||
395 | } | ||||||||
396 | |||||||||
397 | return nullptr; | ||||||||
398 | } | ||||||||
399 | |||||||||
400 | /// Return a value that can be used to compare the *offset* implied by a GEP to | ||||||||
401 | /// zero. For example, if we have &A[i], we want to return 'i' for | ||||||||
402 | /// "icmp ne i, 0". Note that, in general, indices can be complex, and scales | ||||||||
403 | /// are involved. The above expression would also be legal to codegen as | ||||||||
404 | /// "icmp ne (i*4), 0" (assuming A is a pointer to i32). | ||||||||
405 | /// This latter form is less amenable to optimization though, and we are allowed | ||||||||
406 | /// to generate the first by knowing that pointer arithmetic doesn't overflow. | ||||||||
407 | /// | ||||||||
408 | /// If we can't emit an optimized form for this expression, this returns null. | ||||||||
409 | /// | ||||||||
410 | static Value *evaluateGEPOffsetExpression(User *GEP, InstCombinerImpl &IC, | ||||||||
411 | const DataLayout &DL) { | ||||||||
412 | gep_type_iterator GTI = gep_type_begin(GEP); | ||||||||
413 | |||||||||
414 | // Check to see if this gep only has a single variable index. If so, and if | ||||||||
415 | // any constant indices are a multiple of its scale, then we can compute this | ||||||||
416 | // in terms of the scale of the variable index. For example, if the GEP | ||||||||
417 | // implies an offset of "12 + i*4", then we can codegen this as "3 + i", | ||||||||
418 | // because the expression will cross zero at the same point. | ||||||||
419 | unsigned i, e = GEP->getNumOperands(); | ||||||||
420 | int64_t Offset = 0; | ||||||||
421 | for (i = 1; i != e; ++i, ++GTI) { | ||||||||
422 | if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) { | ||||||||
423 | // Compute the aggregate offset of constant indices. | ||||||||
424 | if (CI->isZero()) continue; | ||||||||
425 | |||||||||
426 | // Handle a struct index, which adds its field offset to the pointer. | ||||||||
427 | if (StructType *STy = GTI.getStructTypeOrNull()) { | ||||||||
428 | Offset += DL.getStructLayout(STy)->getElementOffset(CI->getZExtValue()); | ||||||||
429 | } else { | ||||||||
430 | uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType()); | ||||||||
431 | Offset += Size*CI->getSExtValue(); | ||||||||
432 | } | ||||||||
433 | } else { | ||||||||
434 | // Found our variable index. | ||||||||
435 | break; | ||||||||
436 | } | ||||||||
437 | } | ||||||||
438 | |||||||||
439 | // If there are no variable indices, we must have a constant offset, just | ||||||||
440 | // evaluate it the general way. | ||||||||
441 | if (i == e) return nullptr; | ||||||||
442 | |||||||||
443 | Value *VariableIdx = GEP->getOperand(i); | ||||||||
444 | // Determine the scale factor of the variable element. For example, this is | ||||||||
445 | // 4 if the variable index is into an array of i32. | ||||||||
446 | uint64_t VariableScale = DL.getTypeAllocSize(GTI.getIndexedType()); | ||||||||
447 | |||||||||
448 | // Verify that there are no other variable indices. If so, emit the hard way. | ||||||||
449 | for (++i, ++GTI; i != e; ++i, ++GTI) { | ||||||||
450 | ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i)); | ||||||||
451 | if (!CI) return nullptr; | ||||||||
452 | |||||||||
453 | // Compute the aggregate offset of constant indices. | ||||||||
454 | if (CI->isZero()) continue; | ||||||||
455 | |||||||||
456 | // Handle a struct index, which adds its field offset to the pointer. | ||||||||
457 | if (StructType *STy = GTI.getStructTypeOrNull()) { | ||||||||
458 | Offset += DL.getStructLayout(STy)->getElementOffset(CI->getZExtValue()); | ||||||||
459 | } else { | ||||||||
460 | uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType()); | ||||||||
461 | Offset += Size*CI->getSExtValue(); | ||||||||
462 | } | ||||||||
463 | } | ||||||||
464 | |||||||||
465 | // Okay, we know we have a single variable index, which must be a | ||||||||
466 | // pointer/array/vector index. If there is no offset, life is simple, return | ||||||||
467 | // the index. | ||||||||
468 | Type *IntPtrTy = DL.getIntPtrType(GEP->getOperand(0)->getType()); | ||||||||
469 | unsigned IntPtrWidth = IntPtrTy->getIntegerBitWidth(); | ||||||||
470 | if (Offset == 0) { | ||||||||
471 | // Cast to intptrty in case a truncation occurs. If an extension is needed, | ||||||||
472 | // we don't need to bother extending: the extension won't affect where the | ||||||||
473 | // computation crosses zero. | ||||||||
474 | if (VariableIdx->getType()->getPrimitiveSizeInBits().getFixedSize() > | ||||||||
475 | IntPtrWidth) { | ||||||||
476 | VariableIdx = IC.Builder.CreateTrunc(VariableIdx, IntPtrTy); | ||||||||
477 | } | ||||||||
478 | return VariableIdx; | ||||||||
479 | } | ||||||||
480 | |||||||||
481 | // Otherwise, there is an index. The computation we will do will be modulo | ||||||||
482 | // the pointer size. | ||||||||
483 | Offset = SignExtend64(Offset, IntPtrWidth); | ||||||||
484 | VariableScale = SignExtend64(VariableScale, IntPtrWidth); | ||||||||
485 | |||||||||
486 | // To do this transformation, any constant index must be a multiple of the | ||||||||
487 | // variable scale factor. For example, we can evaluate "12 + 4*i" as "3 + i", | ||||||||
488 | // but we can't evaluate "10 + 3*i" in terms of i. Check that the offset is a | ||||||||
489 | // multiple of the variable scale. | ||||||||
490 | int64_t NewOffs = Offset / (int64_t)VariableScale; | ||||||||
491 | if (Offset != NewOffs*(int64_t)VariableScale) | ||||||||
492 | return nullptr; | ||||||||
493 | |||||||||
494 | // Okay, we can do this evaluation. Start by converting the index to intptr. | ||||||||
495 | if (VariableIdx->getType() != IntPtrTy) | ||||||||
496 | VariableIdx = IC.Builder.CreateIntCast(VariableIdx, IntPtrTy, | ||||||||
497 | true /*Signed*/); | ||||||||
498 | Constant *OffsetVal = ConstantInt::get(IntPtrTy, NewOffs); | ||||||||
499 | return IC.Builder.CreateAdd(VariableIdx, OffsetVal, "offset"); | ||||||||
500 | } | ||||||||
501 | |||||||||
502 | /// Returns true if we can rewrite Start as a GEP with pointer Base | ||||||||
503 | /// and some integer offset. The nodes that need to be re-written | ||||||||
504 | /// for this transformation will be added to Explored. | ||||||||
505 | static bool canRewriteGEPAsOffset(Value *Start, Value *Base, | ||||||||
506 | const DataLayout &DL, | ||||||||
507 | SetVector<Value *> &Explored) { | ||||||||
508 | SmallVector<Value *, 16> WorkList(1, Start); | ||||||||
509 | Explored.insert(Base); | ||||||||
510 | |||||||||
511 | // The following traversal gives us an order which can be used | ||||||||
512 | // when doing the final transformation. Since in the final | ||||||||
513 | // transformation we create the PHI replacement instructions first, | ||||||||
514 | // we don't have to get them in any particular order. | ||||||||
515 | // | ||||||||
516 | // However, for other instructions we will have to traverse the | ||||||||
517 | // operands of an instruction first, which means that we have to | ||||||||
518 | // do a post-order traversal. | ||||||||
519 | while (!WorkList.empty()) { | ||||||||
520 | SetVector<PHINode *> PHIs; | ||||||||
521 | |||||||||
522 | while (!WorkList.empty()) { | ||||||||
523 | if (Explored.size() >= 100) | ||||||||
524 | return false; | ||||||||
525 | |||||||||
526 | Value *V = WorkList.back(); | ||||||||
527 | |||||||||
528 | if (Explored.contains(V)) { | ||||||||
529 | WorkList.pop_back(); | ||||||||
530 | continue; | ||||||||
531 | } | ||||||||
532 | |||||||||
533 | if (!isa<IntToPtrInst>(V) && !isa<PtrToIntInst>(V) && | ||||||||
534 | !isa<GetElementPtrInst>(V) && !isa<PHINode>(V)) | ||||||||
535 | // We've found some value that we can't explore which is different from | ||||||||
536 | // the base. Therefore we can't do this transformation. | ||||||||
537 | return false; | ||||||||
538 | |||||||||
539 | if (isa<IntToPtrInst>(V) || isa<PtrToIntInst>(V)) { | ||||||||
540 | auto *CI = cast<CastInst>(V); | ||||||||
541 | if (!CI->isNoopCast(DL)) | ||||||||
542 | return false; | ||||||||
543 | |||||||||
544 | if (Explored.count(CI->getOperand(0)) == 0) | ||||||||
545 | WorkList.push_back(CI->getOperand(0)); | ||||||||
546 | } | ||||||||
547 | |||||||||
548 | if (auto *GEP = dyn_cast<GEPOperator>(V)) { | ||||||||
549 | // We're limiting the GEP to having one index. This will preserve | ||||||||
550 | // the original pointer type. We could handle more cases in the | ||||||||
551 | // future. | ||||||||
552 | if (GEP->getNumIndices() != 1 || !GEP->isInBounds() || | ||||||||
553 | GEP->getType() != Start->getType()) | ||||||||
554 | return false; | ||||||||
555 | |||||||||
556 | if (Explored.count(GEP->getOperand(0)) == 0) | ||||||||
557 | WorkList.push_back(GEP->getOperand(0)); | ||||||||
558 | } | ||||||||
559 | |||||||||
560 | if (WorkList.back() == V) { | ||||||||
561 | WorkList.pop_back(); | ||||||||
562 | // We've finished visiting this node, mark it as such. | ||||||||
563 | Explored.insert(V); | ||||||||
564 | } | ||||||||
565 | |||||||||
566 | if (auto *PN = dyn_cast<PHINode>(V)) { | ||||||||
567 | // We cannot transform PHIs on unsplittable basic blocks. | ||||||||
568 | if (isa<CatchSwitchInst>(PN->getParent()->getTerminator())) | ||||||||
569 | return false; | ||||||||
570 | Explored.insert(PN); | ||||||||
571 | PHIs.insert(PN); | ||||||||
572 | } | ||||||||
573 | } | ||||||||
574 | |||||||||
575 | // Explore the PHI nodes further. | ||||||||
576 | for (auto *PN : PHIs) | ||||||||
577 | for (Value *Op : PN->incoming_values()) | ||||||||
578 | if (Explored.count(Op) == 0) | ||||||||
579 | WorkList.push_back(Op); | ||||||||
580 | } | ||||||||
581 | |||||||||
582 | // Make sure that we can do this. Since we can't insert GEPs in a basic | ||||||||
583 | // block before a PHI node, we can't easily do this transformation if | ||||||||
584 | // we have PHI node users of transformed instructions. | ||||||||
585 | for (Value *Val : Explored) { | ||||||||
586 | for (Value *Use : Val->uses()) { | ||||||||
587 | |||||||||
588 | auto *PHI = dyn_cast<PHINode>(Use); | ||||||||
589 | auto *Inst = dyn_cast<Instruction>(Val); | ||||||||
590 | |||||||||
591 | if (Inst == Base || Inst == PHI || !Inst || !PHI || | ||||||||
592 | Explored.count(PHI) == 0) | ||||||||
593 | continue; | ||||||||
594 | |||||||||
595 | if (PHI->getParent() == Inst->getParent()) | ||||||||
596 | return false; | ||||||||
597 | } | ||||||||
598 | } | ||||||||
599 | return true; | ||||||||
600 | } | ||||||||
601 | |||||||||
602 | // Sets the appropriate insert point on Builder where we can add | ||||||||
603 | // a replacement Instruction for V (if that is possible). | ||||||||
604 | static void setInsertionPoint(IRBuilder<> &Builder, Value *V, | ||||||||
605 | bool Before = true) { | ||||||||
606 | if (auto *PHI = dyn_cast<PHINode>(V)) { | ||||||||
607 | Builder.SetInsertPoint(&*PHI->getParent()->getFirstInsertionPt()); | ||||||||
608 | return; | ||||||||
609 | } | ||||||||
610 | if (auto *I = dyn_cast<Instruction>(V)) { | ||||||||
611 | if (!Before) | ||||||||
612 | I = &*std::next(I->getIterator()); | ||||||||
613 | Builder.SetInsertPoint(I); | ||||||||
614 | return; | ||||||||
615 | } | ||||||||
616 | if (auto *A = dyn_cast<Argument>(V)) { | ||||||||
617 | // Set the insertion point in the entry block. | ||||||||
618 | BasicBlock &Entry = A->getParent()->getEntryBlock(); | ||||||||
619 | Builder.SetInsertPoint(&*Entry.getFirstInsertionPt()); | ||||||||
620 | return; | ||||||||
621 | } | ||||||||
622 | // Otherwise, this is a constant and we don't need to set a new | ||||||||
623 | // insertion point. | ||||||||
624 | assert(isa<Constant>(V) && "Setting insertion point for unknown value!")(static_cast <bool> (isa<Constant>(V) && "Setting insertion point for unknown value!" ) ? void (0) : __assert_fail ("isa<Constant>(V) && \"Setting insertion point for unknown value!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 624, __extension__ __PRETTY_FUNCTION__)); | ||||||||
625 | } | ||||||||
626 | |||||||||
627 | /// Returns a re-written value of Start as an indexed GEP using Base as a | ||||||||
628 | /// pointer. | ||||||||
629 | static Value *rewriteGEPAsOffset(Value *Start, Value *Base, | ||||||||
630 | const DataLayout &DL, | ||||||||
631 | SetVector<Value *> &Explored) { | ||||||||
632 | // Perform all the substitutions. This is a bit tricky because we can | ||||||||
633 | // have cycles in our use-def chains. | ||||||||
634 | // 1. Create the PHI nodes without any incoming values. | ||||||||
635 | // 2. Create all the other values. | ||||||||
636 | // 3. Add the edges for the PHI nodes. | ||||||||
637 | // 4. Emit GEPs to get the original pointers. | ||||||||
638 | // 5. Remove the original instructions. | ||||||||
639 | Type *IndexType = IntegerType::get( | ||||||||
640 | Base->getContext(), DL.getIndexTypeSizeInBits(Start->getType())); | ||||||||
641 | |||||||||
642 | DenseMap<Value *, Value *> NewInsts; | ||||||||
643 | NewInsts[Base] = ConstantInt::getNullValue(IndexType); | ||||||||
644 | |||||||||
645 | // Create the new PHI nodes, without adding any incoming values. | ||||||||
646 | for (Value *Val : Explored) { | ||||||||
647 | if (Val == Base) | ||||||||
648 | continue; | ||||||||
649 | // Create empty phi nodes. This avoids cyclic dependencies when creating | ||||||||
650 | // the remaining instructions. | ||||||||
651 | if (auto *PHI = dyn_cast<PHINode>(Val)) | ||||||||
652 | NewInsts[PHI] = PHINode::Create(IndexType, PHI->getNumIncomingValues(), | ||||||||
653 | PHI->getName() + ".idx", PHI); | ||||||||
654 | } | ||||||||
655 | IRBuilder<> Builder(Base->getContext()); | ||||||||
656 | |||||||||
657 | // Create all the other instructions. | ||||||||
658 | for (Value *Val : Explored) { | ||||||||
659 | |||||||||
660 | if (NewInsts.find(Val) != NewInsts.end()) | ||||||||
661 | continue; | ||||||||
662 | |||||||||
663 | if (auto *CI = dyn_cast<CastInst>(Val)) { | ||||||||
664 | // Don't get rid of the intermediate variable here; the store can grow | ||||||||
665 | // the map which will invalidate the reference to the input value. | ||||||||
666 | Value *V = NewInsts[CI->getOperand(0)]; | ||||||||
667 | NewInsts[CI] = V; | ||||||||
668 | continue; | ||||||||
669 | } | ||||||||
670 | if (auto *GEP = dyn_cast<GEPOperator>(Val)) { | ||||||||
671 | Value *Index = NewInsts[GEP->getOperand(1)] ? NewInsts[GEP->getOperand(1)] | ||||||||
672 | : GEP->getOperand(1); | ||||||||
673 | setInsertionPoint(Builder, GEP); | ||||||||
674 | // Indices might need to be sign extended. GEPs will magically do | ||||||||
675 | // this, but we need to do it ourselves here. | ||||||||
676 | if (Index->getType()->getScalarSizeInBits() != | ||||||||
677 | NewInsts[GEP->getOperand(0)]->getType()->getScalarSizeInBits()) { | ||||||||
678 | Index = Builder.CreateSExtOrTrunc( | ||||||||
679 | Index, NewInsts[GEP->getOperand(0)]->getType(), | ||||||||
680 | GEP->getOperand(0)->getName() + ".sext"); | ||||||||
681 | } | ||||||||
682 | |||||||||
683 | auto *Op = NewInsts[GEP->getOperand(0)]; | ||||||||
684 | if (isa<ConstantInt>(Op) && cast<ConstantInt>(Op)->isZero()) | ||||||||
685 | NewInsts[GEP] = Index; | ||||||||
686 | else | ||||||||
687 | NewInsts[GEP] = Builder.CreateNSWAdd( | ||||||||
688 | Op, Index, GEP->getOperand(0)->getName() + ".add"); | ||||||||
689 | continue; | ||||||||
690 | } | ||||||||
691 | if (isa<PHINode>(Val)) | ||||||||
692 | continue; | ||||||||
693 | |||||||||
694 | llvm_unreachable("Unexpected instruction type")::llvm::llvm_unreachable_internal("Unexpected instruction type" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 694); | ||||||||
695 | } | ||||||||
696 | |||||||||
697 | // Add the incoming values to the PHI nodes. | ||||||||
698 | for (Value *Val : Explored) { | ||||||||
699 | if (Val == Base) | ||||||||
700 | continue; | ||||||||
701 | // All the instructions have been created, we can now add edges to the | ||||||||
702 | // phi nodes. | ||||||||
703 | if (auto *PHI = dyn_cast<PHINode>(Val)) { | ||||||||
704 | PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]); | ||||||||
705 | for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) { | ||||||||
706 | Value *NewIncoming = PHI->getIncomingValue(I); | ||||||||
707 | |||||||||
708 | if (NewInsts.find(NewIncoming) != NewInsts.end()) | ||||||||
709 | NewIncoming = NewInsts[NewIncoming]; | ||||||||
710 | |||||||||
711 | NewPhi->addIncoming(NewIncoming, PHI->getIncomingBlock(I)); | ||||||||
712 | } | ||||||||
713 | } | ||||||||
714 | } | ||||||||
715 | |||||||||
716 | for (Value *Val : Explored) { | ||||||||
717 | if (Val == Base) | ||||||||
718 | continue; | ||||||||
719 | |||||||||
720 | // Depending on the type, for external users we have to emit | ||||||||
721 | // a GEP or a GEP + ptrtoint. | ||||||||
722 | setInsertionPoint(Builder, Val, false); | ||||||||
723 | |||||||||
724 | // If required, create an inttoptr instruction for Base. | ||||||||
725 | Value *NewBase = Base; | ||||||||
726 | if (!Base->getType()->isPointerTy()) | ||||||||
727 | NewBase = Builder.CreateBitOrPointerCast(Base, Start->getType(), | ||||||||
728 | Start->getName() + "to.ptr"); | ||||||||
729 | |||||||||
730 | Value *GEP = Builder.CreateInBoundsGEP( | ||||||||
731 | Start->getType()->getPointerElementType(), NewBase, | ||||||||
732 | makeArrayRef(NewInsts[Val]), Val->getName() + ".ptr"); | ||||||||
733 | |||||||||
734 | if (!Val->getType()->isPointerTy()) { | ||||||||
735 | Value *Cast = Builder.CreatePointerCast(GEP, Val->getType(), | ||||||||
736 | Val->getName() + ".conv"); | ||||||||
737 | GEP = Cast; | ||||||||
738 | } | ||||||||
739 | Val->replaceAllUsesWith(GEP); | ||||||||
740 | } | ||||||||
741 | |||||||||
742 | return NewInsts[Start]; | ||||||||
743 | } | ||||||||
744 | |||||||||
745 | /// Looks through GEPs, IntToPtrInsts and PtrToIntInsts in order to express | ||||||||
746 | /// the input Value as a constant indexed GEP. Returns a pair containing | ||||||||
747 | /// the GEPs Pointer and Index. | ||||||||
748 | static std::pair<Value *, Value *> | ||||||||
749 | getAsConstantIndexedAddress(Value *V, const DataLayout &DL) { | ||||||||
750 | Type *IndexType = IntegerType::get(V->getContext(), | ||||||||
751 | DL.getIndexTypeSizeInBits(V->getType())); | ||||||||
752 | |||||||||
753 | Constant *Index = ConstantInt::getNullValue(IndexType); | ||||||||
754 | while (true) { | ||||||||
755 | if (GEPOperator *GEP = dyn_cast<GEPOperator>(V)) { | ||||||||
756 | // We accept only inbouds GEPs here to exclude the possibility of | ||||||||
757 | // overflow. | ||||||||
758 | if (!GEP->isInBounds()) | ||||||||
759 | break; | ||||||||
760 | if (GEP->hasAllConstantIndices() && GEP->getNumIndices() == 1 && | ||||||||
761 | GEP->getType() == V->getType()) { | ||||||||
762 | V = GEP->getOperand(0); | ||||||||
763 | Constant *GEPIndex = static_cast<Constant *>(GEP->getOperand(1)); | ||||||||
764 | Index = ConstantExpr::getAdd( | ||||||||
765 | Index, ConstantExpr::getSExtOrBitCast(GEPIndex, IndexType)); | ||||||||
766 | continue; | ||||||||
767 | } | ||||||||
768 | break; | ||||||||
769 | } | ||||||||
770 | if (auto *CI = dyn_cast<IntToPtrInst>(V)) { | ||||||||
771 | if (!CI->isNoopCast(DL)) | ||||||||
772 | break; | ||||||||
773 | V = CI->getOperand(0); | ||||||||
774 | continue; | ||||||||
775 | } | ||||||||
776 | if (auto *CI = dyn_cast<PtrToIntInst>(V)) { | ||||||||
777 | if (!CI->isNoopCast(DL)) | ||||||||
778 | break; | ||||||||
779 | V = CI->getOperand(0); | ||||||||
780 | continue; | ||||||||
781 | } | ||||||||
782 | break; | ||||||||
783 | } | ||||||||
784 | return {V, Index}; | ||||||||
785 | } | ||||||||
786 | |||||||||
787 | /// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant. | ||||||||
788 | /// We can look through PHIs, GEPs and casts in order to determine a common base | ||||||||
789 | /// between GEPLHS and RHS. | ||||||||
790 | static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS, | ||||||||
791 | ICmpInst::Predicate Cond, | ||||||||
792 | const DataLayout &DL) { | ||||||||
793 | // FIXME: Support vector of pointers. | ||||||||
794 | if (GEPLHS->getType()->isVectorTy()) | ||||||||
795 | return nullptr; | ||||||||
796 | |||||||||
797 | if (!GEPLHS->hasAllConstantIndices()) | ||||||||
798 | return nullptr; | ||||||||
799 | |||||||||
800 | // Make sure the pointers have the same type. | ||||||||
801 | if (GEPLHS->getType() != RHS->getType()) | ||||||||
802 | return nullptr; | ||||||||
803 | |||||||||
804 | Value *PtrBase, *Index; | ||||||||
805 | std::tie(PtrBase, Index) = getAsConstantIndexedAddress(GEPLHS, DL); | ||||||||
806 | |||||||||
807 | // The set of nodes that will take part in this transformation. | ||||||||
808 | SetVector<Value *> Nodes; | ||||||||
809 | |||||||||
810 | if (!canRewriteGEPAsOffset(RHS, PtrBase, DL, Nodes)) | ||||||||
811 | return nullptr; | ||||||||
812 | |||||||||
813 | // We know we can re-write this as | ||||||||
814 | // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) | ||||||||
815 | // Since we've only looked through inbouds GEPs we know that we | ||||||||
816 | // can't have overflow on either side. We can therefore re-write | ||||||||
817 | // this as: | ||||||||
818 | // OFFSET1 cmp OFFSET2 | ||||||||
819 | Value *NewRHS = rewriteGEPAsOffset(RHS, PtrBase, DL, Nodes); | ||||||||
820 | |||||||||
821 | // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written | ||||||||
822 | // GEP having PtrBase as the pointer base, and has returned in NewRHS the | ||||||||
823 | // offset. Since Index is the offset of LHS to the base pointer, we will now | ||||||||
824 | // compare the offsets instead of comparing the pointers. | ||||||||
825 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Index, NewRHS); | ||||||||
826 | } | ||||||||
827 | |||||||||
828 | /// Fold comparisons between a GEP instruction and something else. At this point | ||||||||
829 | /// we know that the GEP is on the LHS of the comparison. | ||||||||
830 | Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, | ||||||||
831 | ICmpInst::Predicate Cond, | ||||||||
832 | Instruction &I) { | ||||||||
833 | // Don't transform signed compares of GEPs into index compares. Even if the | ||||||||
834 | // GEP is inbounds, the final add of the base pointer can have signed overflow | ||||||||
835 | // and would change the result of the icmp. | ||||||||
836 | // e.g. "&foo[0] <s &foo[1]" can't be folded to "true" because "foo" could be | ||||||||
837 | // the maximum signed value for the pointer type. | ||||||||
838 | if (ICmpInst::isSigned(Cond)) | ||||||||
839 | return nullptr; | ||||||||
840 | |||||||||
841 | // Look through bitcasts and addrspacecasts. We do not however want to remove | ||||||||
842 | // 0 GEPs. | ||||||||
843 | if (!isa<GetElementPtrInst>(RHS)) | ||||||||
844 | RHS = RHS->stripPointerCasts(); | ||||||||
845 | |||||||||
846 | Value *PtrBase = GEPLHS->getOperand(0); | ||||||||
847 | // FIXME: Support vector pointer GEPs. | ||||||||
848 | if (PtrBase == RHS && GEPLHS->isInBounds() && | ||||||||
849 | !GEPLHS->getType()->isVectorTy()) { | ||||||||
850 | // ((gep Ptr, OFFSET) cmp Ptr) ---> (OFFSET cmp 0). | ||||||||
851 | // This transformation (ignoring the base and scales) is valid because we | ||||||||
852 | // know pointers can't overflow since the gep is inbounds. See if we can | ||||||||
853 | // output an optimized form. | ||||||||
854 | Value *Offset = evaluateGEPOffsetExpression(GEPLHS, *this, DL); | ||||||||
855 | |||||||||
856 | // If not, synthesize the offset the hard way. | ||||||||
857 | if (!Offset) | ||||||||
858 | Offset = EmitGEPOffset(GEPLHS); | ||||||||
859 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Offset, | ||||||||
860 | Constant::getNullValue(Offset->getType())); | ||||||||
861 | } | ||||||||
862 | |||||||||
863 | if (GEPLHS->isInBounds() && ICmpInst::isEquality(Cond) && | ||||||||
864 | isa<Constant>(RHS) && cast<Constant>(RHS)->isNullValue() && | ||||||||
865 | !NullPointerIsDefined(I.getFunction(), | ||||||||
866 | RHS->getType()->getPointerAddressSpace())) { | ||||||||
867 | // For most address spaces, an allocation can't be placed at null, but null | ||||||||
868 | // itself is treated as a 0 size allocation in the in bounds rules. Thus, | ||||||||
869 | // the only valid inbounds address derived from null, is null itself. | ||||||||
870 | // Thus, we have four cases to consider: | ||||||||
871 | // 1) Base == nullptr, Offset == 0 -> inbounds, null | ||||||||
872 | // 2) Base == nullptr, Offset != 0 -> poison as the result is out of bounds | ||||||||
873 | // 3) Base != nullptr, Offset == (-base) -> poison (crossing allocations) | ||||||||
874 | // 4) Base != nullptr, Offset != (-base) -> nonnull (and possibly poison) | ||||||||
875 | // | ||||||||
876 | // (Note if we're indexing a type of size 0, that simply collapses into one | ||||||||
877 | // of the buckets above.) | ||||||||
878 | // | ||||||||
879 | // In general, we're allowed to make values less poison (i.e. remove | ||||||||
880 | // sources of full UB), so in this case, we just select between the two | ||||||||
881 | // non-poison cases (1 and 4 above). | ||||||||
882 | // | ||||||||
883 | // For vectors, we apply the same reasoning on a per-lane basis. | ||||||||
884 | auto *Base = GEPLHS->getPointerOperand(); | ||||||||
885 | if (GEPLHS->getType()->isVectorTy() && Base->getType()->isPointerTy()) { | ||||||||
886 | auto EC = cast<VectorType>(GEPLHS->getType())->getElementCount(); | ||||||||
887 | Base = Builder.CreateVectorSplat(EC, Base); | ||||||||
888 | } | ||||||||
889 | return new ICmpInst(Cond, Base, | ||||||||
890 | ConstantExpr::getPointerBitCastOrAddrSpaceCast( | ||||||||
891 | cast<Constant>(RHS), Base->getType())); | ||||||||
892 | } else if (GEPOperator *GEPRHS = dyn_cast<GEPOperator>(RHS)) { | ||||||||
893 | // If the base pointers are different, but the indices are the same, just | ||||||||
894 | // compare the base pointer. | ||||||||
895 | if (PtrBase != GEPRHS->getOperand(0)) { | ||||||||
896 | bool IndicesTheSame = GEPLHS->getNumOperands()==GEPRHS->getNumOperands(); | ||||||||
897 | IndicesTheSame &= GEPLHS->getOperand(0)->getType() == | ||||||||
898 | GEPRHS->getOperand(0)->getType(); | ||||||||
899 | if (IndicesTheSame) | ||||||||
900 | for (unsigned i = 1, e = GEPLHS->getNumOperands(); i != e; ++i) | ||||||||
901 | if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) { | ||||||||
902 | IndicesTheSame = false; | ||||||||
903 | break; | ||||||||
904 | } | ||||||||
905 | |||||||||
906 | // If all indices are the same, just compare the base pointers. | ||||||||
907 | Type *BaseType = GEPLHS->getOperand(0)->getType(); | ||||||||
908 | if (IndicesTheSame && CmpInst::makeCmpResultType(BaseType) == I.getType()) | ||||||||
909 | return new ICmpInst(Cond, GEPLHS->getOperand(0), GEPRHS->getOperand(0)); | ||||||||
910 | |||||||||
911 | // If we're comparing GEPs with two base pointers that only differ in type | ||||||||
912 | // and both GEPs have only constant indices or just one use, then fold | ||||||||
913 | // the compare with the adjusted indices. | ||||||||
914 | // FIXME: Support vector of pointers. | ||||||||
915 | if (GEPLHS->isInBounds() && GEPRHS->isInBounds() && | ||||||||
916 | (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) && | ||||||||
917 | (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse()) && | ||||||||
918 | PtrBase->stripPointerCasts() == | ||||||||
919 | GEPRHS->getOperand(0)->stripPointerCasts() && | ||||||||
920 | !GEPLHS->getType()->isVectorTy()) { | ||||||||
921 | Value *LOffset = EmitGEPOffset(GEPLHS); | ||||||||
922 | Value *ROffset = EmitGEPOffset(GEPRHS); | ||||||||
923 | |||||||||
924 | // If we looked through an addrspacecast between different sized address | ||||||||
925 | // spaces, the LHS and RHS pointers are different sized | ||||||||
926 | // integers. Truncate to the smaller one. | ||||||||
927 | Type *LHSIndexTy = LOffset->getType(); | ||||||||
928 | Type *RHSIndexTy = ROffset->getType(); | ||||||||
929 | if (LHSIndexTy != RHSIndexTy) { | ||||||||
930 | if (LHSIndexTy->getPrimitiveSizeInBits().getFixedSize() < | ||||||||
931 | RHSIndexTy->getPrimitiveSizeInBits().getFixedSize()) { | ||||||||
932 | ROffset = Builder.CreateTrunc(ROffset, LHSIndexTy); | ||||||||
933 | } else | ||||||||
934 | LOffset = Builder.CreateTrunc(LOffset, RHSIndexTy); | ||||||||
935 | } | ||||||||
936 | |||||||||
937 | Value *Cmp = Builder.CreateICmp(ICmpInst::getSignedPredicate(Cond), | ||||||||
938 | LOffset, ROffset); | ||||||||
939 | return replaceInstUsesWith(I, Cmp); | ||||||||
940 | } | ||||||||
941 | |||||||||
942 | // Otherwise, the base pointers are different and the indices are | ||||||||
943 | // different. Try convert this to an indexed compare by looking through | ||||||||
944 | // PHIs/casts. | ||||||||
945 | return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); | ||||||||
946 | } | ||||||||
947 | |||||||||
948 | // If one of the GEPs has all zero indices, recurse. | ||||||||
949 | // FIXME: Handle vector of pointers. | ||||||||
950 | if (!GEPLHS->getType()->isVectorTy() && GEPLHS->hasAllZeroIndices()) | ||||||||
951 | return foldGEPICmp(GEPRHS, GEPLHS->getOperand(0), | ||||||||
952 | ICmpInst::getSwappedPredicate(Cond), I); | ||||||||
953 | |||||||||
954 | // If the other GEP has all zero indices, recurse. | ||||||||
955 | // FIXME: Handle vector of pointers. | ||||||||
956 | if (!GEPRHS->getType()->isVectorTy() && GEPRHS->hasAllZeroIndices()) | ||||||||
957 | return foldGEPICmp(GEPLHS, GEPRHS->getOperand(0), Cond, I); | ||||||||
958 | |||||||||
959 | bool GEPsInBounds = GEPLHS->isInBounds() && GEPRHS->isInBounds(); | ||||||||
960 | if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands()) { | ||||||||
961 | // If the GEPs only differ by one index, compare it. | ||||||||
962 | unsigned NumDifferences = 0; // Keep track of # differences. | ||||||||
963 | unsigned DiffOperand = 0; // The operand that differs. | ||||||||
964 | for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i) | ||||||||
965 | if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) { | ||||||||
966 | Type *LHSType = GEPLHS->getOperand(i)->getType(); | ||||||||
967 | Type *RHSType = GEPRHS->getOperand(i)->getType(); | ||||||||
968 | // FIXME: Better support for vector of pointers. | ||||||||
969 | if (LHSType->getPrimitiveSizeInBits() != | ||||||||
970 | RHSType->getPrimitiveSizeInBits() || | ||||||||
971 | (GEPLHS->getType()->isVectorTy() && | ||||||||
972 | (!LHSType->isVectorTy() || !RHSType->isVectorTy()))) { | ||||||||
973 | // Irreconcilable differences. | ||||||||
974 | NumDifferences = 2; | ||||||||
975 | break; | ||||||||
976 | } | ||||||||
977 | |||||||||
978 | if (NumDifferences++) break; | ||||||||
979 | DiffOperand = i; | ||||||||
980 | } | ||||||||
981 | |||||||||
982 | if (NumDifferences == 0) // SAME GEP? | ||||||||
983 | return replaceInstUsesWith(I, // No comparison is needed here. | ||||||||
984 | ConstantInt::get(I.getType(), ICmpInst::isTrueWhenEqual(Cond))); | ||||||||
985 | |||||||||
986 | else if (NumDifferences == 1 && GEPsInBounds) { | ||||||||
987 | Value *LHSV = GEPLHS->getOperand(DiffOperand); | ||||||||
988 | Value *RHSV = GEPRHS->getOperand(DiffOperand); | ||||||||
989 | // Make sure we do a signed comparison here. | ||||||||
990 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), LHSV, RHSV); | ||||||||
991 | } | ||||||||
992 | } | ||||||||
993 | |||||||||
994 | // Only lower this if the icmp is the only user of the GEP or if we expect | ||||||||
995 | // the result to fold to a constant! | ||||||||
996 | if (GEPsInBounds && (isa<ConstantExpr>(GEPLHS) || GEPLHS->hasOneUse()) && | ||||||||
997 | (isa<ConstantExpr>(GEPRHS) || GEPRHS->hasOneUse())) { | ||||||||
998 | // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2) | ||||||||
999 | Value *L = EmitGEPOffset(GEPLHS); | ||||||||
1000 | Value *R = EmitGEPOffset(GEPRHS); | ||||||||
1001 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), L, R); | ||||||||
1002 | } | ||||||||
1003 | } | ||||||||
1004 | |||||||||
1005 | // Try convert this to an indexed compare by looking through PHIs/casts as a | ||||||||
1006 | // last resort. | ||||||||
1007 | return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); | ||||||||
1008 | } | ||||||||
1009 | |||||||||
1010 | Instruction *InstCombinerImpl::foldAllocaCmp(ICmpInst &ICI, | ||||||||
1011 | const AllocaInst *Alloca, | ||||||||
1012 | const Value *Other) { | ||||||||
1013 | assert(ICI.isEquality() && "Cannot fold non-equality comparison.")(static_cast <bool> (ICI.isEquality() && "Cannot fold non-equality comparison." ) ? void (0) : __assert_fail ("ICI.isEquality() && \"Cannot fold non-equality comparison.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1013, __extension__ __PRETTY_FUNCTION__)); | ||||||||
1014 | |||||||||
1015 | // It would be tempting to fold away comparisons between allocas and any | ||||||||
1016 | // pointer not based on that alloca (e.g. an argument). However, even | ||||||||
1017 | // though such pointers cannot alias, they can still compare equal. | ||||||||
1018 | // | ||||||||
1019 | // But LLVM doesn't specify where allocas get their memory, so if the alloca | ||||||||
1020 | // doesn't escape we can argue that it's impossible to guess its value, and we | ||||||||
1021 | // can therefore act as if any such guesses are wrong. | ||||||||
1022 | // | ||||||||
1023 | // The code below checks that the alloca doesn't escape, and that it's only | ||||||||
1024 | // used in a comparison once (the current instruction). The | ||||||||
1025 | // single-comparison-use condition ensures that we're trivially folding all | ||||||||
1026 | // comparisons against the alloca consistently, and avoids the risk of | ||||||||
1027 | // erroneously folding a comparison of the pointer with itself. | ||||||||
1028 | |||||||||
1029 | unsigned MaxIter = 32; // Break cycles and bound to constant-time. | ||||||||
1030 | |||||||||
1031 | SmallVector<const Use *, 32> Worklist; | ||||||||
1032 | for (const Use &U : Alloca->uses()) { | ||||||||
1033 | if (Worklist.size() >= MaxIter) | ||||||||
1034 | return nullptr; | ||||||||
1035 | Worklist.push_back(&U); | ||||||||
1036 | } | ||||||||
1037 | |||||||||
1038 | unsigned NumCmps = 0; | ||||||||
1039 | while (!Worklist.empty()) { | ||||||||
1040 | assert(Worklist.size() <= MaxIter)(static_cast <bool> (Worklist.size() <= MaxIter) ? void (0) : __assert_fail ("Worklist.size() <= MaxIter", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1040, __extension__ __PRETTY_FUNCTION__)); | ||||||||
1041 | const Use *U = Worklist.pop_back_val(); | ||||||||
1042 | const Value *V = U->getUser(); | ||||||||
1043 | --MaxIter; | ||||||||
1044 | |||||||||
1045 | if (isa<BitCastInst>(V) || isa<GetElementPtrInst>(V) || isa<PHINode>(V) || | ||||||||
1046 | isa<SelectInst>(V)) { | ||||||||
1047 | // Track the uses. | ||||||||
1048 | } else if (isa<LoadInst>(V)) { | ||||||||
1049 | // Loading from the pointer doesn't escape it. | ||||||||
1050 | continue; | ||||||||
1051 | } else if (const auto *SI = dyn_cast<StoreInst>(V)) { | ||||||||
1052 | // Storing *to* the pointer is fine, but storing the pointer escapes it. | ||||||||
1053 | if (SI->getValueOperand() == U->get()) | ||||||||
1054 | return nullptr; | ||||||||
1055 | continue; | ||||||||
1056 | } else if (isa<ICmpInst>(V)) { | ||||||||
1057 | if (NumCmps++) | ||||||||
1058 | return nullptr; // Found more than one cmp. | ||||||||
1059 | continue; | ||||||||
1060 | } else if (const auto *Intrin = dyn_cast<IntrinsicInst>(V)) { | ||||||||
1061 | switch (Intrin->getIntrinsicID()) { | ||||||||
1062 | // These intrinsics don't escape or compare the pointer. Memset is safe | ||||||||
1063 | // because we don't allow ptrtoint. Memcpy and memmove are safe because | ||||||||
1064 | // we don't allow stores, so src cannot point to V. | ||||||||
1065 | case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: | ||||||||
1066 | case Intrinsic::memcpy: case Intrinsic::memmove: case Intrinsic::memset: | ||||||||
1067 | continue; | ||||||||
1068 | default: | ||||||||
1069 | return nullptr; | ||||||||
1070 | } | ||||||||
1071 | } else { | ||||||||
1072 | return nullptr; | ||||||||
1073 | } | ||||||||
1074 | for (const Use &U : V->uses()) { | ||||||||
1075 | if (Worklist.size() >= MaxIter) | ||||||||
1076 | return nullptr; | ||||||||
1077 | Worklist.push_back(&U); | ||||||||
1078 | } | ||||||||
1079 | } | ||||||||
1080 | |||||||||
1081 | Type *CmpTy = CmpInst::makeCmpResultType(Other->getType()); | ||||||||
1082 | return replaceInstUsesWith( | ||||||||
1083 | ICI, | ||||||||
1084 | ConstantInt::get(CmpTy, !CmpInst::isTrueWhenEqual(ICI.getPredicate()))); | ||||||||
1085 | } | ||||||||
1086 | |||||||||
1087 | /// Fold "icmp pred (X+C), X". | ||||||||
1088 | Instruction *InstCombinerImpl::foldICmpAddOpConst(Value *X, const APInt &C, | ||||||||
1089 | ICmpInst::Predicate Pred) { | ||||||||
1090 | // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0, | ||||||||
1091 | // so the values can never be equal. Similarly for all other "or equals" | ||||||||
1092 | // operators. | ||||||||
1093 | assert(!!C && "C should not be zero!")(static_cast <bool> (!!C && "C should not be zero!" ) ? void (0) : __assert_fail ("!!C && \"C should not be zero!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1093, __extension__ __PRETTY_FUNCTION__)); | ||||||||
1094 | |||||||||
1095 | // (X+1) <u X --> X >u (MAXUINT-1) --> X == 255 | ||||||||
1096 | // (X+2) <u X --> X >u (MAXUINT-2) --> X > 253 | ||||||||
1097 | // (X+MAXUINT) <u X --> X >u (MAXUINT-MAXUINT) --> X != 0 | ||||||||
1098 | if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) { | ||||||||
1099 | Constant *R = ConstantInt::get(X->getType(), | ||||||||
1100 | APInt::getMaxValue(C.getBitWidth()) - C); | ||||||||
1101 | return new ICmpInst(ICmpInst::ICMP_UGT, X, R); | ||||||||
1102 | } | ||||||||
1103 | |||||||||
1104 | // (X+1) >u X --> X <u (0-1) --> X != 255 | ||||||||
1105 | // (X+2) >u X --> X <u (0-2) --> X <u 254 | ||||||||
1106 | // (X+MAXUINT) >u X --> X <u (0-MAXUINT) --> X <u 1 --> X == 0 | ||||||||
1107 | if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) | ||||||||
1108 | return new ICmpInst(ICmpInst::ICMP_ULT, X, | ||||||||
1109 | ConstantInt::get(X->getType(), -C)); | ||||||||
1110 | |||||||||
1111 | APInt SMax = APInt::getSignedMaxValue(C.getBitWidth()); | ||||||||
1112 | |||||||||
1113 | // (X+ 1) <s X --> X >s (MAXSINT-1) --> X == 127 | ||||||||
1114 | // (X+ 2) <s X --> X >s (MAXSINT-2) --> X >s 125 | ||||||||
1115 | // (X+MAXSINT) <s X --> X >s (MAXSINT-MAXSINT) --> X >s 0 | ||||||||
1116 | // (X+MINSINT) <s X --> X >s (MAXSINT-MINSINT) --> X >s -1 | ||||||||
1117 | // (X+ -2) <s X --> X >s (MAXSINT- -2) --> X >s 126 | ||||||||
1118 | // (X+ -1) <s X --> X >s (MAXSINT- -1) --> X != 127 | ||||||||
1119 | if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) | ||||||||
1120 | return new ICmpInst(ICmpInst::ICMP_SGT, X, | ||||||||
1121 | ConstantInt::get(X->getType(), SMax - C)); | ||||||||
1122 | |||||||||
1123 | // (X+ 1) >s X --> X <s (MAXSINT-(1-1)) --> X != 127 | ||||||||
1124 | // (X+ 2) >s X --> X <s (MAXSINT-(2-1)) --> X <s 126 | ||||||||
1125 | // (X+MAXSINT) >s X --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1 | ||||||||
1126 | // (X+MINSINT) >s X --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2 | ||||||||
1127 | // (X+ -2) >s X --> X <s (MAXSINT-(-2-1)) --> X <s -126 | ||||||||
1128 | // (X+ -1) >s X --> X <s (MAXSINT-(-1-1)) --> X == -128 | ||||||||
1129 | |||||||||
1130 | assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE)(static_cast <bool> (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) ? void (0) : __assert_fail ("Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1130, __extension__ __PRETTY_FUNCTION__)); | ||||||||
1131 | return new ICmpInst(ICmpInst::ICMP_SLT, X, | ||||||||
1132 | ConstantInt::get(X->getType(), SMax - (C - 1))); | ||||||||
1133 | } | ||||||||
1134 | |||||||||
1135 | /// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" -> | ||||||||
1136 | /// (icmp eq/ne A, Log2(AP2/AP1)) -> | ||||||||
1137 | /// (icmp eq/ne A, Log2(AP2) - Log2(AP1)). | ||||||||
1138 | Instruction *InstCombinerImpl::foldICmpShrConstConst(ICmpInst &I, Value *A, | ||||||||
1139 | const APInt &AP1, | ||||||||
1140 | const APInt &AP2) { | ||||||||
1141 | assert(I.isEquality() && "Cannot fold icmp gt/lt")(static_cast <bool> (I.isEquality() && "Cannot fold icmp gt/lt" ) ? void (0) : __assert_fail ("I.isEquality() && \"Cannot fold icmp gt/lt\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1141, __extension__ __PRETTY_FUNCTION__)); | ||||||||
1142 | |||||||||
1143 | auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { | ||||||||
1144 | if (I.getPredicate() == I.ICMP_NE) | ||||||||
1145 | Pred = CmpInst::getInversePredicate(Pred); | ||||||||
1146 | return new ICmpInst(Pred, LHS, RHS); | ||||||||
1147 | }; | ||||||||
1148 | |||||||||
1149 | // Don't bother doing any work for cases which InstSimplify handles. | ||||||||
1150 | if (AP2.isNullValue()) | ||||||||
1151 | return nullptr; | ||||||||
1152 | |||||||||
1153 | bool IsAShr = isa<AShrOperator>(I.getOperand(0)); | ||||||||
1154 | if (IsAShr) { | ||||||||
1155 | if (AP2.isAllOnesValue()) | ||||||||
1156 | return nullptr; | ||||||||
1157 | if (AP2.isNegative() != AP1.isNegative()) | ||||||||
1158 | return nullptr; | ||||||||
1159 | if (AP2.sgt(AP1)) | ||||||||
1160 | return nullptr; | ||||||||
1161 | } | ||||||||
1162 | |||||||||
1163 | if (!AP1) | ||||||||
1164 | // 'A' must be large enough to shift out the highest set bit. | ||||||||
1165 | return getICmp(I.ICMP_UGT, A, | ||||||||
1166 | ConstantInt::get(A->getType(), AP2.logBase2())); | ||||||||
1167 | |||||||||
1168 | if (AP1 == AP2) | ||||||||
1169 | return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType())); | ||||||||
1170 | |||||||||
1171 | int Shift; | ||||||||
1172 | if (IsAShr && AP1.isNegative()) | ||||||||
1173 | Shift = AP1.countLeadingOnes() - AP2.countLeadingOnes(); | ||||||||
1174 | else | ||||||||
1175 | Shift = AP1.countLeadingZeros() - AP2.countLeadingZeros(); | ||||||||
1176 | |||||||||
1177 | if (Shift > 0) { | ||||||||
1178 | if (IsAShr && AP1 == AP2.ashr(Shift)) { | ||||||||
1179 | // There are multiple solutions if we are comparing against -1 and the LHS | ||||||||
1180 | // of the ashr is not a power of two. | ||||||||
1181 | if (AP1.isAllOnesValue() && !AP2.isPowerOf2()) | ||||||||
1182 | return getICmp(I.ICMP_UGE, A, ConstantInt::get(A->getType(), Shift)); | ||||||||
1183 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); | ||||||||
1184 | } else if (AP1 == AP2.lshr(Shift)) { | ||||||||
1185 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); | ||||||||
1186 | } | ||||||||
1187 | } | ||||||||
1188 | |||||||||
1189 | // Shifting const2 will never be equal to const1. | ||||||||
1190 | // FIXME: This should always be handled by InstSimplify? | ||||||||
1191 | auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE); | ||||||||
1192 | return replaceInstUsesWith(I, TorF); | ||||||||
1193 | } | ||||||||
1194 | |||||||||
1195 | /// Handle "(icmp eq/ne (shl AP2, A), AP1)" -> | ||||||||
1196 | /// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)). | ||||||||
1197 | Instruction *InstCombinerImpl::foldICmpShlConstConst(ICmpInst &I, Value *A, | ||||||||
1198 | const APInt &AP1, | ||||||||
1199 | const APInt &AP2) { | ||||||||
1200 | assert(I.isEquality() && "Cannot fold icmp gt/lt")(static_cast <bool> (I.isEquality() && "Cannot fold icmp gt/lt" ) ? void (0) : __assert_fail ("I.isEquality() && \"Cannot fold icmp gt/lt\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1200, __extension__ __PRETTY_FUNCTION__)); | ||||||||
1201 | |||||||||
1202 | auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { | ||||||||
1203 | if (I.getPredicate() == I.ICMP_NE) | ||||||||
1204 | Pred = CmpInst::getInversePredicate(Pred); | ||||||||
1205 | return new ICmpInst(Pred, LHS, RHS); | ||||||||
1206 | }; | ||||||||
1207 | |||||||||
1208 | // Don't bother doing any work for cases which InstSimplify handles. | ||||||||
1209 | if (AP2.isNullValue()) | ||||||||
1210 | return nullptr; | ||||||||
1211 | |||||||||
1212 | unsigned AP2TrailingZeros = AP2.countTrailingZeros(); | ||||||||
1213 | |||||||||
1214 | if (!AP1 && AP2TrailingZeros != 0) | ||||||||
1215 | return getICmp( | ||||||||
1216 | I.ICMP_UGE, A, | ||||||||
1217 | ConstantInt::get(A->getType(), AP2.getBitWidth() - AP2TrailingZeros)); | ||||||||
1218 | |||||||||
1219 | if (AP1 == AP2) | ||||||||
1220 | return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType())); | ||||||||
1221 | |||||||||
1222 | // Get the distance between the lowest bits that are set. | ||||||||
1223 | int Shift = AP1.countTrailingZeros() - AP2TrailingZeros; | ||||||||
1224 | |||||||||
1225 | if (Shift > 0 && AP2.shl(Shift) == AP1) | ||||||||
1226 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); | ||||||||
1227 | |||||||||
1228 | // Shifting const2 will never be equal to const1. | ||||||||
1229 | // FIXME: This should always be handled by InstSimplify? | ||||||||
1230 | auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE); | ||||||||
1231 | return replaceInstUsesWith(I, TorF); | ||||||||
1232 | } | ||||||||
1233 | |||||||||
1234 | /// The caller has matched a pattern of the form: | ||||||||
1235 | /// I = icmp ugt (add (add A, B), CI2), CI1 | ||||||||
1236 | /// If this is of the form: | ||||||||
1237 | /// sum = a + b | ||||||||
1238 | /// if (sum+128 >u 255) | ||||||||
1239 | /// Then replace it with llvm.sadd.with.overflow.i8. | ||||||||
1240 | /// | ||||||||
1241 | static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B, | ||||||||
1242 | ConstantInt *CI2, ConstantInt *CI1, | ||||||||
1243 | InstCombinerImpl &IC) { | ||||||||
1244 | // The transformation we're trying to do here is to transform this into an | ||||||||
1245 | // llvm.sadd.with.overflow. To do this, we have to replace the original add | ||||||||
1246 | // with a narrower add, and discard the add-with-constant that is part of the | ||||||||
1247 | // range check (if we can't eliminate it, this isn't profitable). | ||||||||
1248 | |||||||||
1249 | // In order to eliminate the add-with-constant, the compare can be its only | ||||||||
1250 | // use. | ||||||||
1251 | Instruction *AddWithCst = cast<Instruction>(I.getOperand(0)); | ||||||||
1252 | if (!AddWithCst->hasOneUse()) | ||||||||
1253 | return nullptr; | ||||||||
1254 | |||||||||
1255 | // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow. | ||||||||
1256 | if (!CI2->getValue().isPowerOf2()) | ||||||||
1257 | return nullptr; | ||||||||
1258 | unsigned NewWidth = CI2->getValue().countTrailingZeros(); | ||||||||
1259 | if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31) | ||||||||
1260 | return nullptr; | ||||||||
1261 | |||||||||
1262 | // The width of the new add formed is 1 more than the bias. | ||||||||
1263 | ++NewWidth; | ||||||||
1264 | |||||||||
1265 | // Check to see that CI1 is an all-ones value with NewWidth bits. | ||||||||
1266 | if (CI1->getBitWidth() == NewWidth || | ||||||||
1267 | CI1->getValue() != APInt::getLowBitsSet(CI1->getBitWidth(), NewWidth)) | ||||||||
1268 | return nullptr; | ||||||||
1269 | |||||||||
1270 | // This is only really a signed overflow check if the inputs have been | ||||||||
1271 | // sign-extended; check for that condition. For example, if CI2 is 2^31 and | ||||||||
1272 | // the operands of the add are 64 bits wide, we need at least 33 sign bits. | ||||||||
1273 | unsigned NeededSignBits = CI1->getBitWidth() - NewWidth + 1; | ||||||||
1274 | if (IC.ComputeNumSignBits(A, 0, &I) < NeededSignBits || | ||||||||
1275 | IC.ComputeNumSignBits(B, 0, &I) < NeededSignBits) | ||||||||
1276 | return nullptr; | ||||||||
1277 | |||||||||
1278 | // In order to replace the original add with a narrower | ||||||||
1279 | // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant | ||||||||
1280 | // and truncates that discard the high bits of the add. Verify that this is | ||||||||
1281 | // the case. | ||||||||
1282 | Instruction *OrigAdd = cast<Instruction>(AddWithCst->getOperand(0)); | ||||||||
1283 | for (User *U : OrigAdd->users()) { | ||||||||
1284 | if (U == AddWithCst) | ||||||||
1285 | continue; | ||||||||
1286 | |||||||||
1287 | // Only accept truncates for now. We would really like a nice recursive | ||||||||
1288 | // predicate like SimplifyDemandedBits, but which goes downwards the use-def | ||||||||
1289 | // chain to see which bits of a value are actually demanded. If the | ||||||||
1290 | // original add had another add which was then immediately truncated, we | ||||||||
1291 | // could still do the transformation. | ||||||||
1292 | TruncInst *TI = dyn_cast<TruncInst>(U); | ||||||||
1293 | if (!TI || TI->getType()->getPrimitiveSizeInBits() > NewWidth) | ||||||||
1294 | return nullptr; | ||||||||
1295 | } | ||||||||
1296 | |||||||||
1297 | // If the pattern matches, truncate the inputs to the narrower type and | ||||||||
1298 | // use the sadd_with_overflow intrinsic to efficiently compute both the | ||||||||
1299 | // result and the overflow bit. | ||||||||
1300 | Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth); | ||||||||
1301 | Function *F = Intrinsic::getDeclaration( | ||||||||
1302 | I.getModule(), Intrinsic::sadd_with_overflow, NewType); | ||||||||
1303 | |||||||||
1304 | InstCombiner::BuilderTy &Builder = IC.Builder; | ||||||||
1305 | |||||||||
1306 | // Put the new code above the original add, in case there are any uses of the | ||||||||
1307 | // add between the add and the compare. | ||||||||
1308 | Builder.SetInsertPoint(OrigAdd); | ||||||||
1309 | |||||||||
1310 | Value *TruncA = Builder.CreateTrunc(A, NewType, A->getName() + ".trunc"); | ||||||||
1311 | Value *TruncB = Builder.CreateTrunc(B, NewType, B->getName() + ".trunc"); | ||||||||
1312 | CallInst *Call = Builder.CreateCall(F, {TruncA, TruncB}, "sadd"); | ||||||||
1313 | Value *Add = Builder.CreateExtractValue(Call, 0, "sadd.result"); | ||||||||
1314 | Value *ZExt = Builder.CreateZExt(Add, OrigAdd->getType()); | ||||||||
1315 | |||||||||
1316 | // The inner add was the result of the narrow add, zero extended to the | ||||||||
1317 | // wider type. Replace it with the result computed by the intrinsic. | ||||||||
1318 | IC.replaceInstUsesWith(*OrigAdd, ZExt); | ||||||||
1319 | IC.eraseInstFromFunction(*OrigAdd); | ||||||||
1320 | |||||||||
1321 | // The original icmp gets replaced with the overflow value. | ||||||||
1322 | return ExtractValueInst::Create(Call, 1, "sadd.overflow"); | ||||||||
1323 | } | ||||||||
1324 | |||||||||
1325 | /// If we have: | ||||||||
1326 | /// icmp eq/ne (urem/srem %x, %y), 0 | ||||||||
1327 | /// iff %y is a power-of-two, we can replace this with a bit test: | ||||||||
1328 | /// icmp eq/ne (and %x, (add %y, -1)), 0 | ||||||||
1329 | Instruction *InstCombinerImpl::foldIRemByPowerOfTwoToBitTest(ICmpInst &I) { | ||||||||
1330 | // This fold is only valid for equality predicates. | ||||||||
1331 | if (!I.isEquality()) | ||||||||
1332 | return nullptr; | ||||||||
1333 | ICmpInst::Predicate Pred; | ||||||||
1334 | Value *X, *Y, *Zero; | ||||||||
1335 | if (!match(&I, m_ICmp(Pred, m_OneUse(m_IRem(m_Value(X), m_Value(Y))), | ||||||||
1336 | m_CombineAnd(m_Zero(), m_Value(Zero))))) | ||||||||
1337 | return nullptr; | ||||||||
1338 | if (!isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, 0, &I)) | ||||||||
1339 | return nullptr; | ||||||||
1340 | // This may increase instruction count, we don't enforce that Y is a constant. | ||||||||
1341 | Value *Mask = Builder.CreateAdd(Y, Constant::getAllOnesValue(Y->getType())); | ||||||||
1342 | Value *Masked = Builder.CreateAnd(X, Mask); | ||||||||
1343 | return ICmpInst::Create(Instruction::ICmp, Pred, Masked, Zero); | ||||||||
1344 | } | ||||||||
1345 | |||||||||
1346 | /// Fold equality-comparison between zero and any (maybe truncated) right-shift | ||||||||
1347 | /// by one-less-than-bitwidth into a sign test on the original value. | ||||||||
1348 | Instruction *InstCombinerImpl::foldSignBitTest(ICmpInst &I) { | ||||||||
1349 | Instruction *Val; | ||||||||
1350 | ICmpInst::Predicate Pred; | ||||||||
1351 | if (!I.isEquality() || !match(&I, m_ICmp(Pred, m_Instruction(Val), m_Zero()))) | ||||||||
1352 | return nullptr; | ||||||||
1353 | |||||||||
1354 | Value *X; | ||||||||
1355 | Type *XTy; | ||||||||
1356 | |||||||||
1357 | Constant *C; | ||||||||
1358 | if (match(Val, m_TruncOrSelf(m_Shr(m_Value(X), m_Constant(C))))) { | ||||||||
1359 | XTy = X->getType(); | ||||||||
1360 | unsigned XBitWidth = XTy->getScalarSizeInBits(); | ||||||||
1361 | if (!match(C, m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_EQ, | ||||||||
1362 | APInt(XBitWidth, XBitWidth - 1)))) | ||||||||
1363 | return nullptr; | ||||||||
1364 | } else if (isa<BinaryOperator>(Val) && | ||||||||
1365 | (X = reassociateShiftAmtsOfTwoSameDirectionShifts( | ||||||||
1366 | cast<BinaryOperator>(Val), SQ.getWithInstruction(Val), | ||||||||
1367 | /*AnalyzeForSignBitExtraction=*/true))) { | ||||||||
1368 | XTy = X->getType(); | ||||||||
1369 | } else | ||||||||
1370 | return nullptr; | ||||||||
1371 | |||||||||
1372 | return ICmpInst::Create(Instruction::ICmp, | ||||||||
1373 | Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_SGE | ||||||||
1374 | : ICmpInst::ICMP_SLT, | ||||||||
1375 | X, ConstantInt::getNullValue(XTy)); | ||||||||
1376 | } | ||||||||
1377 | |||||||||
1378 | // Handle icmp pred X, 0 | ||||||||
1379 | Instruction *InstCombinerImpl::foldICmpWithZero(ICmpInst &Cmp) { | ||||||||
1380 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
1381 | if (!match(Cmp.getOperand(1), m_Zero())) | ||||||||
1382 | return nullptr; | ||||||||
1383 | |||||||||
1384 | // (icmp sgt smin(PosA, B) 0) -> (icmp sgt B 0) | ||||||||
1385 | if (Pred == ICmpInst::ICMP_SGT) { | ||||||||
1386 | Value *A, *B; | ||||||||
1387 | SelectPatternResult SPR = matchSelectPattern(Cmp.getOperand(0), A, B); | ||||||||
1388 | if (SPR.Flavor == SPF_SMIN) { | ||||||||
1389 | if (isKnownPositive(A, DL, 0, &AC, &Cmp, &DT)) | ||||||||
1390 | return new ICmpInst(Pred, B, Cmp.getOperand(1)); | ||||||||
1391 | if (isKnownPositive(B, DL, 0, &AC, &Cmp, &DT)) | ||||||||
1392 | return new ICmpInst(Pred, A, Cmp.getOperand(1)); | ||||||||
1393 | } | ||||||||
1394 | } | ||||||||
1395 | |||||||||
1396 | if (Instruction *New = foldIRemByPowerOfTwoToBitTest(Cmp)) | ||||||||
1397 | return New; | ||||||||
1398 | |||||||||
1399 | // Given: | ||||||||
1400 | // icmp eq/ne (urem %x, %y), 0 | ||||||||
1401 | // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem': | ||||||||
1402 | // icmp eq/ne %x, 0 | ||||||||
1403 | Value *X, *Y; | ||||||||
1404 | if (match(Cmp.getOperand(0), m_URem(m_Value(X), m_Value(Y))) && | ||||||||
1405 | ICmpInst::isEquality(Pred)) { | ||||||||
1406 | KnownBits XKnown = computeKnownBits(X, 0, &Cmp); | ||||||||
1407 | KnownBits YKnown = computeKnownBits(Y, 0, &Cmp); | ||||||||
1408 | if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2) | ||||||||
1409 | return new ICmpInst(Pred, X, Cmp.getOperand(1)); | ||||||||
1410 | } | ||||||||
1411 | |||||||||
1412 | return nullptr; | ||||||||
1413 | } | ||||||||
1414 | |||||||||
1415 | /// Fold icmp Pred X, C. | ||||||||
1416 | /// TODO: This code structure does not make sense. The saturating add fold | ||||||||
1417 | /// should be moved to some other helper and extended as noted below (it is also | ||||||||
1418 | /// possible that code has been made unnecessary - do we canonicalize IR to | ||||||||
1419 | /// overflow/saturating intrinsics or not?). | ||||||||
1420 | Instruction *InstCombinerImpl::foldICmpWithConstant(ICmpInst &Cmp) { | ||||||||
1421 | // Match the following pattern, which is a common idiom when writing | ||||||||
1422 | // overflow-safe integer arithmetic functions. The source performs an addition | ||||||||
1423 | // in wider type and explicitly checks for overflow using comparisons against | ||||||||
1424 | // INT_MIN and INT_MAX. Simplify by using the sadd_with_overflow intrinsic. | ||||||||
1425 | // | ||||||||
1426 | // TODO: This could probably be generalized to handle other overflow-safe | ||||||||
1427 | // operations if we worked out the formulas to compute the appropriate magic | ||||||||
1428 | // constants. | ||||||||
1429 | // | ||||||||
1430 | // sum = a + b | ||||||||
1431 | // if (sum+128 >u 255) ... -> llvm.sadd.with.overflow.i8 | ||||||||
1432 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
1433 | Value *Op0 = Cmp.getOperand(0), *Op1 = Cmp.getOperand(1); | ||||||||
1434 | Value *A, *B; | ||||||||
1435 | ConstantInt *CI, *CI2; // I = icmp ugt (add (add A, B), CI2), CI | ||||||||
1436 | if (Pred == ICmpInst::ICMP_UGT && match(Op1, m_ConstantInt(CI)) && | ||||||||
1437 | match(Op0, m_Add(m_Add(m_Value(A), m_Value(B)), m_ConstantInt(CI2)))) | ||||||||
1438 | if (Instruction *Res = processUGT_ADDCST_ADD(Cmp, A, B, CI2, CI, *this)) | ||||||||
1439 | return Res; | ||||||||
1440 | |||||||||
1441 | // icmp(phi(C1, C2, ...), C) -> phi(icmp(C1, C), icmp(C2, C), ...). | ||||||||
1442 | Constant *C = dyn_cast<Constant>(Op1); | ||||||||
1443 | if (!C || C->canTrap()) | ||||||||
1444 | return nullptr; | ||||||||
1445 | |||||||||
1446 | if (auto *Phi = dyn_cast<PHINode>(Op0)) | ||||||||
1447 | if (all_of(Phi->operands(), [](Value *V) { return isa<Constant>(V); })) { | ||||||||
1448 | Type *Ty = Cmp.getType(); | ||||||||
1449 | Builder.SetInsertPoint(Phi); | ||||||||
1450 | PHINode *NewPhi = | ||||||||
1451 | Builder.CreatePHI(Ty, Phi->getNumOperands()); | ||||||||
1452 | for (BasicBlock *Predecessor : predecessors(Phi->getParent())) { | ||||||||
1453 | auto *Input = | ||||||||
1454 | cast<Constant>(Phi->getIncomingValueForBlock(Predecessor)); | ||||||||
1455 | auto *BoolInput = ConstantExpr::getCompare(Pred, Input, C); | ||||||||
1456 | NewPhi->addIncoming(BoolInput, Predecessor); | ||||||||
1457 | } | ||||||||
1458 | NewPhi->takeName(&Cmp); | ||||||||
1459 | return replaceInstUsesWith(Cmp, NewPhi); | ||||||||
1460 | } | ||||||||
1461 | |||||||||
1462 | return nullptr; | ||||||||
1463 | } | ||||||||
1464 | |||||||||
1465 | /// Canonicalize icmp instructions based on dominating conditions. | ||||||||
1466 | Instruction *InstCombinerImpl::foldICmpWithDominatingICmp(ICmpInst &Cmp) { | ||||||||
1467 | // This is a cheap/incomplete check for dominance - just match a single | ||||||||
1468 | // predecessor with a conditional branch. | ||||||||
1469 | BasicBlock *CmpBB = Cmp.getParent(); | ||||||||
1470 | BasicBlock *DomBB = CmpBB->getSinglePredecessor(); | ||||||||
1471 | if (!DomBB) | ||||||||
1472 | return nullptr; | ||||||||
1473 | |||||||||
1474 | Value *DomCond; | ||||||||
1475 | BasicBlock *TrueBB, *FalseBB; | ||||||||
1476 | if (!match(DomBB->getTerminator(), m_Br(m_Value(DomCond), TrueBB, FalseBB))) | ||||||||
1477 | return nullptr; | ||||||||
1478 | |||||||||
1479 | assert((TrueBB == CmpBB || FalseBB == CmpBB) &&(static_cast <bool> ((TrueBB == CmpBB || FalseBB == CmpBB ) && "Predecessor block does not point to successor?" ) ? void (0) : __assert_fail ("(TrueBB == CmpBB || FalseBB == CmpBB) && \"Predecessor block does not point to successor?\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1480, __extension__ __PRETTY_FUNCTION__)) | ||||||||
1480 | "Predecessor block does not point to successor?")(static_cast <bool> ((TrueBB == CmpBB || FalseBB == CmpBB ) && "Predecessor block does not point to successor?" ) ? void (0) : __assert_fail ("(TrueBB == CmpBB || FalseBB == CmpBB) && \"Predecessor block does not point to successor?\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1480, __extension__ __PRETTY_FUNCTION__)); | ||||||||
1481 | |||||||||
1482 | // The branch should get simplified. Don't bother simplifying this condition. | ||||||||
1483 | if (TrueBB == FalseBB) | ||||||||
1484 | return nullptr; | ||||||||
1485 | |||||||||
1486 | // Try to simplify this compare to T/F based on the dominating condition. | ||||||||
1487 | Optional<bool> Imp = isImpliedCondition(DomCond, &Cmp, DL, TrueBB == CmpBB); | ||||||||
1488 | if (Imp) | ||||||||
1489 | return replaceInstUsesWith(Cmp, ConstantInt::get(Cmp.getType(), *Imp)); | ||||||||
1490 | |||||||||
1491 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
1492 | Value *X = Cmp.getOperand(0), *Y = Cmp.getOperand(1); | ||||||||
1493 | ICmpInst::Predicate DomPred; | ||||||||
1494 | const APInt *C, *DomC; | ||||||||
1495 | if (match(DomCond, m_ICmp(DomPred, m_Specific(X), m_APInt(DomC))) && | ||||||||
1496 | match(Y, m_APInt(C))) { | ||||||||
1497 | // We have 2 compares of a variable with constants. Calculate the constant | ||||||||
1498 | // ranges of those compares to see if we can transform the 2nd compare: | ||||||||
1499 | // DomBB: | ||||||||
1500 | // DomCond = icmp DomPred X, DomC | ||||||||
1501 | // br DomCond, CmpBB, FalseBB | ||||||||
1502 | // CmpBB: | ||||||||
1503 | // Cmp = icmp Pred X, C | ||||||||
1504 | ConstantRange CR = ConstantRange::makeExactICmpRegion(Pred, *C); | ||||||||
1505 | ConstantRange DominatingCR = | ||||||||
1506 | (CmpBB == TrueBB) ? ConstantRange::makeExactICmpRegion(DomPred, *DomC) | ||||||||
1507 | : ConstantRange::makeExactICmpRegion( | ||||||||
1508 | CmpInst::getInversePredicate(DomPred), *DomC); | ||||||||
1509 | ConstantRange Intersection = DominatingCR.intersectWith(CR); | ||||||||
1510 | ConstantRange Difference = DominatingCR.difference(CR); | ||||||||
1511 | if (Intersection.isEmptySet()) | ||||||||
1512 | return replaceInstUsesWith(Cmp, Builder.getFalse()); | ||||||||
1513 | if (Difference.isEmptySet()) | ||||||||
1514 | return replaceInstUsesWith(Cmp, Builder.getTrue()); | ||||||||
1515 | |||||||||
1516 | // Canonicalizing a sign bit comparison that gets used in a branch, | ||||||||
1517 | // pessimizes codegen by generating branch on zero instruction instead | ||||||||
1518 | // of a test and branch. So we avoid canonicalizing in such situations | ||||||||
1519 | // because test and branch instruction has better branch displacement | ||||||||
1520 | // than compare and branch instruction. | ||||||||
1521 | bool UnusedBit; | ||||||||
1522 | bool IsSignBit = isSignBitCheck(Pred, *C, UnusedBit); | ||||||||
1523 | if (Cmp.isEquality() || (IsSignBit && hasBranchUse(Cmp))) | ||||||||
1524 | return nullptr; | ||||||||
1525 | |||||||||
1526 | // Avoid an infinite loop with min/max canonicalization. | ||||||||
1527 | // TODO: This will be unnecessary if we canonicalize to min/max intrinsics. | ||||||||
1528 | if (Cmp.hasOneUse() && | ||||||||
1529 | match(Cmp.user_back(), m_MaxOrMin(m_Value(), m_Value()))) | ||||||||
1530 | return nullptr; | ||||||||
1531 | |||||||||
1532 | if (const APInt *EqC = Intersection.getSingleElement()) | ||||||||
1533 | return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(*EqC)); | ||||||||
1534 | if (const APInt *NeC = Difference.getSingleElement()) | ||||||||
1535 | return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(*NeC)); | ||||||||
1536 | } | ||||||||
1537 | |||||||||
1538 | return nullptr; | ||||||||
1539 | } | ||||||||
1540 | |||||||||
1541 | /// Fold icmp (trunc X, Y), C. | ||||||||
1542 | Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp, | ||||||||
1543 | TruncInst *Trunc, | ||||||||
1544 | const APInt &C) { | ||||||||
1545 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
1546 | Value *X = Trunc->getOperand(0); | ||||||||
1547 | if (C.isOneValue() && C.getBitWidth() > 1) { | ||||||||
1548 | // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1 | ||||||||
1549 | Value *V = nullptr; | ||||||||
1550 | if (Pred == ICmpInst::ICMP_SLT && match(X, m_Signum(m_Value(V)))) | ||||||||
1551 | return new ICmpInst(ICmpInst::ICMP_SLT, V, | ||||||||
1552 | ConstantInt::get(V->getType(), 1)); | ||||||||
1553 | } | ||||||||
1554 | |||||||||
1555 | unsigned DstBits = Trunc->getType()->getScalarSizeInBits(), | ||||||||
1556 | SrcBits = X->getType()->getScalarSizeInBits(); | ||||||||
1557 | if (Cmp.isEquality() && Trunc->hasOneUse()) { | ||||||||
1558 | // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all | ||||||||
1559 | // of the high bits truncated out of x are known. | ||||||||
1560 | KnownBits Known = computeKnownBits(X, 0, &Cmp); | ||||||||
1561 | |||||||||
1562 | // If all the high bits are known, we can do this xform. | ||||||||
1563 | if ((Known.Zero | Known.One).countLeadingOnes() >= SrcBits - DstBits) { | ||||||||
1564 | // Pull in the high bits from known-ones set. | ||||||||
1565 | APInt NewRHS = C.zext(SrcBits); | ||||||||
1566 | NewRHS |= Known.One & APInt::getHighBitsSet(SrcBits, SrcBits - DstBits); | ||||||||
1567 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), NewRHS)); | ||||||||
1568 | } | ||||||||
1569 | } | ||||||||
1570 | |||||||||
1571 | // Look through truncated right-shift of the sign-bit for a sign-bit check: | ||||||||
1572 | // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] < 0 --> ShOp < 0 | ||||||||
1573 | // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] > -1 --> ShOp > -1 | ||||||||
1574 | Value *ShOp; | ||||||||
1575 | const APInt *ShAmtC; | ||||||||
1576 | bool TrueIfSigned; | ||||||||
1577 | if (isSignBitCheck(Pred, C, TrueIfSigned) && | ||||||||
1578 | match(X, m_Shr(m_Value(ShOp), m_APInt(ShAmtC))) && | ||||||||
1579 | DstBits == SrcBits - ShAmtC->getZExtValue()) { | ||||||||
1580 | return TrueIfSigned | ||||||||
1581 | ? new ICmpInst(ICmpInst::ICMP_SLT, ShOp, | ||||||||
1582 | ConstantInt::getNullValue(X->getType())) | ||||||||
1583 | : new ICmpInst(ICmpInst::ICMP_SGT, ShOp, | ||||||||
1584 | ConstantInt::getAllOnesValue(X->getType())); | ||||||||
1585 | } | ||||||||
1586 | |||||||||
1587 | return nullptr; | ||||||||
1588 | } | ||||||||
1589 | |||||||||
1590 | /// Fold icmp (xor X, Y), C. | ||||||||
1591 | Instruction *InstCombinerImpl::foldICmpXorConstant(ICmpInst &Cmp, | ||||||||
1592 | BinaryOperator *Xor, | ||||||||
1593 | const APInt &C) { | ||||||||
1594 | Value *X = Xor->getOperand(0); | ||||||||
1595 | Value *Y = Xor->getOperand(1); | ||||||||
1596 | const APInt *XorC; | ||||||||
1597 | if (!match(Y, m_APInt(XorC))) | ||||||||
1598 | return nullptr; | ||||||||
1599 | |||||||||
1600 | // If this is a comparison that tests the signbit (X < 0) or (x > -1), | ||||||||
1601 | // fold the xor. | ||||||||
1602 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
1603 | bool TrueIfSigned = false; | ||||||||
1604 | if (isSignBitCheck(Cmp.getPredicate(), C, TrueIfSigned)) { | ||||||||
1605 | |||||||||
1606 | // If the sign bit of the XorCst is not set, there is no change to | ||||||||
1607 | // the operation, just stop using the Xor. | ||||||||
1608 | if (!XorC->isNegative()) | ||||||||
1609 | return replaceOperand(Cmp, 0, X); | ||||||||
1610 | |||||||||
1611 | // Emit the opposite comparison. | ||||||||
1612 | if (TrueIfSigned) | ||||||||
1613 | return new ICmpInst(ICmpInst::ICMP_SGT, X, | ||||||||
1614 | ConstantInt::getAllOnesValue(X->getType())); | ||||||||
1615 | else | ||||||||
1616 | return new ICmpInst(ICmpInst::ICMP_SLT, X, | ||||||||
1617 | ConstantInt::getNullValue(X->getType())); | ||||||||
1618 | } | ||||||||
1619 | |||||||||
1620 | if (Xor->hasOneUse()) { | ||||||||
1621 | // (icmp u/s (xor X SignMask), C) -> (icmp s/u X, (xor C SignMask)) | ||||||||
1622 | if (!Cmp.isEquality() && XorC->isSignMask()) { | ||||||||
1623 | Pred = Cmp.getFlippedSignednessPredicate(); | ||||||||
1624 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC)); | ||||||||
1625 | } | ||||||||
1626 | |||||||||
1627 | // (icmp u/s (xor X ~SignMask), C) -> (icmp s/u X, (xor C ~SignMask)) | ||||||||
1628 | if (!Cmp.isEquality() && XorC->isMaxSignedValue()) { | ||||||||
1629 | Pred = Cmp.getFlippedSignednessPredicate(); | ||||||||
1630 | Pred = Cmp.getSwappedPredicate(Pred); | ||||||||
1631 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC)); | ||||||||
1632 | } | ||||||||
1633 | } | ||||||||
1634 | |||||||||
1635 | // Mask constant magic can eliminate an 'xor' with unsigned compares. | ||||||||
1636 | if (Pred == ICmpInst::ICMP_UGT) { | ||||||||
1637 | // (xor X, ~C) >u C --> X <u ~C (when C+1 is a power of 2) | ||||||||
1638 | if (*XorC == ~C && (C + 1).isPowerOf2()) | ||||||||
1639 | return new ICmpInst(ICmpInst::ICMP_ULT, X, Y); | ||||||||
1640 | // (xor X, C) >u C --> X >u C (when C+1 is a power of 2) | ||||||||
1641 | if (*XorC == C && (C + 1).isPowerOf2()) | ||||||||
1642 | return new ICmpInst(ICmpInst::ICMP_UGT, X, Y); | ||||||||
1643 | } | ||||||||
1644 | if (Pred == ICmpInst::ICMP_ULT) { | ||||||||
1645 | // (xor X, -C) <u C --> X >u ~C (when C is a power of 2) | ||||||||
1646 | if (*XorC == -C && C.isPowerOf2()) | ||||||||
1647 | return new ICmpInst(ICmpInst::ICMP_UGT, X, | ||||||||
1648 | ConstantInt::get(X->getType(), ~C)); | ||||||||
1649 | // (xor X, C) <u C --> X >u ~C (when -C is a power of 2) | ||||||||
1650 | if (*XorC == C && (-C).isPowerOf2()) | ||||||||
1651 | return new ICmpInst(ICmpInst::ICMP_UGT, X, | ||||||||
1652 | ConstantInt::get(X->getType(), ~C)); | ||||||||
1653 | } | ||||||||
1654 | return nullptr; | ||||||||
1655 | } | ||||||||
1656 | |||||||||
1657 | /// Fold icmp (and (sh X, Y), C2), C1. | ||||||||
1658 | Instruction *InstCombinerImpl::foldICmpAndShift(ICmpInst &Cmp, | ||||||||
1659 | BinaryOperator *And, | ||||||||
1660 | const APInt &C1, | ||||||||
1661 | const APInt &C2) { | ||||||||
1662 | BinaryOperator *Shift = dyn_cast<BinaryOperator>(And->getOperand(0)); | ||||||||
1663 | if (!Shift || !Shift->isShift()) | ||||||||
1664 | return nullptr; | ||||||||
1665 | |||||||||
1666 | // If this is: (X >> C3) & C2 != C1 (where any shift and any compare could | ||||||||
1667 | // exist), turn it into (X & (C2 << C3)) != (C1 << C3). This happens a LOT in | ||||||||
1668 | // code produced by the clang front-end, for bitfield access. | ||||||||
1669 | // This seemingly simple opportunity to fold away a shift turns out to be | ||||||||
1670 | // rather complicated. See PR17827 for details. | ||||||||
1671 | unsigned ShiftOpcode = Shift->getOpcode(); | ||||||||
1672 | bool IsShl = ShiftOpcode == Instruction::Shl; | ||||||||
1673 | const APInt *C3; | ||||||||
1674 | if (match(Shift->getOperand(1), m_APInt(C3))) { | ||||||||
1675 | APInt NewAndCst, NewCmpCst; | ||||||||
1676 | bool AnyCmpCstBitsShiftedOut; | ||||||||
1677 | if (ShiftOpcode == Instruction::Shl) { | ||||||||
1678 | // For a left shift, we can fold if the comparison is not signed. We can | ||||||||
1679 | // also fold a signed comparison if the mask value and comparison value | ||||||||
1680 | // are not negative. These constraints may not be obvious, but we can | ||||||||
1681 | // prove that they are correct using an SMT solver. | ||||||||
1682 | if (Cmp.isSigned() && (C2.isNegative() || C1.isNegative())) | ||||||||
1683 | return nullptr; | ||||||||
1684 | |||||||||
1685 | NewCmpCst = C1.lshr(*C3); | ||||||||
1686 | NewAndCst = C2.lshr(*C3); | ||||||||
1687 | AnyCmpCstBitsShiftedOut = NewCmpCst.shl(*C3) != C1; | ||||||||
1688 | } else if (ShiftOpcode == Instruction::LShr) { | ||||||||
1689 | // For a logical right shift, we can fold if the comparison is not signed. | ||||||||
1690 | // We can also fold a signed comparison if the shifted mask value and the | ||||||||
1691 | // shifted comparison value are not negative. These constraints may not be | ||||||||
1692 | // obvious, but we can prove that they are correct using an SMT solver. | ||||||||
1693 | NewCmpCst = C1.shl(*C3); | ||||||||
1694 | NewAndCst = C2.shl(*C3); | ||||||||
1695 | AnyCmpCstBitsShiftedOut = NewCmpCst.lshr(*C3) != C1; | ||||||||
1696 | if (Cmp.isSigned() && (NewAndCst.isNegative() || NewCmpCst.isNegative())) | ||||||||
1697 | return nullptr; | ||||||||
1698 | } else { | ||||||||
1699 | // For an arithmetic shift, check that both constants don't use (in a | ||||||||
1700 | // signed sense) the top bits being shifted out. | ||||||||
1701 | assert(ShiftOpcode == Instruction::AShr && "Unknown shift opcode")(static_cast <bool> (ShiftOpcode == Instruction::AShr && "Unknown shift opcode") ? void (0) : __assert_fail ("ShiftOpcode == Instruction::AShr && \"Unknown shift opcode\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1701, __extension__ __PRETTY_FUNCTION__)); | ||||||||
1702 | NewCmpCst = C1.shl(*C3); | ||||||||
1703 | NewAndCst = C2.shl(*C3); | ||||||||
1704 | AnyCmpCstBitsShiftedOut = NewCmpCst.ashr(*C3) != C1; | ||||||||
1705 | if (NewAndCst.ashr(*C3) != C2) | ||||||||
1706 | return nullptr; | ||||||||
1707 | } | ||||||||
1708 | |||||||||
1709 | if (AnyCmpCstBitsShiftedOut) { | ||||||||
1710 | // If we shifted bits out, the fold is not going to work out. As a | ||||||||
1711 | // special case, check to see if this means that the result is always | ||||||||
1712 | // true or false now. | ||||||||
1713 | if (Cmp.getPredicate() == ICmpInst::ICMP_EQ) | ||||||||
1714 | return replaceInstUsesWith(Cmp, ConstantInt::getFalse(Cmp.getType())); | ||||||||
1715 | if (Cmp.getPredicate() == ICmpInst::ICMP_NE) | ||||||||
1716 | return replaceInstUsesWith(Cmp, ConstantInt::getTrue(Cmp.getType())); | ||||||||
1717 | } else { | ||||||||
1718 | Value *NewAnd = Builder.CreateAnd( | ||||||||
1719 | Shift->getOperand(0), ConstantInt::get(And->getType(), NewAndCst)); | ||||||||
1720 | return new ICmpInst(Cmp.getPredicate(), | ||||||||
1721 | NewAnd, ConstantInt::get(And->getType(), NewCmpCst)); | ||||||||
1722 | } | ||||||||
1723 | } | ||||||||
1724 | |||||||||
1725 | // Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is | ||||||||
1726 | // preferable because it allows the C2 << Y expression to be hoisted out of a | ||||||||
1727 | // loop if Y is invariant and X is not. | ||||||||
1728 | if (Shift->hasOneUse() && C1.isNullValue() && Cmp.isEquality() && | ||||||||
1729 | !Shift->isArithmeticShift() && !isa<Constant>(Shift->getOperand(0))) { | ||||||||
1730 | // Compute C2 << Y. | ||||||||
1731 | Value *NewShift = | ||||||||
1732 | IsShl ? Builder.CreateLShr(And->getOperand(1), Shift->getOperand(1)) | ||||||||
1733 | : Builder.CreateShl(And->getOperand(1), Shift->getOperand(1)); | ||||||||
1734 | |||||||||
1735 | // Compute X & (C2 << Y). | ||||||||
1736 | Value *NewAnd = Builder.CreateAnd(Shift->getOperand(0), NewShift); | ||||||||
1737 | return replaceOperand(Cmp, 0, NewAnd); | ||||||||
1738 | } | ||||||||
1739 | |||||||||
1740 | return nullptr; | ||||||||
1741 | } | ||||||||
1742 | |||||||||
1743 | /// Fold icmp (and X, C2), C1. | ||||||||
1744 | Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp, | ||||||||
1745 | BinaryOperator *And, | ||||||||
1746 | const APInt &C1) { | ||||||||
1747 | bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE; | ||||||||
1748 | |||||||||
1749 | // For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1 | ||||||||
1750 | // TODO: We canonicalize to the longer form for scalars because we have | ||||||||
1751 | // better analysis/folds for icmp, and codegen may be better with icmp. | ||||||||
1752 | if (isICMP_NE && Cmp.getType()->isVectorTy() && C1.isNullValue() && | ||||||||
1753 | match(And->getOperand(1), m_One())) | ||||||||
1754 | return new TruncInst(And->getOperand(0), Cmp.getType()); | ||||||||
1755 | |||||||||
1756 | const APInt *C2; | ||||||||
1757 | Value *X; | ||||||||
1758 | if (!match(And, m_And(m_Value(X), m_APInt(C2)))) | ||||||||
1759 | return nullptr; | ||||||||
1760 | |||||||||
1761 | // Don't perform the following transforms if the AND has multiple uses | ||||||||
1762 | if (!And->hasOneUse()) | ||||||||
1763 | return nullptr; | ||||||||
1764 | |||||||||
1765 | if (Cmp.isEquality() && C1.isNullValue()) { | ||||||||
1766 | // Restrict this fold to single-use 'and' (PR10267). | ||||||||
1767 | // Replace (and X, (1 << size(X)-1) != 0) with X s< 0 | ||||||||
1768 | if (C2->isSignMask()) { | ||||||||
1769 | Constant *Zero = Constant::getNullValue(X->getType()); | ||||||||
1770 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE; | ||||||||
1771 | return new ICmpInst(NewPred, X, Zero); | ||||||||
1772 | } | ||||||||
1773 | |||||||||
1774 | // Restrict this fold only for single-use 'and' (PR10267). | ||||||||
1775 | // ((%x & C) == 0) --> %x u< (-C) iff (-C) is power of two. | ||||||||
1776 | if ((~(*C2) + 1).isPowerOf2()) { | ||||||||
1777 | Constant *NegBOC = | ||||||||
1778 | ConstantExpr::getNeg(cast<Constant>(And->getOperand(1))); | ||||||||
1779 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; | ||||||||
1780 | return new ICmpInst(NewPred, X, NegBOC); | ||||||||
1781 | } | ||||||||
1782 | } | ||||||||
1783 | |||||||||
1784 | // If the LHS is an 'and' of a truncate and we can widen the and/compare to | ||||||||
1785 | // the input width without changing the value produced, eliminate the cast: | ||||||||
1786 | // | ||||||||
1787 | // icmp (and (trunc W), C2), C1 -> icmp (and W, C2'), C1' | ||||||||
1788 | // | ||||||||
1789 | // We can do this transformation if the constants do not have their sign bits | ||||||||
1790 | // set or if it is an equality comparison. Extending a relational comparison | ||||||||
1791 | // when we're checking the sign bit would not work. | ||||||||
1792 | Value *W; | ||||||||
1793 | if (match(And->getOperand(0), m_OneUse(m_Trunc(m_Value(W)))) && | ||||||||
1794 | (Cmp.isEquality() || (!C1.isNegative() && !C2->isNegative()))) { | ||||||||
1795 | // TODO: Is this a good transform for vectors? Wider types may reduce | ||||||||
1796 | // throughput. Should this transform be limited (even for scalars) by using | ||||||||
1797 | // shouldChangeType()? | ||||||||
1798 | if (!Cmp.getType()->isVectorTy()) { | ||||||||
1799 | Type *WideType = W->getType(); | ||||||||
1800 | unsigned WideScalarBits = WideType->getScalarSizeInBits(); | ||||||||
1801 | Constant *ZextC1 = ConstantInt::get(WideType, C1.zext(WideScalarBits)); | ||||||||
1802 | Constant *ZextC2 = ConstantInt::get(WideType, C2->zext(WideScalarBits)); | ||||||||
1803 | Value *NewAnd = Builder.CreateAnd(W, ZextC2, And->getName()); | ||||||||
1804 | return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1); | ||||||||
1805 | } | ||||||||
1806 | } | ||||||||
1807 | |||||||||
1808 | if (Instruction *I = foldICmpAndShift(Cmp, And, C1, *C2)) | ||||||||
1809 | return I; | ||||||||
1810 | |||||||||
1811 | // (icmp pred (and (or (lshr A, B), A), 1), 0) --> | ||||||||
1812 | // (icmp pred (and A, (or (shl 1, B), 1), 0)) | ||||||||
1813 | // | ||||||||
1814 | // iff pred isn't signed | ||||||||
1815 | if (!Cmp.isSigned() && C1.isNullValue() && And->getOperand(0)->hasOneUse() && | ||||||||
1816 | match(And->getOperand(1), m_One())) { | ||||||||
1817 | Constant *One = cast<Constant>(And->getOperand(1)); | ||||||||
1818 | Value *Or = And->getOperand(0); | ||||||||
1819 | Value *A, *B, *LShr; | ||||||||
1820 | if (match(Or, m_Or(m_Value(LShr), m_Value(A))) && | ||||||||
1821 | match(LShr, m_LShr(m_Specific(A), m_Value(B)))) { | ||||||||
1822 | unsigned UsesRemoved = 0; | ||||||||
1823 | if (And->hasOneUse()) | ||||||||
1824 | ++UsesRemoved; | ||||||||
1825 | if (Or->hasOneUse()) | ||||||||
1826 | ++UsesRemoved; | ||||||||
1827 | if (LShr->hasOneUse()) | ||||||||
1828 | ++UsesRemoved; | ||||||||
1829 | |||||||||
1830 | // Compute A & ((1 << B) | 1) | ||||||||
1831 | Value *NewOr = nullptr; | ||||||||
1832 | if (auto *C = dyn_cast<Constant>(B)) { | ||||||||
1833 | if (UsesRemoved >= 1) | ||||||||
1834 | NewOr = ConstantExpr::getOr(ConstantExpr::getNUWShl(One, C), One); | ||||||||
1835 | } else { | ||||||||
1836 | if (UsesRemoved >= 3) | ||||||||
1837 | NewOr = Builder.CreateOr(Builder.CreateShl(One, B, LShr->getName(), | ||||||||
1838 | /*HasNUW=*/true), | ||||||||
1839 | One, Or->getName()); | ||||||||
1840 | } | ||||||||
1841 | if (NewOr) { | ||||||||
1842 | Value *NewAnd = Builder.CreateAnd(A, NewOr, And->getName()); | ||||||||
1843 | return replaceOperand(Cmp, 0, NewAnd); | ||||||||
1844 | } | ||||||||
1845 | } | ||||||||
1846 | } | ||||||||
1847 | |||||||||
1848 | return nullptr; | ||||||||
1849 | } | ||||||||
1850 | |||||||||
1851 | /// Fold icmp (and X, Y), C. | ||||||||
1852 | Instruction *InstCombinerImpl::foldICmpAndConstant(ICmpInst &Cmp, | ||||||||
1853 | BinaryOperator *And, | ||||||||
1854 | const APInt &C) { | ||||||||
1855 | if (Instruction *I = foldICmpAndConstConst(Cmp, And, C)) | ||||||||
1856 | return I; | ||||||||
1857 | |||||||||
1858 | const ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
1859 | bool TrueIfNeg; | ||||||||
1860 | if (isSignBitCheck(Pred, C, TrueIfNeg)) { | ||||||||
1861 | // ((X - 1) & ~X) < 0 --> X == 0 | ||||||||
1862 | // ((X - 1) & ~X) >= 0 --> X != 0 | ||||||||
1863 | Value *X; | ||||||||
1864 | if (match(And->getOperand(0), m_Add(m_Value(X), m_AllOnes())) && | ||||||||
1865 | match(And->getOperand(1), m_Not(m_Specific(X)))) { | ||||||||
1866 | auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE; | ||||||||
1867 | return new ICmpInst(NewPred, X, ConstantInt::getNullValue(X->getType())); | ||||||||
1868 | } | ||||||||
1869 | } | ||||||||
1870 | |||||||||
1871 | // TODO: These all require that Y is constant too, so refactor with the above. | ||||||||
1872 | |||||||||
1873 | // Try to optimize things like "A[i] & 42 == 0" to index computations. | ||||||||
1874 | Value *X = And->getOperand(0); | ||||||||
1875 | Value *Y = And->getOperand(1); | ||||||||
1876 | if (auto *LI = dyn_cast<LoadInst>(X)) | ||||||||
1877 | if (auto *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0))) | ||||||||
1878 | if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) | ||||||||
1879 | if (GV->isConstant() && GV->hasDefinitiveInitializer() && | ||||||||
1880 | !LI->isVolatile() && isa<ConstantInt>(Y)) { | ||||||||
1881 | ConstantInt *C2 = cast<ConstantInt>(Y); | ||||||||
1882 | if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, Cmp, C2)) | ||||||||
1883 | return Res; | ||||||||
1884 | } | ||||||||
1885 | |||||||||
1886 | if (!Cmp.isEquality()) | ||||||||
1887 | return nullptr; | ||||||||
1888 | |||||||||
1889 | // X & -C == -C -> X > u ~C | ||||||||
1890 | // X & -C != -C -> X <= u ~C | ||||||||
1891 | // iff C is a power of 2 | ||||||||
1892 | if (Cmp.getOperand(1) == Y && (-C).isPowerOf2()) { | ||||||||
1893 | auto NewPred = | ||||||||
1894 | Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGT : CmpInst::ICMP_ULE; | ||||||||
1895 | return new ICmpInst(NewPred, X, SubOne(cast<Constant>(Cmp.getOperand(1)))); | ||||||||
1896 | } | ||||||||
1897 | |||||||||
1898 | // (X & C2) == 0 -> (trunc X) >= 0 | ||||||||
1899 | // (X & C2) != 0 -> (trunc X) < 0 | ||||||||
1900 | // iff C2 is a power of 2 and it masks the sign bit of a legal integer type. | ||||||||
1901 | const APInt *C2; | ||||||||
1902 | if (And->hasOneUse() && C.isNullValue() && match(Y, m_APInt(C2))) { | ||||||||
1903 | int32_t ExactLogBase2 = C2->exactLogBase2(); | ||||||||
1904 | if (ExactLogBase2 != -1 && DL.isLegalInteger(ExactLogBase2 + 1)) { | ||||||||
1905 | Type *NTy = IntegerType::get(Cmp.getContext(), ExactLogBase2 + 1); | ||||||||
1906 | if (auto *AndVTy = dyn_cast<VectorType>(And->getType())) | ||||||||
1907 | NTy = VectorType::get(NTy, AndVTy->getElementCount()); | ||||||||
1908 | Value *Trunc = Builder.CreateTrunc(X, NTy); | ||||||||
1909 | auto NewPred = | ||||||||
1910 | Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_SGE : CmpInst::ICMP_SLT; | ||||||||
1911 | return new ICmpInst(NewPred, Trunc, Constant::getNullValue(NTy)); | ||||||||
1912 | } | ||||||||
1913 | } | ||||||||
1914 | |||||||||
1915 | return nullptr; | ||||||||
1916 | } | ||||||||
1917 | |||||||||
1918 | /// Fold icmp (or X, Y), C. | ||||||||
1919 | Instruction *InstCombinerImpl::foldICmpOrConstant(ICmpInst &Cmp, | ||||||||
1920 | BinaryOperator *Or, | ||||||||
1921 | const APInt &C) { | ||||||||
1922 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
1923 | if (C.isOneValue()) { | ||||||||
1924 | // icmp slt signum(V) 1 --> icmp slt V, 1 | ||||||||
1925 | Value *V = nullptr; | ||||||||
1926 | if (Pred == ICmpInst::ICMP_SLT && match(Or, m_Signum(m_Value(V)))) | ||||||||
1927 | return new ICmpInst(ICmpInst::ICMP_SLT, V, | ||||||||
1928 | ConstantInt::get(V->getType(), 1)); | ||||||||
1929 | } | ||||||||
1930 | |||||||||
1931 | Value *OrOp0 = Or->getOperand(0), *OrOp1 = Or->getOperand(1); | ||||||||
1932 | const APInt *MaskC; | ||||||||
1933 | if (match(OrOp1, m_APInt(MaskC)) && Cmp.isEquality()) { | ||||||||
1934 | if (*MaskC == C && (C + 1).isPowerOf2()) { | ||||||||
1935 | // X | C == C --> X <=u C | ||||||||
1936 | // X | C != C --> X >u C | ||||||||
1937 | // iff C+1 is a power of 2 (C is a bitmask of the low bits) | ||||||||
1938 | Pred = (Pred == CmpInst::ICMP_EQ) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT; | ||||||||
1939 | return new ICmpInst(Pred, OrOp0, OrOp1); | ||||||||
1940 | } | ||||||||
1941 | |||||||||
1942 | // More general: canonicalize 'equality with set bits mask' to | ||||||||
1943 | // 'equality with clear bits mask'. | ||||||||
1944 | // (X | MaskC) == C --> (X & ~MaskC) == C ^ MaskC | ||||||||
1945 | // (X | MaskC) != C --> (X & ~MaskC) != C ^ MaskC | ||||||||
1946 | if (Or->hasOneUse()) { | ||||||||
1947 | Value *And = Builder.CreateAnd(OrOp0, ~(*MaskC)); | ||||||||
1948 | Constant *NewC = ConstantInt::get(Or->getType(), C ^ (*MaskC)); | ||||||||
1949 | return new ICmpInst(Pred, And, NewC); | ||||||||
1950 | } | ||||||||
1951 | } | ||||||||
1952 | |||||||||
1953 | if (!Cmp.isEquality() || !C.isNullValue() || !Or->hasOneUse()) | ||||||||
1954 | return nullptr; | ||||||||
1955 | |||||||||
1956 | Value *P, *Q; | ||||||||
1957 | if (match(Or, m_Or(m_PtrToInt(m_Value(P)), m_PtrToInt(m_Value(Q))))) { | ||||||||
1958 | // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0 | ||||||||
1959 | // -> and (icmp eq P, null), (icmp eq Q, null). | ||||||||
1960 | Value *CmpP = | ||||||||
1961 | Builder.CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType())); | ||||||||
1962 | Value *CmpQ = | ||||||||
1963 | Builder.CreateICmp(Pred, Q, ConstantInt::getNullValue(Q->getType())); | ||||||||
1964 | auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; | ||||||||
1965 | return BinaryOperator::Create(BOpc, CmpP, CmpQ); | ||||||||
1966 | } | ||||||||
1967 | |||||||||
1968 | // Are we using xors to bitwise check for a pair of (in)equalities? Convert to | ||||||||
1969 | // a shorter form that has more potential to be folded even further. | ||||||||
1970 | Value *X1, *X2, *X3, *X4; | ||||||||
1971 | if (match(OrOp0, m_OneUse(m_Xor(m_Value(X1), m_Value(X2)))) && | ||||||||
1972 | match(OrOp1, m_OneUse(m_Xor(m_Value(X3), m_Value(X4))))) { | ||||||||
1973 | // ((X1 ^ X2) || (X3 ^ X4)) == 0 --> (X1 == X2) && (X3 == X4) | ||||||||
1974 | // ((X1 ^ X2) || (X3 ^ X4)) != 0 --> (X1 != X2) || (X3 != X4) | ||||||||
1975 | Value *Cmp12 = Builder.CreateICmp(Pred, X1, X2); | ||||||||
1976 | Value *Cmp34 = Builder.CreateICmp(Pred, X3, X4); | ||||||||
1977 | auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; | ||||||||
1978 | return BinaryOperator::Create(BOpc, Cmp12, Cmp34); | ||||||||
1979 | } | ||||||||
1980 | |||||||||
1981 | return nullptr; | ||||||||
1982 | } | ||||||||
1983 | |||||||||
1984 | /// Fold icmp (mul X, Y), C. | ||||||||
1985 | Instruction *InstCombinerImpl::foldICmpMulConstant(ICmpInst &Cmp, | ||||||||
1986 | BinaryOperator *Mul, | ||||||||
1987 | const APInt &C) { | ||||||||
1988 | const APInt *MulC; | ||||||||
1989 | if (!match(Mul->getOperand(1), m_APInt(MulC))) | ||||||||
1990 | return nullptr; | ||||||||
1991 | |||||||||
1992 | // If this is a test of the sign bit and the multiply is sign-preserving with | ||||||||
1993 | // a constant operand, use the multiply LHS operand instead. | ||||||||
1994 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
1995 | if (isSignTest(Pred, C) && Mul->hasNoSignedWrap()) { | ||||||||
1996 | if (MulC->isNegative()) | ||||||||
1997 | Pred = ICmpInst::getSwappedPredicate(Pred); | ||||||||
1998 | return new ICmpInst(Pred, Mul->getOperand(0), | ||||||||
1999 | Constant::getNullValue(Mul->getType())); | ||||||||
2000 | } | ||||||||
2001 | |||||||||
2002 | // If the multiply does not wrap, try to divide the compare constant by the | ||||||||
2003 | // multiplication factor. | ||||||||
2004 | if (Cmp.isEquality() && !MulC->isNullValue()) { | ||||||||
2005 | // (mul nsw X, MulC) == C --> X == C /s MulC | ||||||||
2006 | if (Mul->hasNoSignedWrap() && C.srem(*MulC).isNullValue()) { | ||||||||
2007 | Constant *NewC = ConstantInt::get(Mul->getType(), C.sdiv(*MulC)); | ||||||||
2008 | return new ICmpInst(Pred, Mul->getOperand(0), NewC); | ||||||||
2009 | } | ||||||||
2010 | // (mul nuw X, MulC) == C --> X == C /u MulC | ||||||||
2011 | if (Mul->hasNoUnsignedWrap() && C.urem(*MulC).isNullValue()) { | ||||||||
2012 | Constant *NewC = ConstantInt::get(Mul->getType(), C.udiv(*MulC)); | ||||||||
2013 | return new ICmpInst(Pred, Mul->getOperand(0), NewC); | ||||||||
2014 | } | ||||||||
2015 | } | ||||||||
2016 | |||||||||
2017 | return nullptr; | ||||||||
2018 | } | ||||||||
2019 | |||||||||
2020 | /// Fold icmp (shl 1, Y), C. | ||||||||
2021 | static Instruction *foldICmpShlOne(ICmpInst &Cmp, Instruction *Shl, | ||||||||
2022 | const APInt &C) { | ||||||||
2023 | Value *Y; | ||||||||
2024 | if (!match(Shl, m_Shl(m_One(), m_Value(Y)))) | ||||||||
2025 | return nullptr; | ||||||||
2026 | |||||||||
2027 | Type *ShiftType = Shl->getType(); | ||||||||
2028 | unsigned TypeBits = C.getBitWidth(); | ||||||||
2029 | bool CIsPowerOf2 = C.isPowerOf2(); | ||||||||
2030 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
2031 | if (Cmp.isUnsigned()) { | ||||||||
2032 | // (1 << Y) pred C -> Y pred Log2(C) | ||||||||
2033 | if (!CIsPowerOf2) { | ||||||||
2034 | // (1 << Y) < 30 -> Y <= 4 | ||||||||
2035 | // (1 << Y) <= 30 -> Y <= 4 | ||||||||
2036 | // (1 << Y) >= 30 -> Y > 4 | ||||||||
2037 | // (1 << Y) > 30 -> Y > 4 | ||||||||
2038 | if (Pred == ICmpInst::ICMP_ULT) | ||||||||
2039 | Pred = ICmpInst::ICMP_ULE; | ||||||||
2040 | else if (Pred == ICmpInst::ICMP_UGE) | ||||||||
2041 | Pred = ICmpInst::ICMP_UGT; | ||||||||
2042 | } | ||||||||
2043 | |||||||||
2044 | // (1 << Y) >= 2147483648 -> Y >= 31 -> Y == 31 | ||||||||
2045 | // (1 << Y) < 2147483648 -> Y < 31 -> Y != 31 | ||||||||
2046 | unsigned CLog2 = C.logBase2(); | ||||||||
2047 | if (CLog2 == TypeBits - 1) { | ||||||||
2048 | if (Pred == ICmpInst::ICMP_UGE) | ||||||||
2049 | Pred = ICmpInst::ICMP_EQ; | ||||||||
2050 | else if (Pred == ICmpInst::ICMP_ULT) | ||||||||
2051 | Pred = ICmpInst::ICMP_NE; | ||||||||
2052 | } | ||||||||
2053 | return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, CLog2)); | ||||||||
2054 | } else if (Cmp.isSigned()) { | ||||||||
2055 | Constant *BitWidthMinusOne = ConstantInt::get(ShiftType, TypeBits - 1); | ||||||||
2056 | if (C.isAllOnesValue()) { | ||||||||
2057 | // (1 << Y) <= -1 -> Y == 31 | ||||||||
2058 | if (Pred == ICmpInst::ICMP_SLE) | ||||||||
2059 | return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); | ||||||||
2060 | |||||||||
2061 | // (1 << Y) > -1 -> Y != 31 | ||||||||
2062 | if (Pred == ICmpInst::ICMP_SGT) | ||||||||
2063 | return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); | ||||||||
2064 | } else if (!C) { | ||||||||
2065 | // (1 << Y) < 0 -> Y == 31 | ||||||||
2066 | // (1 << Y) <= 0 -> Y == 31 | ||||||||
2067 | if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) | ||||||||
2068 | return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); | ||||||||
2069 | |||||||||
2070 | // (1 << Y) >= 0 -> Y != 31 | ||||||||
2071 | // (1 << Y) > 0 -> Y != 31 | ||||||||
2072 | if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) | ||||||||
2073 | return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); | ||||||||
2074 | } | ||||||||
2075 | } else if (Cmp.isEquality() && CIsPowerOf2) { | ||||||||
2076 | return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, C.logBase2())); | ||||||||
2077 | } | ||||||||
2078 | |||||||||
2079 | return nullptr; | ||||||||
2080 | } | ||||||||
2081 | |||||||||
2082 | /// Fold icmp (shl X, Y), C. | ||||||||
2083 | Instruction *InstCombinerImpl::foldICmpShlConstant(ICmpInst &Cmp, | ||||||||
2084 | BinaryOperator *Shl, | ||||||||
2085 | const APInt &C) { | ||||||||
2086 | const APInt *ShiftVal; | ||||||||
2087 | if (Cmp.isEquality() && match(Shl->getOperand(0), m_APInt(ShiftVal))) | ||||||||
2088 | return foldICmpShlConstConst(Cmp, Shl->getOperand(1), C, *ShiftVal); | ||||||||
2089 | |||||||||
2090 | const APInt *ShiftAmt; | ||||||||
2091 | if (!match(Shl->getOperand(1), m_APInt(ShiftAmt))) | ||||||||
2092 | return foldICmpShlOne(Cmp, Shl, C); | ||||||||
2093 | |||||||||
2094 | // Check that the shift amount is in range. If not, don't perform undefined | ||||||||
2095 | // shifts. When the shift is visited, it will be simplified. | ||||||||
2096 | unsigned TypeBits = C.getBitWidth(); | ||||||||
2097 | if (ShiftAmt->uge(TypeBits)) | ||||||||
2098 | return nullptr; | ||||||||
2099 | |||||||||
2100 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
2101 | Value *X = Shl->getOperand(0); | ||||||||
2102 | Type *ShType = Shl->getType(); | ||||||||
2103 | |||||||||
2104 | // NSW guarantees that we are only shifting out sign bits from the high bits, | ||||||||
2105 | // so we can ASHR the compare constant without needing a mask and eliminate | ||||||||
2106 | // the shift. | ||||||||
2107 | if (Shl->hasNoSignedWrap()) { | ||||||||
2108 | if (Pred == ICmpInst::ICMP_SGT) { | ||||||||
2109 | // icmp Pred (shl nsw X, ShiftAmt), C --> icmp Pred X, (C >>s ShiftAmt) | ||||||||
2110 | APInt ShiftedC = C.ashr(*ShiftAmt); | ||||||||
2111 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||||||
2112 | } | ||||||||
2113 | if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && | ||||||||
2114 | C.ashr(*ShiftAmt).shl(*ShiftAmt) == C) { | ||||||||
2115 | APInt ShiftedC = C.ashr(*ShiftAmt); | ||||||||
2116 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||||||
2117 | } | ||||||||
2118 | if (Pred == ICmpInst::ICMP_SLT) { | ||||||||
2119 | // SLE is the same as above, but SLE is canonicalized to SLT, so convert: | ||||||||
2120 | // (X << S) <=s C is equiv to X <=s (C >> S) for all C | ||||||||
2121 | // (X << S) <s (C + 1) is equiv to X <s (C >> S) + 1 if C <s SMAX | ||||||||
2122 | // (X << S) <s C is equiv to X <s ((C - 1) >> S) + 1 if C >s SMIN | ||||||||
2123 | assert(!C.isMinSignedValue() && "Unexpected icmp slt")(static_cast <bool> (!C.isMinSignedValue() && "Unexpected icmp slt" ) ? void (0) : __assert_fail ("!C.isMinSignedValue() && \"Unexpected icmp slt\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2123, __extension__ __PRETTY_FUNCTION__)); | ||||||||
2124 | APInt ShiftedC = (C - 1).ashr(*ShiftAmt) + 1; | ||||||||
2125 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||||||
2126 | } | ||||||||
2127 | // If this is a signed comparison to 0 and the shift is sign preserving, | ||||||||
2128 | // use the shift LHS operand instead; isSignTest may change 'Pred', so only | ||||||||
2129 | // do that if we're sure to not continue on in this function. | ||||||||
2130 | if (isSignTest(Pred, C)) | ||||||||
2131 | return new ICmpInst(Pred, X, Constant::getNullValue(ShType)); | ||||||||
2132 | } | ||||||||
2133 | |||||||||
2134 | // NUW guarantees that we are only shifting out zero bits from the high bits, | ||||||||
2135 | // so we can LSHR the compare constant without needing a mask and eliminate | ||||||||
2136 | // the shift. | ||||||||
2137 | if (Shl->hasNoUnsignedWrap()) { | ||||||||
2138 | if (Pred == ICmpInst::ICMP_UGT) { | ||||||||
2139 | // icmp Pred (shl nuw X, ShiftAmt), C --> icmp Pred X, (C >>u ShiftAmt) | ||||||||
2140 | APInt ShiftedC = C.lshr(*ShiftAmt); | ||||||||
2141 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||||||
2142 | } | ||||||||
2143 | if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && | ||||||||
2144 | C.lshr(*ShiftAmt).shl(*ShiftAmt) == C) { | ||||||||
2145 | APInt ShiftedC = C.lshr(*ShiftAmt); | ||||||||
2146 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||||||
2147 | } | ||||||||
2148 | if (Pred == ICmpInst::ICMP_ULT) { | ||||||||
2149 | // ULE is the same as above, but ULE is canonicalized to ULT, so convert: | ||||||||
2150 | // (X << S) <=u C is equiv to X <=u (C >> S) for all C | ||||||||
2151 | // (X << S) <u (C + 1) is equiv to X <u (C >> S) + 1 if C <u ~0u | ||||||||
2152 | // (X << S) <u C is equiv to X <u ((C - 1) >> S) + 1 if C >u 0 | ||||||||
2153 | assert(C.ugt(0) && "ult 0 should have been eliminated")(static_cast <bool> (C.ugt(0) && "ult 0 should have been eliminated" ) ? void (0) : __assert_fail ("C.ugt(0) && \"ult 0 should have been eliminated\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2153, __extension__ __PRETTY_FUNCTION__)); | ||||||||
2154 | APInt ShiftedC = (C - 1).lshr(*ShiftAmt) + 1; | ||||||||
2155 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||||||
2156 | } | ||||||||
2157 | } | ||||||||
2158 | |||||||||
2159 | if (Cmp.isEquality() && Shl->hasOneUse()) { | ||||||||
2160 | // Strength-reduce the shift into an 'and'. | ||||||||
2161 | Constant *Mask = ConstantInt::get( | ||||||||
2162 | ShType, | ||||||||
2163 | APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt->getZExtValue())); | ||||||||
2164 | Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask"); | ||||||||
2165 | Constant *LShrC = ConstantInt::get(ShType, C.lshr(*ShiftAmt)); | ||||||||
2166 | return new ICmpInst(Pred, And, LShrC); | ||||||||
2167 | } | ||||||||
2168 | |||||||||
2169 | // Otherwise, if this is a comparison of the sign bit, simplify to and/test. | ||||||||
2170 | bool TrueIfSigned = false; | ||||||||
2171 | if (Shl->hasOneUse() && isSignBitCheck(Pred, C, TrueIfSigned)) { | ||||||||
2172 | // (X << 31) <s 0 --> (X & 1) != 0 | ||||||||
2173 | Constant *Mask = ConstantInt::get( | ||||||||
2174 | ShType, | ||||||||
2175 | APInt::getOneBitSet(TypeBits, TypeBits - ShiftAmt->getZExtValue() - 1)); | ||||||||
2176 | Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask"); | ||||||||
2177 | return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ, | ||||||||
2178 | And, Constant::getNullValue(ShType)); | ||||||||
2179 | } | ||||||||
2180 | |||||||||
2181 | // Simplify 'shl' inequality test into 'and' equality test. | ||||||||
2182 | if (Cmp.isUnsigned() && Shl->hasOneUse()) { | ||||||||
2183 | // (X l<< C2) u<=/u> C1 iff C1+1 is power of two -> X & (~C1 l>> C2) ==/!= 0 | ||||||||
2184 | if ((C + 1).isPowerOf2() && | ||||||||
2185 | (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT)) { | ||||||||
2186 | Value *And = Builder.CreateAnd(X, (~C).lshr(ShiftAmt->getZExtValue())); | ||||||||
2187 | return new ICmpInst(Pred == ICmpInst::ICMP_ULE ? ICmpInst::ICMP_EQ | ||||||||
2188 | : ICmpInst::ICMP_NE, | ||||||||
2189 | And, Constant::getNullValue(ShType)); | ||||||||
2190 | } | ||||||||
2191 | // (X l<< C2) u</u>= C1 iff C1 is power of two -> X & (-C1 l>> C2) ==/!= 0 | ||||||||
2192 | if (C.isPowerOf2() && | ||||||||
2193 | (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) { | ||||||||
2194 | Value *And = | ||||||||
2195 | Builder.CreateAnd(X, (~(C - 1)).lshr(ShiftAmt->getZExtValue())); | ||||||||
2196 | return new ICmpInst(Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_EQ | ||||||||
2197 | : ICmpInst::ICMP_NE, | ||||||||
2198 | And, Constant::getNullValue(ShType)); | ||||||||
2199 | } | ||||||||
2200 | } | ||||||||
2201 | |||||||||
2202 | // Transform (icmp pred iM (shl iM %v, N), C) | ||||||||
2203 | // -> (icmp pred i(M-N) (trunc %v iM to i(M-N)), (trunc (C>>N)) | ||||||||
2204 | // Transform the shl to a trunc if (trunc (C>>N)) has no loss and M-N. | ||||||||
2205 | // This enables us to get rid of the shift in favor of a trunc that may be | ||||||||
2206 | // free on the target. It has the additional benefit of comparing to a | ||||||||
2207 | // smaller constant that may be more target-friendly. | ||||||||
2208 | unsigned Amt = ShiftAmt->getLimitedValue(TypeBits - 1); | ||||||||
2209 | if (Shl->hasOneUse() && Amt != 0 && C.countTrailingZeros() >= Amt && | ||||||||
2210 | DL.isLegalInteger(TypeBits - Amt)) { | ||||||||
2211 | Type *TruncTy = IntegerType::get(Cmp.getContext(), TypeBits - Amt); | ||||||||
2212 | if (auto *ShVTy = dyn_cast<VectorType>(ShType)) | ||||||||
2213 | TruncTy = VectorType::get(TruncTy, ShVTy->getElementCount()); | ||||||||
2214 | Constant *NewC = | ||||||||
2215 | ConstantInt::get(TruncTy, C.ashr(*ShiftAmt).trunc(TypeBits - Amt)); | ||||||||
2216 | return new ICmpInst(Pred, Builder.CreateTrunc(X, TruncTy), NewC); | ||||||||
2217 | } | ||||||||
2218 | |||||||||
2219 | return nullptr; | ||||||||
2220 | } | ||||||||
2221 | |||||||||
2222 | /// Fold icmp ({al}shr X, Y), C. | ||||||||
2223 | Instruction *InstCombinerImpl::foldICmpShrConstant(ICmpInst &Cmp, | ||||||||
2224 | BinaryOperator *Shr, | ||||||||
2225 | const APInt &C) { | ||||||||
2226 | // An exact shr only shifts out zero bits, so: | ||||||||
2227 | // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0 | ||||||||
2228 | Value *X = Shr->getOperand(0); | ||||||||
2229 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
2230 | if (Cmp.isEquality() && Shr->isExact() && Shr->hasOneUse() && | ||||||||
2231 | C.isNullValue()) | ||||||||
2232 | return new ICmpInst(Pred, X, Cmp.getOperand(1)); | ||||||||
2233 | |||||||||
2234 | const APInt *ShiftVal; | ||||||||
2235 | if (Cmp.isEquality() && match(Shr->getOperand(0), m_APInt(ShiftVal))) | ||||||||
2236 | return foldICmpShrConstConst(Cmp, Shr->getOperand(1), C, *ShiftVal); | ||||||||
2237 | |||||||||
2238 | const APInt *ShiftAmt; | ||||||||
2239 | if (!match(Shr->getOperand(1), m_APInt(ShiftAmt))) | ||||||||
2240 | return nullptr; | ||||||||
2241 | |||||||||
2242 | // Check that the shift amount is in range. If not, don't perform undefined | ||||||||
2243 | // shifts. When the shift is visited it will be simplified. | ||||||||
2244 | unsigned TypeBits = C.getBitWidth(); | ||||||||
2245 | unsigned ShAmtVal = ShiftAmt->getLimitedValue(TypeBits); | ||||||||
2246 | if (ShAmtVal >= TypeBits || ShAmtVal == 0) | ||||||||
2247 | return nullptr; | ||||||||
2248 | |||||||||
2249 | bool IsAShr = Shr->getOpcode() == Instruction::AShr; | ||||||||
2250 | bool IsExact = Shr->isExact(); | ||||||||
2251 | Type *ShrTy = Shr->getType(); | ||||||||
2252 | // TODO: If we could guarantee that InstSimplify would handle all of the | ||||||||
2253 | // constant-value-based preconditions in the folds below, then we could assert | ||||||||
2254 | // those conditions rather than checking them. This is difficult because of | ||||||||
2255 | // undef/poison (PR34838). | ||||||||
2256 | if (IsAShr) { | ||||||||
2257 | if (Pred == CmpInst::ICMP_SLT || (Pred == CmpInst::ICMP_SGT && IsExact)) { | ||||||||
2258 | // icmp slt (ashr X, ShAmtC), C --> icmp slt X, (C << ShAmtC) | ||||||||
2259 | // icmp sgt (ashr exact X, ShAmtC), C --> icmp sgt X, (C << ShAmtC) | ||||||||
2260 | APInt ShiftedC = C.shl(ShAmtVal); | ||||||||
2261 | if (ShiftedC.ashr(ShAmtVal) == C) | ||||||||
2262 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); | ||||||||
2263 | } | ||||||||
2264 | if (Pred == CmpInst::ICMP_SGT) { | ||||||||
2265 | // icmp sgt (ashr X, ShAmtC), C --> icmp sgt X, ((C + 1) << ShAmtC) - 1 | ||||||||
2266 | APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1; | ||||||||
2267 | if (!C.isMaxSignedValue() && !(C + 1).shl(ShAmtVal).isMinSignedValue() && | ||||||||
2268 | (ShiftedC + 1).ashr(ShAmtVal) == (C + 1)) | ||||||||
2269 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); | ||||||||
2270 | } | ||||||||
2271 | |||||||||
2272 | // If the compare constant has significant bits above the lowest sign-bit, | ||||||||
2273 | // then convert an unsigned cmp to a test of the sign-bit: | ||||||||
2274 | // (ashr X, ShiftC) u> C --> X s< 0 | ||||||||
2275 | // (ashr X, ShiftC) u< C --> X s> -1 | ||||||||
2276 | if (C.getBitWidth() > 2 && C.getNumSignBits() <= ShAmtVal) { | ||||||||
2277 | if (Pred == CmpInst::ICMP_UGT) { | ||||||||
2278 | return new ICmpInst(CmpInst::ICMP_SLT, X, | ||||||||
2279 | ConstantInt::getNullValue(ShrTy)); | ||||||||
2280 | } | ||||||||
2281 | if (Pred == CmpInst::ICMP_ULT) { | ||||||||
2282 | return new ICmpInst(CmpInst::ICMP_SGT, X, | ||||||||
2283 | ConstantInt::getAllOnesValue(ShrTy)); | ||||||||
2284 | } | ||||||||
2285 | } | ||||||||
2286 | } else { | ||||||||
2287 | if (Pred == CmpInst::ICMP_ULT || (Pred == CmpInst::ICMP_UGT && IsExact)) { | ||||||||
2288 | // icmp ult (lshr X, ShAmtC), C --> icmp ult X, (C << ShAmtC) | ||||||||
2289 | // icmp ugt (lshr exact X, ShAmtC), C --> icmp ugt X, (C << ShAmtC) | ||||||||
2290 | APInt ShiftedC = C.shl(ShAmtVal); | ||||||||
2291 | if (ShiftedC.lshr(ShAmtVal) == C) | ||||||||
2292 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); | ||||||||
2293 | } | ||||||||
2294 | if (Pred == CmpInst::ICMP_UGT) { | ||||||||
2295 | // icmp ugt (lshr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1 | ||||||||
2296 | APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1; | ||||||||
2297 | if ((ShiftedC + 1).lshr(ShAmtVal) == (C + 1)) | ||||||||
2298 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); | ||||||||
2299 | } | ||||||||
2300 | } | ||||||||
2301 | |||||||||
2302 | if (!Cmp.isEquality()) | ||||||||
2303 | return nullptr; | ||||||||
2304 | |||||||||
2305 | // Handle equality comparisons of shift-by-constant. | ||||||||
2306 | |||||||||
2307 | // If the comparison constant changes with the shift, the comparison cannot | ||||||||
2308 | // succeed (bits of the comparison constant cannot match the shifted value). | ||||||||
2309 | // This should be known by InstSimplify and already be folded to true/false. | ||||||||
2310 | assert(((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) ||(static_cast <bool> (((IsAShr && C.shl(ShAmtVal ).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal) .lshr(ShAmtVal) == C)) && "Expected icmp+shr simplify did not occur." ) ? void (0) : __assert_fail ("((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && \"Expected icmp+shr simplify did not occur.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2312, __extension__ __PRETTY_FUNCTION__)) | ||||||||
2311 | (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) &&(static_cast <bool> (((IsAShr && C.shl(ShAmtVal ).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal) .lshr(ShAmtVal) == C)) && "Expected icmp+shr simplify did not occur." ) ? void (0) : __assert_fail ("((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && \"Expected icmp+shr simplify did not occur.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2312, __extension__ __PRETTY_FUNCTION__)) | ||||||||
2312 | "Expected icmp+shr simplify did not occur.")(static_cast <bool> (((IsAShr && C.shl(ShAmtVal ).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal) .lshr(ShAmtVal) == C)) && "Expected icmp+shr simplify did not occur." ) ? void (0) : __assert_fail ("((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && \"Expected icmp+shr simplify did not occur.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2312, __extension__ __PRETTY_FUNCTION__)); | ||||||||
2313 | |||||||||
2314 | // If the bits shifted out are known zero, compare the unshifted value: | ||||||||
2315 | // (X & 4) >> 1 == 2 --> (X & 4) == 4. | ||||||||
2316 | if (Shr->isExact()) | ||||||||
2317 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, C << ShAmtVal)); | ||||||||
2318 | |||||||||
2319 | if (C.isNullValue()) { | ||||||||
2320 | // == 0 is u< 1. | ||||||||
2321 | if (Pred == CmpInst::ICMP_EQ) | ||||||||
2322 | return new ICmpInst(CmpInst::ICMP_ULT, X, | ||||||||
2323 | ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal))); | ||||||||
2324 | else | ||||||||
2325 | return new ICmpInst(CmpInst::ICMP_UGT, X, | ||||||||
2326 | ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal) - 1)); | ||||||||
2327 | } | ||||||||
2328 | |||||||||
2329 | if (Shr->hasOneUse()) { | ||||||||
2330 | // Canonicalize the shift into an 'and': | ||||||||
2331 | // icmp eq/ne (shr X, ShAmt), C --> icmp eq/ne (and X, HiMask), (C << ShAmt) | ||||||||
2332 | APInt Val(APInt::getHighBitsSet(TypeBits, TypeBits - ShAmtVal)); | ||||||||
2333 | Constant *Mask = ConstantInt::get(ShrTy, Val); | ||||||||
2334 | Value *And = Builder.CreateAnd(X, Mask, Shr->getName() + ".mask"); | ||||||||
2335 | return new ICmpInst(Pred, And, ConstantInt::get(ShrTy, C << ShAmtVal)); | ||||||||
2336 | } | ||||||||
2337 | |||||||||
2338 | return nullptr; | ||||||||
2339 | } | ||||||||
2340 | |||||||||
2341 | Instruction *InstCombinerImpl::foldICmpSRemConstant(ICmpInst &Cmp, | ||||||||
2342 | BinaryOperator *SRem, | ||||||||
2343 | const APInt &C) { | ||||||||
2344 | // Match an 'is positive' or 'is negative' comparison of remainder by a | ||||||||
2345 | // constant power-of-2 value: | ||||||||
2346 | // (X % pow2C) sgt/slt 0 | ||||||||
2347 | const ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
2348 | if (Pred != ICmpInst::ICMP_SGT && Pred != ICmpInst::ICMP_SLT) | ||||||||
2349 | return nullptr; | ||||||||
2350 | |||||||||
2351 | // TODO: The one-use check is standard because we do not typically want to | ||||||||
2352 | // create longer instruction sequences, but this might be a special-case | ||||||||
2353 | // because srem is not good for analysis or codegen. | ||||||||
2354 | if (!SRem->hasOneUse()) | ||||||||
2355 | return nullptr; | ||||||||
2356 | |||||||||
2357 | const APInt *DivisorC; | ||||||||
2358 | if (!C.isNullValue() || !match(SRem->getOperand(1), m_Power2(DivisorC))) | ||||||||
2359 | return nullptr; | ||||||||
2360 | |||||||||
2361 | // Mask off the sign bit and the modulo bits (low-bits). | ||||||||
2362 | Type *Ty = SRem->getType(); | ||||||||
2363 | APInt SignMask = APInt::getSignMask(Ty->getScalarSizeInBits()); | ||||||||
2364 | Constant *MaskC = ConstantInt::get(Ty, SignMask | (*DivisorC - 1)); | ||||||||
2365 | Value *And = Builder.CreateAnd(SRem->getOperand(0), MaskC); | ||||||||
2366 | |||||||||
2367 | // For 'is positive?' check that the sign-bit is clear and at least 1 masked | ||||||||
2368 | // bit is set. Example: | ||||||||
2369 | // (i8 X % 32) s> 0 --> (X & 159) s> 0 | ||||||||
2370 | if (Pred == ICmpInst::ICMP_SGT) | ||||||||
2371 | return new ICmpInst(ICmpInst::ICMP_SGT, And, ConstantInt::getNullValue(Ty)); | ||||||||
2372 | |||||||||
2373 | // For 'is negative?' check that the sign-bit is set and at least 1 masked | ||||||||
2374 | // bit is set. Example: | ||||||||
2375 | // (i16 X % 4) s< 0 --> (X & 32771) u> 32768 | ||||||||
2376 | return new ICmpInst(ICmpInst::ICMP_UGT, And, ConstantInt::get(Ty, SignMask)); | ||||||||
2377 | } | ||||||||
2378 | |||||||||
2379 | /// Fold icmp (udiv X, Y), C. | ||||||||
2380 | Instruction *InstCombinerImpl::foldICmpUDivConstant(ICmpInst &Cmp, | ||||||||
2381 | BinaryOperator *UDiv, | ||||||||
2382 | const APInt &C) { | ||||||||
2383 | const APInt *C2; | ||||||||
2384 | if (!match(UDiv->getOperand(0), m_APInt(C2))) | ||||||||
2385 | return nullptr; | ||||||||
2386 | |||||||||
2387 | assert(*C2 != 0 && "udiv 0, X should have been simplified already.")(static_cast <bool> (*C2 != 0 && "udiv 0, X should have been simplified already." ) ? void (0) : __assert_fail ("*C2 != 0 && \"udiv 0, X should have been simplified already.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2387, __extension__ __PRETTY_FUNCTION__)); | ||||||||
2388 | |||||||||
2389 | // (icmp ugt (udiv C2, Y), C) -> (icmp ule Y, C2/(C+1)) | ||||||||
2390 | Value *Y = UDiv->getOperand(1); | ||||||||
2391 | if (Cmp.getPredicate() == ICmpInst::ICMP_UGT) { | ||||||||
2392 | assert(!C.isMaxValue() &&(static_cast <bool> (!C.isMaxValue() && "icmp ugt X, UINT_MAX should have been simplified already." ) ? void (0) : __assert_fail ("!C.isMaxValue() && \"icmp ugt X, UINT_MAX should have been simplified already.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2393, __extension__ __PRETTY_FUNCTION__)) | ||||||||
2393 | "icmp ugt X, UINT_MAX should have been simplified already.")(static_cast <bool> (!C.isMaxValue() && "icmp ugt X, UINT_MAX should have been simplified already." ) ? void (0) : __assert_fail ("!C.isMaxValue() && \"icmp ugt X, UINT_MAX should have been simplified already.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2393, __extension__ __PRETTY_FUNCTION__)); | ||||||||
2394 | return new ICmpInst(ICmpInst::ICMP_ULE, Y, | ||||||||
2395 | ConstantInt::get(Y->getType(), C2->udiv(C + 1))); | ||||||||
2396 | } | ||||||||
2397 | |||||||||
2398 | // (icmp ult (udiv C2, Y), C) -> (icmp ugt Y, C2/C) | ||||||||
2399 | if (Cmp.getPredicate() == ICmpInst::ICMP_ULT) { | ||||||||
2400 | assert(C != 0 && "icmp ult X, 0 should have been simplified already.")(static_cast <bool> (C != 0 && "icmp ult X, 0 should have been simplified already." ) ? void (0) : __assert_fail ("C != 0 && \"icmp ult X, 0 should have been simplified already.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2400, __extension__ __PRETTY_FUNCTION__)); | ||||||||
2401 | return new ICmpInst(ICmpInst::ICMP_UGT, Y, | ||||||||
2402 | ConstantInt::get(Y->getType(), C2->udiv(C))); | ||||||||
2403 | } | ||||||||
2404 | |||||||||
2405 | return nullptr; | ||||||||
2406 | } | ||||||||
2407 | |||||||||
2408 | /// Fold icmp ({su}div X, Y), C. | ||||||||
2409 | Instruction *InstCombinerImpl::foldICmpDivConstant(ICmpInst &Cmp, | ||||||||
2410 | BinaryOperator *Div, | ||||||||
2411 | const APInt &C) { | ||||||||
2412 | // Fold: icmp pred ([us]div X, C2), C -> range test | ||||||||
2413 | // Fold this div into the comparison, producing a range check. | ||||||||
2414 | // Determine, based on the divide type, what the range is being | ||||||||
2415 | // checked. If there is an overflow on the low or high side, remember | ||||||||
2416 | // it, otherwise compute the range [low, hi) bounding the new value. | ||||||||
2417 | // See: InsertRangeTest above for the kinds of replacements possible. | ||||||||
2418 | const APInt *C2; | ||||||||
2419 | if (!match(Div->getOperand(1), m_APInt(C2))) | ||||||||
2420 | return nullptr; | ||||||||
2421 | |||||||||
2422 | // FIXME: If the operand types don't match the type of the divide | ||||||||
2423 | // then don't attempt this transform. The code below doesn't have the | ||||||||
2424 | // logic to deal with a signed divide and an unsigned compare (and | ||||||||
2425 | // vice versa). This is because (x /s C2) <s C produces different | ||||||||
2426 | // results than (x /s C2) <u C or (x /u C2) <s C or even | ||||||||
2427 | // (x /u C2) <u C. Simply casting the operands and result won't | ||||||||
2428 | // work. :( The if statement below tests that condition and bails | ||||||||
2429 | // if it finds it. | ||||||||
2430 | bool DivIsSigned = Div->getOpcode() == Instruction::SDiv; | ||||||||
2431 | if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned()) | ||||||||
2432 | return nullptr; | ||||||||
2433 | |||||||||
2434 | // The ProdOV computation fails on divide by 0 and divide by -1. Cases with | ||||||||
2435 | // INT_MIN will also fail if the divisor is 1. Although folds of all these | ||||||||
2436 | // division-by-constant cases should be present, we can not assert that they | ||||||||
2437 | // have happened before we reach this icmp instruction. | ||||||||
2438 | if (C2->isNullValue() || C2->isOneValue() || | ||||||||
2439 | (DivIsSigned && C2->isAllOnesValue())) | ||||||||
2440 | return nullptr; | ||||||||
2441 | |||||||||
2442 | // Compute Prod = C * C2. We are essentially solving an equation of | ||||||||
2443 | // form X / C2 = C. We solve for X by multiplying C2 and C. | ||||||||
2444 | // By solving for X, we can turn this into a range check instead of computing | ||||||||
2445 | // a divide. | ||||||||
2446 | APInt Prod = C * *C2; | ||||||||
2447 | |||||||||
2448 | // Determine if the product overflows by seeing if the product is not equal to | ||||||||
2449 | // the divide. Make sure we do the same kind of divide as in the LHS | ||||||||
2450 | // instruction that we're folding. | ||||||||
2451 | bool ProdOV = (DivIsSigned ? Prod.sdiv(*C2) : Prod.udiv(*C2)) != C; | ||||||||
2452 | |||||||||
2453 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
2454 | |||||||||
2455 | // If the division is known to be exact, then there is no remainder from the | ||||||||
2456 | // divide, so the covered range size is unit, otherwise it is the divisor. | ||||||||
2457 | APInt RangeSize = Div->isExact() ? APInt(C2->getBitWidth(), 1) : *C2; | ||||||||
2458 | |||||||||
2459 | // Figure out the interval that is being checked. For example, a comparison | ||||||||
2460 | // like "X /u 5 == 0" is really checking that X is in the interval [0, 5). | ||||||||
2461 | // Compute this interval based on the constants involved and the signedness of | ||||||||
2462 | // the compare/divide. This computes a half-open interval, keeping track of | ||||||||
2463 | // whether either value in the interval overflows. After analysis each | ||||||||
2464 | // overflow variable is set to 0 if it's corresponding bound variable is valid | ||||||||
2465 | // -1 if overflowed off the bottom end, or +1 if overflowed off the top end. | ||||||||
2466 | int LoOverflow = 0, HiOverflow = 0; | ||||||||
2467 | APInt LoBound, HiBound; | ||||||||
2468 | |||||||||
2469 | if (!DivIsSigned) { // udiv | ||||||||
2470 | // e.g. X/5 op 3 --> [15, 20) | ||||||||
2471 | LoBound = Prod; | ||||||||
2472 | HiOverflow = LoOverflow = ProdOV; | ||||||||
2473 | if (!HiOverflow) { | ||||||||
2474 | // If this is not an exact divide, then many values in the range collapse | ||||||||
2475 | // to the same result value. | ||||||||
2476 | HiOverflow = addWithOverflow(HiBound, LoBound, RangeSize, false); | ||||||||
2477 | } | ||||||||
2478 | } else if (C2->isStrictlyPositive()) { // Divisor is > 0. | ||||||||
2479 | if (C.isNullValue()) { // (X / pos) op 0 | ||||||||
2480 | // Can't overflow. e.g. X/2 op 0 --> [-1, 2) | ||||||||
2481 | LoBound = -(RangeSize - 1); | ||||||||
2482 | HiBound = RangeSize; | ||||||||
2483 | } else if (C.isStrictlyPositive()) { // (X / pos) op pos | ||||||||
2484 | LoBound = Prod; // e.g. X/5 op 3 --> [15, 20) | ||||||||
2485 | HiOverflow = LoOverflow = ProdOV; | ||||||||
2486 | if (!HiOverflow) | ||||||||
2487 | HiOverflow = addWithOverflow(HiBound, Prod, RangeSize, true); | ||||||||
2488 | } else { // (X / pos) op neg | ||||||||
2489 | // e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14) | ||||||||
2490 | HiBound = Prod + 1; | ||||||||
2491 | LoOverflow = HiOverflow = ProdOV ? -1 : 0; | ||||||||
2492 | if (!LoOverflow) { | ||||||||
2493 | APInt DivNeg = -RangeSize; | ||||||||
2494 | LoOverflow = addWithOverflow(LoBound, HiBound, DivNeg, true) ? -1 : 0; | ||||||||
2495 | } | ||||||||
2496 | } | ||||||||
2497 | } else if (C2->isNegative()) { // Divisor is < 0. | ||||||||
2498 | if (Div->isExact()) | ||||||||
2499 | RangeSize.negate(); | ||||||||
2500 | if (C.isNullValue()) { // (X / neg) op 0 | ||||||||
2501 | // e.g. X/-5 op 0 --> [-4, 5) | ||||||||
2502 | LoBound = RangeSize + 1; | ||||||||
2503 | HiBound = -RangeSize; | ||||||||
2504 | if (HiBound == *C2) { // -INTMIN = INTMIN | ||||||||
2505 | HiOverflow = 1; // [INTMIN+1, overflow) | ||||||||
2506 | HiBound = APInt(); // e.g. X/INTMIN = 0 --> X > INTMIN | ||||||||
2507 | } | ||||||||
2508 | } else if (C.isStrictlyPositive()) { // (X / neg) op pos | ||||||||
2509 | // e.g. X/-5 op 3 --> [-19, -14) | ||||||||
2510 | HiBound = Prod + 1; | ||||||||
2511 | HiOverflow = LoOverflow = ProdOV ? -1 : 0; | ||||||||
2512 | if (!LoOverflow) | ||||||||
2513 | LoOverflow = addWithOverflow(LoBound, HiBound, RangeSize, true) ? -1:0; | ||||||||
2514 | } else { // (X / neg) op neg | ||||||||
2515 | LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20) | ||||||||
2516 | LoOverflow = HiOverflow = ProdOV; | ||||||||
2517 | if (!HiOverflow) | ||||||||
2518 | HiOverflow = subWithOverflow(HiBound, Prod, RangeSize, true); | ||||||||
2519 | } | ||||||||
2520 | |||||||||
2521 | // Dividing by a negative swaps the condition. LT <-> GT | ||||||||
2522 | Pred = ICmpInst::getSwappedPredicate(Pred); | ||||||||
2523 | } | ||||||||
2524 | |||||||||
2525 | Value *X = Div->getOperand(0); | ||||||||
2526 | switch (Pred) { | ||||||||
2527 | default: llvm_unreachable("Unhandled icmp opcode!")::llvm::llvm_unreachable_internal("Unhandled icmp opcode!", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2527); | ||||||||
2528 | case ICmpInst::ICMP_EQ: | ||||||||
2529 | if (LoOverflow && HiOverflow) | ||||||||
2530 | return replaceInstUsesWith(Cmp, Builder.getFalse()); | ||||||||
2531 | if (HiOverflow) | ||||||||
2532 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : | ||||||||
2533 | ICmpInst::ICMP_UGE, X, | ||||||||
2534 | ConstantInt::get(Div->getType(), LoBound)); | ||||||||
2535 | if (LoOverflow) | ||||||||
2536 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : | ||||||||
2537 | ICmpInst::ICMP_ULT, X, | ||||||||
2538 | ConstantInt::get(Div->getType(), HiBound)); | ||||||||
2539 | return replaceInstUsesWith( | ||||||||
2540 | Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, true)); | ||||||||
2541 | case ICmpInst::ICMP_NE: | ||||||||
2542 | if (LoOverflow && HiOverflow) | ||||||||
2543 | return replaceInstUsesWith(Cmp, Builder.getTrue()); | ||||||||
2544 | if (HiOverflow) | ||||||||
2545 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : | ||||||||
2546 | ICmpInst::ICMP_ULT, X, | ||||||||
2547 | ConstantInt::get(Div->getType(), LoBound)); | ||||||||
2548 | if (LoOverflow) | ||||||||
2549 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : | ||||||||
2550 | ICmpInst::ICMP_UGE, X, | ||||||||
2551 | ConstantInt::get(Div->getType(), HiBound)); | ||||||||
2552 | return replaceInstUsesWith(Cmp, | ||||||||
2553 | insertRangeTest(X, LoBound, HiBound, | ||||||||
2554 | DivIsSigned, false)); | ||||||||
2555 | case ICmpInst::ICMP_ULT: | ||||||||
2556 | case ICmpInst::ICMP_SLT: | ||||||||
2557 | if (LoOverflow == +1) // Low bound is greater than input range. | ||||||||
2558 | return replaceInstUsesWith(Cmp, Builder.getTrue()); | ||||||||
2559 | if (LoOverflow == -1) // Low bound is less than input range. | ||||||||
2560 | return replaceInstUsesWith(Cmp, Builder.getFalse()); | ||||||||
2561 | return new ICmpInst(Pred, X, ConstantInt::get(Div->getType(), LoBound)); | ||||||||
2562 | case ICmpInst::ICMP_UGT: | ||||||||
2563 | case ICmpInst::ICMP_SGT: | ||||||||
2564 | if (HiOverflow == +1) // High bound greater than input range. | ||||||||
2565 | return replaceInstUsesWith(Cmp, Builder.getFalse()); | ||||||||
2566 | if (HiOverflow == -1) // High bound less than input range. | ||||||||
2567 | return replaceInstUsesWith(Cmp, Builder.getTrue()); | ||||||||
2568 | if (Pred == ICmpInst::ICMP_UGT) | ||||||||
2569 | return new ICmpInst(ICmpInst::ICMP_UGE, X, | ||||||||
2570 | ConstantInt::get(Div->getType(), HiBound)); | ||||||||
2571 | return new ICmpInst(ICmpInst::ICMP_SGE, X, | ||||||||
2572 | ConstantInt::get(Div->getType(), HiBound)); | ||||||||
2573 | } | ||||||||
2574 | |||||||||
2575 | return nullptr; | ||||||||
2576 | } | ||||||||
2577 | |||||||||
2578 | /// Fold icmp (sub X, Y), C. | ||||||||
2579 | Instruction *InstCombinerImpl::foldICmpSubConstant(ICmpInst &Cmp, | ||||||||
2580 | BinaryOperator *Sub, | ||||||||
2581 | const APInt &C) { | ||||||||
2582 | Value *X = Sub->getOperand(0), *Y = Sub->getOperand(1); | ||||||||
2583 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
2584 | const APInt *C2; | ||||||||
2585 | APInt SubResult; | ||||||||
2586 | |||||||||
2587 | // icmp eq/ne (sub C, Y), C -> icmp eq/ne Y, 0 | ||||||||
2588 | if (match(X, m_APInt(C2)) && *C2 == C && Cmp.isEquality()) | ||||||||
2589 | return new ICmpInst(Cmp.getPredicate(), Y, | ||||||||
2590 | ConstantInt::get(Y->getType(), 0)); | ||||||||
2591 | |||||||||
2592 | // (icmp P (sub nuw|nsw C2, Y), C) -> (icmp swap(P) Y, C2-C) | ||||||||
2593 | if (match(X, m_APInt(C2)) && | ||||||||
2594 | ((Cmp.isUnsigned() && Sub->hasNoUnsignedWrap()) || | ||||||||
2595 | (Cmp.isSigned() && Sub->hasNoSignedWrap())) && | ||||||||
2596 | !subWithOverflow(SubResult, *C2, C, Cmp.isSigned())) | ||||||||
2597 | return new ICmpInst(Cmp.getSwappedPredicate(), Y, | ||||||||
2598 | ConstantInt::get(Y->getType(), SubResult)); | ||||||||
2599 | |||||||||
2600 | // The following transforms are only worth it if the only user of the subtract | ||||||||
2601 | // is the icmp. | ||||||||
2602 | if (!Sub->hasOneUse()) | ||||||||
2603 | return nullptr; | ||||||||
2604 | |||||||||
2605 | if (Sub->hasNoSignedWrap()) { | ||||||||
2606 | // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y) | ||||||||
2607 | if (Pred == ICmpInst::ICMP_SGT && C.isAllOnesValue()) | ||||||||
2608 | return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); | ||||||||
2609 | |||||||||
2610 | // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y) | ||||||||
2611 | if (Pred == ICmpInst::ICMP_SGT && C.isNullValue()) | ||||||||
2612 | return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); | ||||||||
2613 | |||||||||
2614 | // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y) | ||||||||
2615 | if (Pred == ICmpInst::ICMP_SLT && C.isNullValue()) | ||||||||
2616 | return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); | ||||||||
2617 | |||||||||
2618 | // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y) | ||||||||
2619 | if (Pred == ICmpInst::ICMP_SLT && C.isOneValue()) | ||||||||
2620 | return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); | ||||||||
2621 | } | ||||||||
2622 | |||||||||
2623 | if (!match(X, m_APInt(C2))) | ||||||||
2624 | return nullptr; | ||||||||
2625 | |||||||||
2626 | // C2 - Y <u C -> (Y | (C - 1)) == C2 | ||||||||
2627 | // iff (C2 & (C - 1)) == C - 1 and C is a power of 2 | ||||||||
2628 | if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && | ||||||||
2629 | (*C2 & (C - 1)) == (C - 1)) | ||||||||
2630 | return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateOr(Y, C - 1), X); | ||||||||
2631 | |||||||||
2632 | // C2 - Y >u C -> (Y | C) != C2 | ||||||||
2633 | // iff C2 & C == C and C + 1 is a power of 2 | ||||||||
2634 | if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == C) | ||||||||
2635 | return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateOr(Y, C), X); | ||||||||
2636 | |||||||||
2637 | return nullptr; | ||||||||
2638 | } | ||||||||
2639 | |||||||||
2640 | /// Fold icmp (add X, Y), C. | ||||||||
2641 | Instruction *InstCombinerImpl::foldICmpAddConstant(ICmpInst &Cmp, | ||||||||
2642 | BinaryOperator *Add, | ||||||||
2643 | const APInt &C) { | ||||||||
2644 | Value *Y = Add->getOperand(1); | ||||||||
2645 | const APInt *C2; | ||||||||
2646 | if (Cmp.isEquality() || !match(Y, m_APInt(C2))) | ||||||||
2647 | return nullptr; | ||||||||
2648 | |||||||||
2649 | // Fold icmp pred (add X, C2), C. | ||||||||
2650 | Value *X = Add->getOperand(0); | ||||||||
2651 | Type *Ty = Add->getType(); | ||||||||
2652 | const CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
2653 | |||||||||
2654 | // If the add does not wrap, we can always adjust the compare by subtracting | ||||||||
2655 | // the constants. Equality comparisons are handled elsewhere. SGE/SLE/UGE/ULE | ||||||||
2656 | // are canonicalized to SGT/SLT/UGT/ULT. | ||||||||
2657 | if ((Add->hasNoSignedWrap() && | ||||||||
2658 | (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT)) || | ||||||||
2659 | (Add->hasNoUnsignedWrap() && | ||||||||
2660 | (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULT))) { | ||||||||
2661 | bool Overflow; | ||||||||
2662 | APInt NewC = | ||||||||
2663 | Cmp.isSigned() ? C.ssub_ov(*C2, Overflow) : C.usub_ov(*C2, Overflow); | ||||||||
2664 | // If there is overflow, the result must be true or false. | ||||||||
2665 | // TODO: Can we assert there is no overflow because InstSimplify always | ||||||||
2666 | // handles those cases? | ||||||||
2667 | if (!Overflow) | ||||||||
2668 | // icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2) | ||||||||
2669 | return new ICmpInst(Pred, X, ConstantInt::get(Ty, NewC)); | ||||||||
2670 | } | ||||||||
2671 | |||||||||
2672 | auto CR = ConstantRange::makeExactICmpRegion(Pred, C).subtract(*C2); | ||||||||
2673 | const APInt &Upper = CR.getUpper(); | ||||||||
2674 | const APInt &Lower = CR.getLower(); | ||||||||
2675 | if (Cmp.isSigned()) { | ||||||||
2676 | if (Lower.isSignMask()) | ||||||||
2677 | return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, Upper)); | ||||||||
2678 | if (Upper.isSignMask()) | ||||||||
2679 | return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, Lower)); | ||||||||
2680 | } else { | ||||||||
2681 | if (Lower.isMinValue()) | ||||||||
2682 | return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, Upper)); | ||||||||
2683 | if (Upper.isMinValue()) | ||||||||
2684 | return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, Lower)); | ||||||||
2685 | } | ||||||||
2686 | |||||||||
2687 | // This set of folds is intentionally placed after folds that use no-wrapping | ||||||||
2688 | // flags because those folds are likely better for later analysis/codegen. | ||||||||
2689 | const APInt SMax = APInt::getSignedMaxValue(Ty->getScalarSizeInBits()); | ||||||||
2690 | const APInt SMin = APInt::getSignedMinValue(Ty->getScalarSizeInBits()); | ||||||||
2691 | |||||||||
2692 | // Fold compare with offset to opposite sign compare if it eliminates offset: | ||||||||
2693 | // (X + C2) >u C --> X <s -C2 (if C == C2 + SMAX) | ||||||||
2694 | if (Pred == CmpInst::ICMP_UGT && C == *C2 + SMax) | ||||||||
2695 | return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, -(*C2))); | ||||||||
2696 | |||||||||
2697 | // (X + C2) <u C --> X >s ~C2 (if C == C2 + SMIN) | ||||||||
2698 | if (Pred == CmpInst::ICMP_ULT && C == *C2 + SMin) | ||||||||
2699 | return new ICmpInst(ICmpInst::ICMP_SGT, X, ConstantInt::get(Ty, ~(*C2))); | ||||||||
2700 | |||||||||
2701 | // (X + C2) >s C --> X <u (SMAX - C) (if C == C2 - 1) | ||||||||
2702 | if (Pred == CmpInst::ICMP_SGT && C == *C2 - 1) | ||||||||
2703 | return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, SMax - C)); | ||||||||
2704 | |||||||||
2705 | // (X + C2) <s C --> X >u (C ^ SMAX) (if C == C2) | ||||||||
2706 | if (Pred == CmpInst::ICMP_SLT && C == *C2) | ||||||||
2707 | return new ICmpInst(ICmpInst::ICMP_UGT, X, ConstantInt::get(Ty, C ^ SMax)); | ||||||||
2708 | |||||||||
2709 | if (!Add->hasOneUse()) | ||||||||
2710 | return nullptr; | ||||||||
2711 | |||||||||
2712 | // X+C <u C2 -> (X & -C2) == C | ||||||||
2713 | // iff C & (C2-1) == 0 | ||||||||
2714 | // C2 is a power of 2 | ||||||||
2715 | if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && (*C2 & (C - 1)) == 0) | ||||||||
2716 | return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateAnd(X, -C), | ||||||||
2717 | ConstantExpr::getNeg(cast<Constant>(Y))); | ||||||||
2718 | |||||||||
2719 | // X+C >u C2 -> (X & ~C2) != C | ||||||||
2720 | // iff C & C2 == 0 | ||||||||
2721 | // C2+1 is a power of 2 | ||||||||
2722 | if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == 0) | ||||||||
2723 | return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateAnd(X, ~C), | ||||||||
2724 | ConstantExpr::getNeg(cast<Constant>(Y))); | ||||||||
2725 | |||||||||
2726 | return nullptr; | ||||||||
2727 | } | ||||||||
2728 | |||||||||
2729 | bool InstCombinerImpl::matchThreeWayIntCompare(SelectInst *SI, Value *&LHS, | ||||||||
2730 | Value *&RHS, ConstantInt *&Less, | ||||||||
2731 | ConstantInt *&Equal, | ||||||||
2732 | ConstantInt *&Greater) { | ||||||||
2733 | // TODO: Generalize this to work with other comparison idioms or ensure | ||||||||
2734 | // they get canonicalized into this form. | ||||||||
2735 | |||||||||
2736 | // select i1 (a == b), | ||||||||
2737 | // i32 Equal, | ||||||||
2738 | // i32 (select i1 (a < b), i32 Less, i32 Greater) | ||||||||
2739 | // where Equal, Less and Greater are placeholders for any three constants. | ||||||||
2740 | ICmpInst::Predicate PredA; | ||||||||
2741 | if (!match(SI->getCondition(), m_ICmp(PredA, m_Value(LHS), m_Value(RHS))) || | ||||||||
2742 | !ICmpInst::isEquality(PredA)) | ||||||||
2743 | return false; | ||||||||
2744 | Value *EqualVal = SI->getTrueValue(); | ||||||||
2745 | Value *UnequalVal = SI->getFalseValue(); | ||||||||
2746 | // We still can get non-canonical predicate here, so canonicalize. | ||||||||
2747 | if (PredA == ICmpInst::ICMP_NE) | ||||||||
2748 | std::swap(EqualVal, UnequalVal); | ||||||||
2749 | if (!match(EqualVal, m_ConstantInt(Equal))) | ||||||||
2750 | return false; | ||||||||
2751 | ICmpInst::Predicate PredB; | ||||||||
2752 | Value *LHS2, *RHS2; | ||||||||
2753 | if (!match(UnequalVal, m_Select(m_ICmp(PredB, m_Value(LHS2), m_Value(RHS2)), | ||||||||
2754 | m_ConstantInt(Less), m_ConstantInt(Greater)))) | ||||||||
2755 | return false; | ||||||||
2756 | // We can get predicate mismatch here, so canonicalize if possible: | ||||||||
2757 | // First, ensure that 'LHS' match. | ||||||||
2758 | if (LHS2 != LHS) { | ||||||||
2759 | // x sgt y <--> y slt x | ||||||||
2760 | std::swap(LHS2, RHS2); | ||||||||
2761 | PredB = ICmpInst::getSwappedPredicate(PredB); | ||||||||
2762 | } | ||||||||
2763 | if (LHS2 != LHS) | ||||||||
2764 | return false; | ||||||||
2765 | // We also need to canonicalize 'RHS'. | ||||||||
2766 | if (PredB == ICmpInst::ICMP_SGT && isa<Constant>(RHS2)) { | ||||||||
2767 | // x sgt C-1 <--> x sge C <--> not(x slt C) | ||||||||
2768 | auto FlippedStrictness = | ||||||||
2769 | InstCombiner::getFlippedStrictnessPredicateAndConstant( | ||||||||
2770 | PredB, cast<Constant>(RHS2)); | ||||||||
2771 | if (!FlippedStrictness) | ||||||||
2772 | return false; | ||||||||
2773 | assert(FlippedStrictness->first == ICmpInst::ICMP_SGE && "Sanity check")(static_cast <bool> (FlippedStrictness->first == ICmpInst ::ICMP_SGE && "Sanity check") ? void (0) : __assert_fail ("FlippedStrictness->first == ICmpInst::ICMP_SGE && \"Sanity check\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2773, __extension__ __PRETTY_FUNCTION__)); | ||||||||
2774 | RHS2 = FlippedStrictness->second; | ||||||||
2775 | // And kind-of perform the result swap. | ||||||||
2776 | std::swap(Less, Greater); | ||||||||
2777 | PredB = ICmpInst::ICMP_SLT; | ||||||||
2778 | } | ||||||||
2779 | return PredB == ICmpInst::ICMP_SLT && RHS == RHS2; | ||||||||
2780 | } | ||||||||
2781 | |||||||||
2782 | Instruction *InstCombinerImpl::foldICmpSelectConstant(ICmpInst &Cmp, | ||||||||
2783 | SelectInst *Select, | ||||||||
2784 | ConstantInt *C) { | ||||||||
2785 | |||||||||
2786 | assert(C && "Cmp RHS should be a constant int!")(static_cast <bool> (C && "Cmp RHS should be a constant int!" ) ? void (0) : __assert_fail ("C && \"Cmp RHS should be a constant int!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2786, __extension__ __PRETTY_FUNCTION__)); | ||||||||
2787 | // If we're testing a constant value against the result of a three way | ||||||||
2788 | // comparison, the result can be expressed directly in terms of the | ||||||||
2789 | // original values being compared. Note: We could possibly be more | ||||||||
2790 | // aggressive here and remove the hasOneUse test. The original select is | ||||||||
2791 | // really likely to simplify or sink when we remove a test of the result. | ||||||||
2792 | Value *OrigLHS, *OrigRHS; | ||||||||
2793 | ConstantInt *C1LessThan, *C2Equal, *C3GreaterThan; | ||||||||
2794 | if (Cmp.hasOneUse() && | ||||||||
2795 | matchThreeWayIntCompare(Select, OrigLHS, OrigRHS, C1LessThan, C2Equal, | ||||||||
2796 | C3GreaterThan)) { | ||||||||
2797 | assert(C1LessThan && C2Equal && C3GreaterThan)(static_cast <bool> (C1LessThan && C2Equal && C3GreaterThan) ? void (0) : __assert_fail ("C1LessThan && C2Equal && C3GreaterThan" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2797, __extension__ __PRETTY_FUNCTION__)); | ||||||||
2798 | |||||||||
2799 | bool TrueWhenLessThan = | ||||||||
2800 | ConstantExpr::getCompare(Cmp.getPredicate(), C1LessThan, C) | ||||||||
2801 | ->isAllOnesValue(); | ||||||||
2802 | bool TrueWhenEqual = | ||||||||
2803 | ConstantExpr::getCompare(Cmp.getPredicate(), C2Equal, C) | ||||||||
2804 | ->isAllOnesValue(); | ||||||||
2805 | bool TrueWhenGreaterThan = | ||||||||
2806 | ConstantExpr::getCompare(Cmp.getPredicate(), C3GreaterThan, C) | ||||||||
2807 | ->isAllOnesValue(); | ||||||||
2808 | |||||||||
2809 | // This generates the new instruction that will replace the original Cmp | ||||||||
2810 | // Instruction. Instead of enumerating the various combinations when | ||||||||
2811 | // TrueWhenLessThan, TrueWhenEqual and TrueWhenGreaterThan are true versus | ||||||||
2812 | // false, we rely on chaining of ORs and future passes of InstCombine to | ||||||||
2813 | // simplify the OR further (i.e. a s< b || a == b becomes a s<= b). | ||||||||
2814 | |||||||||
2815 | // When none of the three constants satisfy the predicate for the RHS (C), | ||||||||
2816 | // the entire original Cmp can be simplified to a false. | ||||||||
2817 | Value *Cond = Builder.getFalse(); | ||||||||
2818 | if (TrueWhenLessThan) | ||||||||
2819 | Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SLT, | ||||||||
2820 | OrigLHS, OrigRHS)); | ||||||||
2821 | if (TrueWhenEqual) | ||||||||
2822 | Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_EQ, | ||||||||
2823 | OrigLHS, OrigRHS)); | ||||||||
2824 | if (TrueWhenGreaterThan) | ||||||||
2825 | Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SGT, | ||||||||
2826 | OrigLHS, OrigRHS)); | ||||||||
2827 | |||||||||
2828 | return replaceInstUsesWith(Cmp, Cond); | ||||||||
2829 | } | ||||||||
2830 | return nullptr; | ||||||||
2831 | } | ||||||||
2832 | |||||||||
2833 | static Instruction *foldICmpBitCast(ICmpInst &Cmp, | ||||||||
2834 | InstCombiner::BuilderTy &Builder) { | ||||||||
2835 | auto *Bitcast = dyn_cast<BitCastInst>(Cmp.getOperand(0)); | ||||||||
2836 | if (!Bitcast) | ||||||||
2837 | return nullptr; | ||||||||
2838 | |||||||||
2839 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
2840 | Value *Op1 = Cmp.getOperand(1); | ||||||||
2841 | Value *BCSrcOp = Bitcast->getOperand(0); | ||||||||
2842 | |||||||||
2843 | // Make sure the bitcast doesn't change the number of vector elements. | ||||||||
2844 | if (Bitcast->getSrcTy()->getScalarSizeInBits() == | ||||||||
2845 | Bitcast->getDestTy()->getScalarSizeInBits()) { | ||||||||
2846 | // Zero-equality and sign-bit checks are preserved through sitofp + bitcast. | ||||||||
2847 | Value *X; | ||||||||
2848 | if (match(BCSrcOp, m_SIToFP(m_Value(X)))) { | ||||||||
2849 | // icmp eq (bitcast (sitofp X)), 0 --> icmp eq X, 0 | ||||||||
2850 | // icmp ne (bitcast (sitofp X)), 0 --> icmp ne X, 0 | ||||||||
2851 | // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0 | ||||||||
2852 | // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0 | ||||||||
2853 | if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT || | ||||||||
2854 | Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) && | ||||||||
2855 | match(Op1, m_Zero())) | ||||||||
2856 | return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType())); | ||||||||
2857 | |||||||||
2858 | // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1 | ||||||||
2859 | if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_One())) | ||||||||
2860 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), 1)); | ||||||||
2861 | |||||||||
2862 | // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1 | ||||||||
2863 | if (Pred == ICmpInst::ICMP_SGT && match(Op1, m_AllOnes())) | ||||||||
2864 | return new ICmpInst(Pred, X, | ||||||||
2865 | ConstantInt::getAllOnesValue(X->getType())); | ||||||||
2866 | } | ||||||||
2867 | |||||||||
2868 | // Zero-equality checks are preserved through unsigned floating-point casts: | ||||||||
2869 | // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0 | ||||||||
2870 | // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0 | ||||||||
2871 | if (match(BCSrcOp, m_UIToFP(m_Value(X)))) | ||||||||
2872 | if (Cmp.isEquality() && match(Op1, m_Zero())) | ||||||||
2873 | return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType())); | ||||||||
2874 | |||||||||
2875 | // If this is a sign-bit test of a bitcast of a casted FP value, eliminate | ||||||||
2876 | // the FP extend/truncate because that cast does not change the sign-bit. | ||||||||
2877 | // This is true for all standard IEEE-754 types and the X86 80-bit type. | ||||||||
2878 | // The sign-bit is always the most significant bit in those types. | ||||||||
2879 | const APInt *C; | ||||||||
2880 | bool TrueIfSigned; | ||||||||
2881 | if (match(Op1, m_APInt(C)) && Bitcast->hasOneUse() && | ||||||||
2882 | InstCombiner::isSignBitCheck(Pred, *C, TrueIfSigned)) { | ||||||||
2883 | if (match(BCSrcOp, m_FPExt(m_Value(X))) || | ||||||||
2884 | match(BCSrcOp, m_FPTrunc(m_Value(X)))) { | ||||||||
2885 | // (bitcast (fpext/fptrunc X)) to iX) < 0 --> (bitcast X to iY) < 0 | ||||||||
2886 | // (bitcast (fpext/fptrunc X)) to iX) > -1 --> (bitcast X to iY) > -1 | ||||||||
2887 | Type *XType = X->getType(); | ||||||||
2888 | |||||||||
2889 | // We can't currently handle Power style floating point operations here. | ||||||||
2890 | if (!(XType->isPPC_FP128Ty() || BCSrcOp->getType()->isPPC_FP128Ty())) { | ||||||||
2891 | |||||||||
2892 | Type *NewType = Builder.getIntNTy(XType->getScalarSizeInBits()); | ||||||||
2893 | if (auto *XVTy = dyn_cast<VectorType>(XType)) | ||||||||
2894 | NewType = VectorType::get(NewType, XVTy->getElementCount()); | ||||||||
2895 | Value *NewBitcast = Builder.CreateBitCast(X, NewType); | ||||||||
2896 | if (TrueIfSigned) | ||||||||
2897 | return new ICmpInst(ICmpInst::ICMP_SLT, NewBitcast, | ||||||||
2898 | ConstantInt::getNullValue(NewType)); | ||||||||
2899 | else | ||||||||
2900 | return new ICmpInst(ICmpInst::ICMP_SGT, NewBitcast, | ||||||||
2901 | ConstantInt::getAllOnesValue(NewType)); | ||||||||
2902 | } | ||||||||
2903 | } | ||||||||
2904 | } | ||||||||
2905 | } | ||||||||
2906 | |||||||||
2907 | // Test to see if the operands of the icmp are casted versions of other | ||||||||
2908 | // values. If the ptr->ptr cast can be stripped off both arguments, do so. | ||||||||
2909 | if (Bitcast->getType()->isPointerTy() && | ||||||||
2910 | (isa<Constant>(Op1) || isa<BitCastInst>(Op1))) { | ||||||||
2911 | // If operand #1 is a bitcast instruction, it must also be a ptr->ptr cast | ||||||||
2912 | // so eliminate it as well. | ||||||||
2913 | if (auto *BC2 = dyn_cast<BitCastInst>(Op1)) | ||||||||
2914 | Op1 = BC2->getOperand(0); | ||||||||
2915 | |||||||||
2916 | Op1 = Builder.CreateBitCast(Op1, BCSrcOp->getType()); | ||||||||
2917 | return new ICmpInst(Pred, BCSrcOp, Op1); | ||||||||
2918 | } | ||||||||
2919 | |||||||||
2920 | // Folding: icmp <pred> iN X, C | ||||||||
2921 | // where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN | ||||||||
2922 | // and C is a splat of a K-bit pattern | ||||||||
2923 | // and SC is a constant vector = <C', C', C', ..., C'> | ||||||||
2924 | // Into: | ||||||||
2925 | // %E = extractelement <M x iK> %vec, i32 C' | ||||||||
2926 | // icmp <pred> iK %E, trunc(C) | ||||||||
2927 | const APInt *C; | ||||||||
2928 | if (!match(Cmp.getOperand(1), m_APInt(C)) || | ||||||||
2929 | !Bitcast->getType()->isIntegerTy() || | ||||||||
2930 | !Bitcast->getSrcTy()->isIntOrIntVectorTy()) | ||||||||
2931 | return nullptr; | ||||||||
2932 | |||||||||
2933 | Value *Vec; | ||||||||
2934 | ArrayRef<int> Mask; | ||||||||
2935 | if (match(BCSrcOp, m_Shuffle(m_Value(Vec), m_Undef(), m_Mask(Mask)))) { | ||||||||
2936 | // Check whether every element of Mask is the same constant | ||||||||
2937 | if (is_splat(Mask)) { | ||||||||
2938 | auto *VecTy = cast<VectorType>(BCSrcOp->getType()); | ||||||||
2939 | auto *EltTy = cast<IntegerType>(VecTy->getElementType()); | ||||||||
2940 | if (C->isSplat(EltTy->getBitWidth())) { | ||||||||
2941 | // Fold the icmp based on the value of C | ||||||||
2942 | // If C is M copies of an iK sized bit pattern, | ||||||||
2943 | // then: | ||||||||
2944 | // => %E = extractelement <N x iK> %vec, i32 Elem | ||||||||
2945 | // icmp <pred> iK %SplatVal, <pattern> | ||||||||
2946 | Value *Elem = Builder.getInt32(Mask[0]); | ||||||||
2947 | Value *Extract = Builder.CreateExtractElement(Vec, Elem); | ||||||||
2948 | Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth())); | ||||||||
2949 | return new ICmpInst(Pred, Extract, NewC); | ||||||||
2950 | } | ||||||||
2951 | } | ||||||||
2952 | } | ||||||||
2953 | return nullptr; | ||||||||
2954 | } | ||||||||
2955 | |||||||||
2956 | /// Try to fold integer comparisons with a constant operand: icmp Pred X, C | ||||||||
2957 | /// where X is some kind of instruction. | ||||||||
2958 | Instruction *InstCombinerImpl::foldICmpInstWithConstant(ICmpInst &Cmp) { | ||||||||
2959 | const APInt *C; | ||||||||
2960 | if (!match(Cmp.getOperand(1), m_APInt(C))) | ||||||||
2961 | return nullptr; | ||||||||
2962 | |||||||||
2963 | if (auto *BO = dyn_cast<BinaryOperator>(Cmp.getOperand(0))) { | ||||||||
2964 | switch (BO->getOpcode()) { | ||||||||
2965 | case Instruction::Xor: | ||||||||
2966 | if (Instruction *I = foldICmpXorConstant(Cmp, BO, *C)) | ||||||||
2967 | return I; | ||||||||
2968 | break; | ||||||||
2969 | case Instruction::And: | ||||||||
2970 | if (Instruction *I = foldICmpAndConstant(Cmp, BO, *C)) | ||||||||
2971 | return I; | ||||||||
2972 | break; | ||||||||
2973 | case Instruction::Or: | ||||||||
2974 | if (Instruction *I = foldICmpOrConstant(Cmp, BO, *C)) | ||||||||
2975 | return I; | ||||||||
2976 | break; | ||||||||
2977 | case Instruction::Mul: | ||||||||
2978 | if (Instruction *I = foldICmpMulConstant(Cmp, BO, *C)) | ||||||||
2979 | return I; | ||||||||
2980 | break; | ||||||||
2981 | case Instruction::Shl: | ||||||||
2982 | if (Instruction *I = foldICmpShlConstant(Cmp, BO, *C)) | ||||||||
2983 | return I; | ||||||||
2984 | break; | ||||||||
2985 | case Instruction::LShr: | ||||||||
2986 | case Instruction::AShr: | ||||||||
2987 | if (Instruction *I = foldICmpShrConstant(Cmp, BO, *C)) | ||||||||
2988 | return I; | ||||||||
2989 | break; | ||||||||
2990 | case Instruction::SRem: | ||||||||
2991 | if (Instruction *I = foldICmpSRemConstant(Cmp, BO, *C)) | ||||||||
2992 | return I; | ||||||||
2993 | break; | ||||||||
2994 | case Instruction::UDiv: | ||||||||
2995 | if (Instruction *I = foldICmpUDivConstant(Cmp, BO, *C)) | ||||||||
2996 | return I; | ||||||||
2997 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||||
2998 | case Instruction::SDiv: | ||||||||
2999 | if (Instruction *I = foldICmpDivConstant(Cmp, BO, *C)) | ||||||||
3000 | return I; | ||||||||
3001 | break; | ||||||||
3002 | case Instruction::Sub: | ||||||||
3003 | if (Instruction *I = foldICmpSubConstant(Cmp, BO, *C)) | ||||||||
3004 | return I; | ||||||||
3005 | break; | ||||||||
3006 | case Instruction::Add: | ||||||||
3007 | if (Instruction *I = foldICmpAddConstant(Cmp, BO, *C)) | ||||||||
3008 | return I; | ||||||||
3009 | break; | ||||||||
3010 | default: | ||||||||
3011 | break; | ||||||||
3012 | } | ||||||||
3013 | // TODO: These folds could be refactored to be part of the above calls. | ||||||||
3014 | if (Instruction *I = foldICmpBinOpEqualityWithConstant(Cmp, BO, *C)) | ||||||||
3015 | return I; | ||||||||
3016 | } | ||||||||
3017 | |||||||||
3018 | // Match against CmpInst LHS being instructions other than binary operators. | ||||||||
3019 | |||||||||
3020 | if (auto *SI = dyn_cast<SelectInst>(Cmp.getOperand(0))) { | ||||||||
3021 | // For now, we only support constant integers while folding the | ||||||||
3022 | // ICMP(SELECT)) pattern. We can extend this to support vector of integers | ||||||||
3023 | // similar to the cases handled by binary ops above. | ||||||||
3024 | if (ConstantInt *ConstRHS = dyn_cast<ConstantInt>(Cmp.getOperand(1))) | ||||||||
3025 | if (Instruction *I = foldICmpSelectConstant(Cmp, SI, ConstRHS)) | ||||||||
3026 | return I; | ||||||||
3027 | } | ||||||||
3028 | |||||||||
3029 | if (auto *TI = dyn_cast<TruncInst>(Cmp.getOperand(0))) { | ||||||||
3030 | if (Instruction *I = foldICmpTruncConstant(Cmp, TI, *C)) | ||||||||
3031 | return I; | ||||||||
3032 | } | ||||||||
3033 | |||||||||
3034 | if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0))) | ||||||||
3035 | if (Instruction *I = foldICmpIntrinsicWithConstant(Cmp, II, *C)) | ||||||||
3036 | return I; | ||||||||
3037 | |||||||||
3038 | return nullptr; | ||||||||
3039 | } | ||||||||
3040 | |||||||||
3041 | /// Fold an icmp equality instruction with binary operator LHS and constant RHS: | ||||||||
3042 | /// icmp eq/ne BO, C. | ||||||||
3043 | Instruction *InstCombinerImpl::foldICmpBinOpEqualityWithConstant( | ||||||||
3044 | ICmpInst &Cmp, BinaryOperator *BO, const APInt &C) { | ||||||||
3045 | // TODO: Some of these folds could work with arbitrary constants, but this | ||||||||
3046 | // function is limited to scalar and vector splat constants. | ||||||||
3047 | if (!Cmp.isEquality()) | ||||||||
3048 | return nullptr; | ||||||||
3049 | |||||||||
3050 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
3051 | bool isICMP_NE = Pred == ICmpInst::ICMP_NE; | ||||||||
3052 | Constant *RHS = cast<Constant>(Cmp.getOperand(1)); | ||||||||
3053 | Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1); | ||||||||
3054 | |||||||||
3055 | switch (BO->getOpcode()) { | ||||||||
3056 | case Instruction::SRem: | ||||||||
3057 | // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one. | ||||||||
3058 | if (C.isNullValue() && BO->hasOneUse()) { | ||||||||
3059 | const APInt *BOC; | ||||||||
3060 | if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) { | ||||||||
3061 | Value *NewRem = Builder.CreateURem(BOp0, BOp1, BO->getName()); | ||||||||
3062 | return new ICmpInst(Pred, NewRem, | ||||||||
3063 | Constant::getNullValue(BO->getType())); | ||||||||
3064 | } | ||||||||
3065 | } | ||||||||
3066 | break; | ||||||||
3067 | case Instruction::Add: { | ||||||||
3068 | // Replace ((add A, B) != C) with (A != C-B) if B & C are constants. | ||||||||
3069 | if (Constant *BOC = dyn_cast<Constant>(BOp1)) { | ||||||||
3070 | if (BO->hasOneUse()) | ||||||||
3071 | return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(RHS, BOC)); | ||||||||
3072 | } else if (C.isNullValue()) { | ||||||||
3073 | // Replace ((add A, B) != 0) with (A != -B) if A or B is | ||||||||
3074 | // efficiently invertible, or if the add has just this one use. | ||||||||
3075 | if (Value *NegVal = dyn_castNegVal(BOp1)) | ||||||||
3076 | return new ICmpInst(Pred, BOp0, NegVal); | ||||||||
3077 | if (Value *NegVal = dyn_castNegVal(BOp0)) | ||||||||
3078 | return new ICmpInst(Pred, NegVal, BOp1); | ||||||||
3079 | if (BO->hasOneUse()) { | ||||||||
3080 | Value *Neg = Builder.CreateNeg(BOp1); | ||||||||
3081 | Neg->takeName(BO); | ||||||||
3082 | return new ICmpInst(Pred, BOp0, Neg); | ||||||||
3083 | } | ||||||||
3084 | } | ||||||||
3085 | break; | ||||||||
3086 | } | ||||||||
3087 | case Instruction::Xor: | ||||||||
3088 | if (BO->hasOneUse()) { | ||||||||
3089 | if (Constant *BOC = dyn_cast<Constant>(BOp1)) { | ||||||||
3090 | // For the xor case, we can xor two constants together, eliminating | ||||||||
3091 | // the explicit xor. | ||||||||
3092 | return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC)); | ||||||||
3093 | } else if (C.isNullValue()) { | ||||||||
3094 | // Replace ((xor A, B) != 0) with (A != B) | ||||||||
3095 | return new ICmpInst(Pred, BOp0, BOp1); | ||||||||
3096 | } | ||||||||
3097 | } | ||||||||
3098 | break; | ||||||||
3099 | case Instruction::Sub: | ||||||||
3100 | if (BO->hasOneUse()) { | ||||||||
3101 | // Only check for constant LHS here, as constant RHS will be canonicalized | ||||||||
3102 | // to add and use the fold above. | ||||||||
3103 | if (Constant *BOC = dyn_cast<Constant>(BOp0)) { | ||||||||
3104 | // Replace ((sub BOC, B) != C) with (B != BOC-C). | ||||||||
3105 | return new ICmpInst(Pred, BOp1, ConstantExpr::getSub(BOC, RHS)); | ||||||||
3106 | } else if (C.isNullValue()) { | ||||||||
3107 | // Replace ((sub A, B) != 0) with (A != B). | ||||||||
3108 | return new ICmpInst(Pred, BOp0, BOp1); | ||||||||
3109 | } | ||||||||
3110 | } | ||||||||
3111 | break; | ||||||||
3112 | case Instruction::Or: { | ||||||||
3113 | const APInt *BOC; | ||||||||
3114 | if (match(BOp1, m_APInt(BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) { | ||||||||
3115 | // Comparing if all bits outside of a constant mask are set? | ||||||||
3116 | // Replace (X | C) == -1 with (X & ~C) == ~C. | ||||||||
3117 | // This removes the -1 constant. | ||||||||
3118 | Constant *NotBOC = ConstantExpr::getNot(cast<Constant>(BOp1)); | ||||||||
3119 | Value *And = Builder.CreateAnd(BOp0, NotBOC); | ||||||||
3120 | return new ICmpInst(Pred, And, NotBOC); | ||||||||
3121 | } | ||||||||
3122 | break; | ||||||||
3123 | } | ||||||||
3124 | case Instruction::And: { | ||||||||
3125 | const APInt *BOC; | ||||||||
3126 | if (match(BOp1, m_APInt(BOC))) { | ||||||||
3127 | // If we have ((X & C) == C), turn it into ((X & C) != 0). | ||||||||
3128 | if (C == *BOC && C.isPowerOf2()) | ||||||||
3129 | return new ICmpInst(isICMP_NE ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE, | ||||||||
3130 | BO, Constant::getNullValue(RHS->getType())); | ||||||||
3131 | } | ||||||||
3132 | break; | ||||||||
3133 | } | ||||||||
3134 | case Instruction::UDiv: | ||||||||
3135 | if (C.isNullValue()) { | ||||||||
3136 | // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A) | ||||||||
3137 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; | ||||||||
3138 | return new ICmpInst(NewPred, BOp1, BOp0); | ||||||||
3139 | } | ||||||||
3140 | break; | ||||||||
3141 | default: | ||||||||
3142 | break; | ||||||||
3143 | } | ||||||||
3144 | return nullptr; | ||||||||
3145 | } | ||||||||
3146 | |||||||||
3147 | /// Fold an equality icmp with LLVM intrinsic and constant operand. | ||||||||
3148 | Instruction *InstCombinerImpl::foldICmpEqIntrinsicWithConstant( | ||||||||
3149 | ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) { | ||||||||
3150 | Type *Ty = II->getType(); | ||||||||
3151 | unsigned BitWidth = C.getBitWidth(); | ||||||||
3152 | switch (II->getIntrinsicID()) { | ||||||||
3153 | case Intrinsic::abs: | ||||||||
3154 | // abs(A) == 0 -> A == 0 | ||||||||
3155 | // abs(A) == INT_MIN -> A == INT_MIN | ||||||||
3156 | if (C.isNullValue() || C.isMinSignedValue()) | ||||||||
3157 | return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), | ||||||||
3158 | ConstantInt::get(Ty, C)); | ||||||||
3159 | break; | ||||||||
3160 | |||||||||
3161 | case Intrinsic::bswap: | ||||||||
3162 | // bswap(A) == C -> A == bswap(C) | ||||||||
3163 | return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), | ||||||||
3164 | ConstantInt::get(Ty, C.byteSwap())); | ||||||||
3165 | |||||||||
3166 | case Intrinsic::ctlz: | ||||||||
3167 | case Intrinsic::cttz: { | ||||||||
3168 | // ctz(A) == bitwidth(A) -> A == 0 and likewise for != | ||||||||
3169 | if (C == BitWidth) | ||||||||
3170 | return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), | ||||||||
3171 | ConstantInt::getNullValue(Ty)); | ||||||||
3172 | |||||||||
3173 | // ctz(A) == C -> A & Mask1 == Mask2, where Mask2 only has bit C set | ||||||||
3174 | // and Mask1 has bits 0..C+1 set. Similar for ctl, but for high bits. | ||||||||
3175 | // Limit to one use to ensure we don't increase instruction count. | ||||||||
3176 | unsigned Num = C.getLimitedValue(BitWidth); | ||||||||
3177 | if (Num != BitWidth && II->hasOneUse()) { | ||||||||
3178 | bool IsTrailing = II->getIntrinsicID() == Intrinsic::cttz; | ||||||||
3179 | APInt Mask1 = IsTrailing ? APInt::getLowBitsSet(BitWidth, Num + 1) | ||||||||
3180 | : APInt::getHighBitsSet(BitWidth, Num + 1); | ||||||||
3181 | APInt Mask2 = IsTrailing | ||||||||
3182 | ? APInt::getOneBitSet(BitWidth, Num) | ||||||||
3183 | : APInt::getOneBitSet(BitWidth, BitWidth - Num - 1); | ||||||||
3184 | return new ICmpInst(Cmp.getPredicate(), | ||||||||
3185 | Builder.CreateAnd(II->getArgOperand(0), Mask1), | ||||||||
3186 | ConstantInt::get(Ty, Mask2)); | ||||||||
3187 | } | ||||||||
3188 | break; | ||||||||
3189 | } | ||||||||
3190 | |||||||||
3191 | case Intrinsic::ctpop: { | ||||||||
3192 | // popcount(A) == 0 -> A == 0 and likewise for != | ||||||||
3193 | // popcount(A) == bitwidth(A) -> A == -1 and likewise for != | ||||||||
3194 | bool IsZero = C.isNullValue(); | ||||||||
3195 | if (IsZero || C == BitWidth) | ||||||||
3196 | return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), | ||||||||
3197 | IsZero ? Constant::getNullValue(Ty) : Constant::getAllOnesValue(Ty)); | ||||||||
3198 | |||||||||
3199 | break; | ||||||||
3200 | } | ||||||||
3201 | |||||||||
3202 | case Intrinsic::uadd_sat: { | ||||||||
3203 | // uadd.sat(a, b) == 0 -> (a | b) == 0 | ||||||||
3204 | if (C.isNullValue()) { | ||||||||
3205 | Value *Or = Builder.CreateOr(II->getArgOperand(0), II->getArgOperand(1)); | ||||||||
3206 | return new ICmpInst(Cmp.getPredicate(), Or, Constant::getNullValue(Ty)); | ||||||||
3207 | } | ||||||||
3208 | break; | ||||||||
3209 | } | ||||||||
3210 | |||||||||
3211 | case Intrinsic::usub_sat: { | ||||||||
3212 | // usub.sat(a, b) == 0 -> a <= b | ||||||||
3213 | if (C.isNullValue()) { | ||||||||
3214 | ICmpInst::Predicate NewPred = Cmp.getPredicate() == ICmpInst::ICMP_EQ | ||||||||
3215 | ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; | ||||||||
3216 | return new ICmpInst(NewPred, II->getArgOperand(0), II->getArgOperand(1)); | ||||||||
3217 | } | ||||||||
3218 | break; | ||||||||
3219 | } | ||||||||
3220 | default: | ||||||||
3221 | break; | ||||||||
3222 | } | ||||||||
3223 | |||||||||
3224 | return nullptr; | ||||||||
3225 | } | ||||||||
3226 | |||||||||
3227 | /// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C. | ||||||||
3228 | Instruction *InstCombinerImpl::foldICmpIntrinsicWithConstant(ICmpInst &Cmp, | ||||||||
3229 | IntrinsicInst *II, | ||||||||
3230 | const APInt &C) { | ||||||||
3231 | if (Cmp.isEquality()) | ||||||||
3232 | return foldICmpEqIntrinsicWithConstant(Cmp, II, C); | ||||||||
3233 | |||||||||
3234 | Type *Ty = II->getType(); | ||||||||
3235 | unsigned BitWidth = C.getBitWidth(); | ||||||||
3236 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
3237 | switch (II->getIntrinsicID()) { | ||||||||
3238 | case Intrinsic::ctpop: { | ||||||||
3239 | // (ctpop X > BitWidth - 1) --> X == -1 | ||||||||
3240 | Value *X = II->getArgOperand(0); | ||||||||
3241 | if (C == BitWidth - 1 && Pred == ICmpInst::ICMP_UGT) | ||||||||
3242 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, X, | ||||||||
3243 | ConstantInt::getAllOnesValue(Ty)); | ||||||||
3244 | // (ctpop X < BitWidth) --> X != -1 | ||||||||
3245 | if (C == BitWidth && Pred == ICmpInst::ICMP_ULT) | ||||||||
3246 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, X, | ||||||||
3247 | ConstantInt::getAllOnesValue(Ty)); | ||||||||
3248 | break; | ||||||||
3249 | } | ||||||||
3250 | case Intrinsic::ctlz: { | ||||||||
3251 | // ctlz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX < 0b00010000 | ||||||||
3252 | if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) { | ||||||||
3253 | unsigned Num = C.getLimitedValue(); | ||||||||
3254 | APInt Limit = APInt::getOneBitSet(BitWidth, BitWidth - Num - 1); | ||||||||
3255 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT, | ||||||||
3256 | II->getArgOperand(0), ConstantInt::get(Ty, Limit)); | ||||||||
3257 | } | ||||||||
3258 | |||||||||
3259 | // ctlz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX > 0b00011111 | ||||||||
3260 | if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) { | ||||||||
3261 | unsigned Num = C.getLimitedValue(); | ||||||||
3262 | APInt Limit = APInt::getLowBitsSet(BitWidth, BitWidth - Num); | ||||||||
3263 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, | ||||||||
3264 | II->getArgOperand(0), ConstantInt::get(Ty, Limit)); | ||||||||
3265 | } | ||||||||
3266 | break; | ||||||||
3267 | } | ||||||||
3268 | case Intrinsic::cttz: { | ||||||||
3269 | // Limit to one use to ensure we don't increase instruction count. | ||||||||
3270 | if (!II->hasOneUse()) | ||||||||
3271 | return nullptr; | ||||||||
3272 | |||||||||
3273 | // cttz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX & 0b00001111 == 0 | ||||||||
3274 | if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) { | ||||||||
3275 | APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue() + 1); | ||||||||
3276 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, | ||||||||
3277 | Builder.CreateAnd(II->getArgOperand(0), Mask), | ||||||||
3278 | ConstantInt::getNullValue(Ty)); | ||||||||
3279 | } | ||||||||
3280 | |||||||||
3281 | // cttz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX & 0b00000111 != 0 | ||||||||
3282 | if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) { | ||||||||
3283 | APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue()); | ||||||||
3284 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, | ||||||||
3285 | Builder.CreateAnd(II->getArgOperand(0), Mask), | ||||||||
3286 | ConstantInt::getNullValue(Ty)); | ||||||||
3287 | } | ||||||||
3288 | break; | ||||||||
3289 | } | ||||||||
3290 | default: | ||||||||
3291 | break; | ||||||||
3292 | } | ||||||||
3293 | |||||||||
3294 | return nullptr; | ||||||||
3295 | } | ||||||||
3296 | |||||||||
3297 | /// Handle icmp with constant (but not simple integer constant) RHS. | ||||||||
3298 | Instruction *InstCombinerImpl::foldICmpInstWithConstantNotInt(ICmpInst &I) { | ||||||||
3299 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
3300 | Constant *RHSC = dyn_cast<Constant>(Op1); | ||||||||
3301 | Instruction *LHSI = dyn_cast<Instruction>(Op0); | ||||||||
3302 | if (!RHSC || !LHSI) | ||||||||
3303 | return nullptr; | ||||||||
3304 | |||||||||
3305 | switch (LHSI->getOpcode()) { | ||||||||
3306 | case Instruction::GetElementPtr: | ||||||||
3307 | // icmp pred GEP (P, int 0, int 0, int 0), null -> icmp pred P, null | ||||||||
3308 | if (RHSC->isNullValue() && | ||||||||
3309 | cast<GetElementPtrInst>(LHSI)->hasAllZeroIndices()) | ||||||||
3310 | return new ICmpInst( | ||||||||
3311 | I.getPredicate(), LHSI->getOperand(0), | ||||||||
3312 | Constant::getNullValue(LHSI->getOperand(0)->getType())); | ||||||||
3313 | break; | ||||||||
3314 | case Instruction::PHI: | ||||||||
3315 | // Only fold icmp into the PHI if the phi and icmp are in the same | ||||||||
3316 | // block. If in the same block, we're encouraging jump threading. If | ||||||||
3317 | // not, we are just pessimizing the code by making an i1 phi. | ||||||||
3318 | if (LHSI->getParent() == I.getParent()) | ||||||||
3319 | if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI))) | ||||||||
3320 | return NV; | ||||||||
3321 | break; | ||||||||
3322 | case Instruction::Select: { | ||||||||
3323 | // If either operand of the select is a constant, we can fold the | ||||||||
3324 | // comparison into the select arms, which will cause one to be | ||||||||
3325 | // constant folded and the select turned into a bitwise or. | ||||||||
3326 | Value *Op1 = nullptr, *Op2 = nullptr; | ||||||||
3327 | ConstantInt *CI = nullptr; | ||||||||
3328 | |||||||||
3329 | auto SimplifyOp = [&](Value *V) { | ||||||||
3330 | Value *Op = nullptr; | ||||||||
3331 | if (Constant *C = dyn_cast<Constant>(V)) { | ||||||||
3332 | Op = ConstantExpr::getICmp(I.getPredicate(), C, RHSC); | ||||||||
3333 | } else if (RHSC->isNullValue()) { | ||||||||
3334 | // If null is being compared, check if it can be further simplified. | ||||||||
3335 | Op = SimplifyICmpInst(I.getPredicate(), V, RHSC, SQ); | ||||||||
3336 | } | ||||||||
3337 | return Op; | ||||||||
3338 | }; | ||||||||
3339 | Op1 = SimplifyOp(LHSI->getOperand(1)); | ||||||||
3340 | if (Op1) | ||||||||
3341 | CI = dyn_cast<ConstantInt>(Op1); | ||||||||
3342 | |||||||||
3343 | Op2 = SimplifyOp(LHSI->getOperand(2)); | ||||||||
3344 | if (Op2) | ||||||||
3345 | CI = dyn_cast<ConstantInt>(Op2); | ||||||||
3346 | |||||||||
3347 | // We only want to perform this transformation if it will not lead to | ||||||||
3348 | // additional code. This is true if either both sides of the select | ||||||||
3349 | // fold to a constant (in which case the icmp is replaced with a select | ||||||||
3350 | // which will usually simplify) or this is the only user of the | ||||||||
3351 | // select (in which case we are trading a select+icmp for a simpler | ||||||||
3352 | // select+icmp) or all uses of the select can be replaced based on | ||||||||
3353 | // dominance information ("Global cases"). | ||||||||
3354 | bool Transform = false; | ||||||||
3355 | if (Op1 && Op2) | ||||||||
3356 | Transform = true; | ||||||||
3357 | else if (Op1 || Op2) { | ||||||||
3358 | // Local case | ||||||||
3359 | if (LHSI->hasOneUse()) | ||||||||
3360 | Transform = true; | ||||||||
3361 | // Global cases | ||||||||
3362 | else if (CI && !CI->isZero()) | ||||||||
3363 | // When Op1 is constant try replacing select with second operand. | ||||||||
3364 | // Otherwise Op2 is constant and try replacing select with first | ||||||||
3365 | // operand. | ||||||||
3366 | Transform = | ||||||||
3367 | replacedSelectWithOperand(cast<SelectInst>(LHSI), &I, Op1 ? 2 : 1); | ||||||||
3368 | } | ||||||||
3369 | if (Transform) { | ||||||||
3370 | if (!Op1) | ||||||||
3371 | Op1 = Builder.CreateICmp(I.getPredicate(), LHSI->getOperand(1), RHSC, | ||||||||
3372 | I.getName()); | ||||||||
3373 | if (!Op2) | ||||||||
3374 | Op2 = Builder.CreateICmp(I.getPredicate(), LHSI->getOperand(2), RHSC, | ||||||||
3375 | I.getName()); | ||||||||
3376 | return SelectInst::Create(LHSI->getOperand(0), Op1, Op2); | ||||||||
3377 | } | ||||||||
3378 | break; | ||||||||
3379 | } | ||||||||
3380 | case Instruction::IntToPtr: | ||||||||
3381 | // icmp pred inttoptr(X), null -> icmp pred X, 0 | ||||||||
3382 | if (RHSC->isNullValue() && | ||||||||
3383 | DL.getIntPtrType(RHSC->getType()) == LHSI->getOperand(0)->getType()) | ||||||||
3384 | return new ICmpInst( | ||||||||
3385 | I.getPredicate(), LHSI->getOperand(0), | ||||||||
3386 | Constant::getNullValue(LHSI->getOperand(0)->getType())); | ||||||||
3387 | break; | ||||||||
3388 | |||||||||
3389 | case Instruction::Load: | ||||||||
3390 | // Try to optimize things like "A[i] > 4" to index computations. | ||||||||
3391 | if (GetElementPtrInst *GEP = | ||||||||
3392 | dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) { | ||||||||
3393 | if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) | ||||||||
3394 | if (GV->isConstant() && GV->hasDefinitiveInitializer() && | ||||||||
3395 | !cast<LoadInst>(LHSI)->isVolatile()) | ||||||||
3396 | if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, I)) | ||||||||
3397 | return Res; | ||||||||
3398 | } | ||||||||
3399 | break; | ||||||||
3400 | } | ||||||||
3401 | |||||||||
3402 | return nullptr; | ||||||||
3403 | } | ||||||||
3404 | |||||||||
3405 | /// Some comparisons can be simplified. | ||||||||
3406 | /// In this case, we are looking for comparisons that look like | ||||||||
3407 | /// a check for a lossy truncation. | ||||||||
3408 | /// Folds: | ||||||||
3409 | /// icmp SrcPred (x & Mask), x to icmp DstPred x, Mask | ||||||||
3410 | /// Where Mask is some pattern that produces all-ones in low bits: | ||||||||
3411 | /// (-1 >> y) | ||||||||
3412 | /// ((-1 << y) >> y) <- non-canonical, has extra uses | ||||||||
3413 | /// ~(-1 << y) | ||||||||
3414 | /// ((1 << y) + (-1)) <- non-canonical, has extra uses | ||||||||
3415 | /// The Mask can be a constant, too. | ||||||||
3416 | /// For some predicates, the operands are commutative. | ||||||||
3417 | /// For others, x can only be on a specific side. | ||||||||
3418 | static Value *foldICmpWithLowBitMaskedVal(ICmpInst &I, | ||||||||
3419 | InstCombiner::BuilderTy &Builder) { | ||||||||
3420 | ICmpInst::Predicate SrcPred; | ||||||||
3421 | Value *X, *M, *Y; | ||||||||
3422 | auto m_VariableMask = m_CombineOr( | ||||||||
3423 | m_CombineOr(m_Not(m_Shl(m_AllOnes(), m_Value())), | ||||||||
3424 | m_Add(m_Shl(m_One(), m_Value()), m_AllOnes())), | ||||||||
3425 | m_CombineOr(m_LShr(m_AllOnes(), m_Value()), | ||||||||
3426 | m_LShr(m_Shl(m_AllOnes(), m_Value(Y)), m_Deferred(Y)))); | ||||||||
3427 | auto m_Mask = m_CombineOr(m_VariableMask, m_LowBitMask()); | ||||||||
3428 | if (!match(&I, m_c_ICmp(SrcPred, | ||||||||
3429 | m_c_And(m_CombineAnd(m_Mask, m_Value(M)), m_Value(X)), | ||||||||
3430 | m_Deferred(X)))) | ||||||||
3431 | return nullptr; | ||||||||
3432 | |||||||||
3433 | ICmpInst::Predicate DstPred; | ||||||||
3434 | switch (SrcPred) { | ||||||||
3435 | case ICmpInst::Predicate::ICMP_EQ: | ||||||||
3436 | // x & (-1 >> y) == x -> x u<= (-1 >> y) | ||||||||
3437 | DstPred = ICmpInst::Predicate::ICMP_ULE; | ||||||||
3438 | break; | ||||||||
3439 | case ICmpInst::Predicate::ICMP_NE: | ||||||||
3440 | // x & (-1 >> y) != x -> x u> (-1 >> y) | ||||||||
3441 | DstPred = ICmpInst::Predicate::ICMP_UGT; | ||||||||
3442 | break; | ||||||||
3443 | case ICmpInst::Predicate::ICMP_ULT: | ||||||||
3444 | // x & (-1 >> y) u< x -> x u> (-1 >> y) | ||||||||
3445 | // x u> x & (-1 >> y) -> x u> (-1 >> y) | ||||||||
3446 | DstPred = ICmpInst::Predicate::ICMP_UGT; | ||||||||
3447 | break; | ||||||||
3448 | case ICmpInst::Predicate::ICMP_UGE: | ||||||||
3449 | // x & (-1 >> y) u>= x -> x u<= (-1 >> y) | ||||||||
3450 | // x u<= x & (-1 >> y) -> x u<= (-1 >> y) | ||||||||
3451 | DstPred = ICmpInst::Predicate::ICMP_ULE; | ||||||||
3452 | break; | ||||||||
3453 | case ICmpInst::Predicate::ICMP_SLT: | ||||||||
3454 | // x & (-1 >> y) s< x -> x s> (-1 >> y) | ||||||||
3455 | // x s> x & (-1 >> y) -> x s> (-1 >> y) | ||||||||
3456 | if (!match(M, m_Constant())) // Can not do this fold with non-constant. | ||||||||
3457 | return nullptr; | ||||||||
3458 | if (!match(M, m_NonNegative())) // Must not have any -1 vector elements. | ||||||||
3459 | return nullptr; | ||||||||
3460 | DstPred = ICmpInst::Predicate::ICMP_SGT; | ||||||||
3461 | break; | ||||||||
3462 | case ICmpInst::Predicate::ICMP_SGE: | ||||||||
3463 | // x & (-1 >> y) s>= x -> x s<= (-1 >> y) | ||||||||
3464 | // x s<= x & (-1 >> y) -> x s<= (-1 >> y) | ||||||||
3465 | if (!match(M, m_Constant())) // Can not do this fold with non-constant. | ||||||||
3466 | return nullptr; | ||||||||
3467 | if (!match(M, m_NonNegative())) // Must not have any -1 vector elements. | ||||||||
3468 | return nullptr; | ||||||||
3469 | DstPred = ICmpInst::Predicate::ICMP_SLE; | ||||||||
3470 | break; | ||||||||
3471 | case ICmpInst::Predicate::ICMP_SGT: | ||||||||
3472 | case ICmpInst::Predicate::ICMP_SLE: | ||||||||
3473 | return nullptr; | ||||||||
3474 | case ICmpInst::Predicate::ICMP_UGT: | ||||||||
3475 | case ICmpInst::Predicate::ICMP_ULE: | ||||||||
3476 | llvm_unreachable("Instsimplify took care of commut. variant")::llvm::llvm_unreachable_internal("Instsimplify took care of commut. variant" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3476); | ||||||||
3477 | break; | ||||||||
3478 | default: | ||||||||
3479 | llvm_unreachable("All possible folds are handled.")::llvm::llvm_unreachable_internal("All possible folds are handled." , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3479); | ||||||||
3480 | } | ||||||||
3481 | |||||||||
3482 | // The mask value may be a vector constant that has undefined elements. But it | ||||||||
3483 | // may not be safe to propagate those undefs into the new compare, so replace | ||||||||
3484 | // those elements by copying an existing, defined, and safe scalar constant. | ||||||||
3485 | Type *OpTy = M->getType(); | ||||||||
3486 | auto *VecC = dyn_cast<Constant>(M); | ||||||||
3487 | auto *OpVTy = dyn_cast<FixedVectorType>(OpTy); | ||||||||
3488 | if (OpVTy && VecC && VecC->containsUndefOrPoisonElement()) { | ||||||||
3489 | Constant *SafeReplacementConstant = nullptr; | ||||||||
3490 | for (unsigned i = 0, e = OpVTy->getNumElements(); i != e; ++i) { | ||||||||
3491 | if (!isa<UndefValue>(VecC->getAggregateElement(i))) { | ||||||||
3492 | SafeReplacementConstant = VecC->getAggregateElement(i); | ||||||||
3493 | break; | ||||||||
3494 | } | ||||||||
3495 | } | ||||||||
3496 | assert(SafeReplacementConstant && "Failed to find undef replacement")(static_cast <bool> (SafeReplacementConstant && "Failed to find undef replacement") ? void (0) : __assert_fail ("SafeReplacementConstant && \"Failed to find undef replacement\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3496, __extension__ __PRETTY_FUNCTION__)); | ||||||||
3497 | M = Constant::replaceUndefsWith(VecC, SafeReplacementConstant); | ||||||||
3498 | } | ||||||||
3499 | |||||||||
3500 | return Builder.CreateICmp(DstPred, X, M); | ||||||||
3501 | } | ||||||||
3502 | |||||||||
3503 | /// Some comparisons can be simplified. | ||||||||
3504 | /// In this case, we are looking for comparisons that look like | ||||||||
3505 | /// a check for a lossy signed truncation. | ||||||||
3506 | /// Folds: (MaskedBits is a constant.) | ||||||||
3507 | /// ((%x << MaskedBits) a>> MaskedBits) SrcPred %x | ||||||||
3508 | /// Into: | ||||||||
3509 | /// (add %x, (1 << (KeptBits-1))) DstPred (1 << KeptBits) | ||||||||
3510 | /// Where KeptBits = bitwidth(%x) - MaskedBits | ||||||||
3511 | static Value * | ||||||||
3512 | foldICmpWithTruncSignExtendedVal(ICmpInst &I, | ||||||||
3513 | InstCombiner::BuilderTy &Builder) { | ||||||||
3514 | ICmpInst::Predicate SrcPred; | ||||||||
3515 | Value *X; | ||||||||
3516 | const APInt *C0, *C1; // FIXME: non-splats, potentially with undef. | ||||||||
3517 | // We are ok with 'shl' having multiple uses, but 'ashr' must be one-use. | ||||||||
3518 | if (!match(&I, m_c_ICmp(SrcPred, | ||||||||
3519 | m_OneUse(m_AShr(m_Shl(m_Value(X), m_APInt(C0)), | ||||||||
3520 | m_APInt(C1))), | ||||||||
3521 | m_Deferred(X)))) | ||||||||
3522 | return nullptr; | ||||||||
3523 | |||||||||
3524 | // Potential handling of non-splats: for each element: | ||||||||
3525 | // * if both are undef, replace with constant 0. | ||||||||
3526 | // Because (1<<0) is OK and is 1, and ((1<<0)>>1) is also OK and is 0. | ||||||||
3527 | // * if both are not undef, and are different, bailout. | ||||||||
3528 | // * else, only one is undef, then pick the non-undef one. | ||||||||
3529 | |||||||||
3530 | // The shift amount must be equal. | ||||||||
3531 | if (*C0 != *C1) | ||||||||
3532 | return nullptr; | ||||||||
3533 | const APInt &MaskedBits = *C0; | ||||||||
3534 | assert(MaskedBits != 0 && "shift by zero should be folded away already.")(static_cast <bool> (MaskedBits != 0 && "shift by zero should be folded away already." ) ? void (0) : __assert_fail ("MaskedBits != 0 && \"shift by zero should be folded away already.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3534, __extension__ __PRETTY_FUNCTION__)); | ||||||||
3535 | |||||||||
3536 | ICmpInst::Predicate DstPred; | ||||||||
3537 | switch (SrcPred) { | ||||||||
3538 | case ICmpInst::Predicate::ICMP_EQ: | ||||||||
3539 | // ((%x << MaskedBits) a>> MaskedBits) == %x | ||||||||
3540 | // => | ||||||||
3541 | // (add %x, (1 << (KeptBits-1))) u< (1 << KeptBits) | ||||||||
3542 | DstPred = ICmpInst::Predicate::ICMP_ULT; | ||||||||
3543 | break; | ||||||||
3544 | case ICmpInst::Predicate::ICMP_NE: | ||||||||
3545 | // ((%x << MaskedBits) a>> MaskedBits) != %x | ||||||||
3546 | // => | ||||||||
3547 | // (add %x, (1 << (KeptBits-1))) u>= (1 << KeptBits) | ||||||||
3548 | DstPred = ICmpInst::Predicate::ICMP_UGE; | ||||||||
3549 | break; | ||||||||
3550 | // FIXME: are more folds possible? | ||||||||
3551 | default: | ||||||||
3552 | return nullptr; | ||||||||
3553 | } | ||||||||
3554 | |||||||||
3555 | auto *XType = X->getType(); | ||||||||
3556 | const unsigned XBitWidth = XType->getScalarSizeInBits(); | ||||||||
3557 | const APInt BitWidth = APInt(XBitWidth, XBitWidth); | ||||||||
3558 | assert(BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched")(static_cast <bool> (BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched") ? void (0) : __assert_fail ("BitWidth.ugt(MaskedBits) && \"shifts should leave some bits untouched\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3558, __extension__ __PRETTY_FUNCTION__)); | ||||||||
3559 | |||||||||
3560 | // KeptBits = bitwidth(%x) - MaskedBits | ||||||||
3561 | const APInt KeptBits = BitWidth - MaskedBits; | ||||||||
3562 | assert(KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable")(static_cast <bool> (KeptBits.ugt(0) && KeptBits .ult(BitWidth) && "unreachable") ? void (0) : __assert_fail ("KeptBits.ugt(0) && KeptBits.ult(BitWidth) && \"unreachable\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3562, __extension__ __PRETTY_FUNCTION__)); | ||||||||
3563 | // ICmpCst = (1 << KeptBits) | ||||||||
3564 | const APInt ICmpCst = APInt(XBitWidth, 1).shl(KeptBits); | ||||||||
3565 | assert(ICmpCst.isPowerOf2())(static_cast <bool> (ICmpCst.isPowerOf2()) ? void (0) : __assert_fail ("ICmpCst.isPowerOf2()", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3565, __extension__ __PRETTY_FUNCTION__)); | ||||||||
3566 | // AddCst = (1 << (KeptBits-1)) | ||||||||
3567 | const APInt AddCst = ICmpCst.lshr(1); | ||||||||
3568 | assert(AddCst.ult(ICmpCst) && AddCst.isPowerOf2())(static_cast <bool> (AddCst.ult(ICmpCst) && AddCst .isPowerOf2()) ? void (0) : __assert_fail ("AddCst.ult(ICmpCst) && AddCst.isPowerOf2()" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3568, __extension__ __PRETTY_FUNCTION__)); | ||||||||
3569 | |||||||||
3570 | // T0 = add %x, AddCst | ||||||||
3571 | Value *T0 = Builder.CreateAdd(X, ConstantInt::get(XType, AddCst)); | ||||||||
3572 | // T1 = T0 DstPred ICmpCst | ||||||||
3573 | Value *T1 = Builder.CreateICmp(DstPred, T0, ConstantInt::get(XType, ICmpCst)); | ||||||||
3574 | |||||||||
3575 | return T1; | ||||||||
3576 | } | ||||||||
3577 | |||||||||
3578 | // Given pattern: | ||||||||
3579 | // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0 | ||||||||
3580 | // we should move shifts to the same hand of 'and', i.e. rewrite as | ||||||||
3581 | // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x) | ||||||||
3582 | // We are only interested in opposite logical shifts here. | ||||||||
3583 | // One of the shifts can be truncated. | ||||||||
3584 | // If we can, we want to end up creating 'lshr' shift. | ||||||||
3585 | static Value * | ||||||||
3586 | foldShiftIntoShiftInAnotherHandOfAndInICmp(ICmpInst &I, const SimplifyQuery SQ, | ||||||||
3587 | InstCombiner::BuilderTy &Builder) { | ||||||||
3588 | if (!I.isEquality() || !match(I.getOperand(1), m_Zero()) || | ||||||||
3589 | !I.getOperand(0)->hasOneUse()) | ||||||||
3590 | return nullptr; | ||||||||
3591 | |||||||||
3592 | auto m_AnyLogicalShift = m_LogicalShift(m_Value(), m_Value()); | ||||||||
3593 | |||||||||
3594 | // Look for an 'and' of two logical shifts, one of which may be truncated. | ||||||||
3595 | // We use m_TruncOrSelf() on the RHS to correctly handle commutative case. | ||||||||
3596 | Instruction *XShift, *MaybeTruncation, *YShift; | ||||||||
3597 | if (!match( | ||||||||
| |||||||||
| |||||||||
3598 | I.getOperand(0), | ||||||||
3599 | m_c_And(m_CombineAnd(m_AnyLogicalShift, m_Instruction(XShift)), | ||||||||
3600 | m_CombineAnd(m_TruncOrSelf(m_CombineAnd( | ||||||||
3601 | m_AnyLogicalShift, m_Instruction(YShift))), | ||||||||
3602 | m_Instruction(MaybeTruncation))))) | ||||||||
3603 | return nullptr; | ||||||||
3604 | |||||||||
3605 | // We potentially looked past 'trunc', but only when matching YShift, | ||||||||
3606 | // therefore YShift must have the widest type. | ||||||||
3607 | Instruction *WidestShift = YShift; | ||||||||
3608 | // Therefore XShift must have the shallowest type. | ||||||||
3609 | // Or they both have identical types if there was no truncation. | ||||||||
3610 | Instruction *NarrowestShift = XShift; | ||||||||
3611 | |||||||||
3612 | Type *WidestTy = WidestShift->getType(); | ||||||||
3613 | Type *NarrowestTy = NarrowestShift->getType(); | ||||||||
3614 | assert(NarrowestTy == I.getOperand(0)->getType() &&(static_cast <bool> (NarrowestTy == I.getOperand(0)-> getType() && "We did not look past any shifts while matching XShift though." ) ? void (0) : __assert_fail ("NarrowestTy == I.getOperand(0)->getType() && \"We did not look past any shifts while matching XShift though.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3615, __extension__ __PRETTY_FUNCTION__)) | ||||||||
3615 | "We did not look past any shifts while matching XShift though.")(static_cast <bool> (NarrowestTy == I.getOperand(0)-> getType() && "We did not look past any shifts while matching XShift though." ) ? void (0) : __assert_fail ("NarrowestTy == I.getOperand(0)->getType() && \"We did not look past any shifts while matching XShift though.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3615, __extension__ __PRETTY_FUNCTION__)); | ||||||||
3616 | bool HadTrunc = WidestTy != I.getOperand(0)->getType(); | ||||||||
3617 | |||||||||
3618 | // If YShift is a 'lshr', swap the shifts around. | ||||||||
3619 | if (match(YShift, m_LShr(m_Value(), m_Value()))) | ||||||||
3620 | std::swap(XShift, YShift); | ||||||||
3621 | |||||||||
3622 | // The shifts must be in opposite directions. | ||||||||
3623 | auto XShiftOpcode = XShift->getOpcode(); | ||||||||
3624 | if (XShiftOpcode == YShift->getOpcode()) | ||||||||
3625 | return nullptr; // Do not care about same-direction shifts here. | ||||||||
3626 | |||||||||
3627 | Value *X, *XShAmt, *Y, *YShAmt; | ||||||||
3628 | match(XShift, m_BinOp(m_Value(X), m_ZExtOrSelf(m_Value(XShAmt)))); | ||||||||
3629 | match(YShift, m_BinOp(m_Value(Y), m_ZExtOrSelf(m_Value(YShAmt)))); | ||||||||
3630 | |||||||||
3631 | // If one of the values being shifted is a constant, then we will end with | ||||||||
3632 | // and+icmp, and [zext+]shift instrs will be constant-folded. If they are not, | ||||||||
3633 | // however, we will need to ensure that we won't increase instruction count. | ||||||||
3634 | if (!isa<Constant>(X) && !isa<Constant>(Y)) { | ||||||||
3635 | // At least one of the hands of the 'and' should be one-use shift. | ||||||||
3636 | if (!match(I.getOperand(0), | ||||||||
3637 | m_c_And(m_OneUse(m_AnyLogicalShift), m_Value()))) | ||||||||
3638 | return nullptr; | ||||||||
3639 | if (HadTrunc) { | ||||||||
3640 | // Due to the 'trunc', we will need to widen X. For that either the old | ||||||||
3641 | // 'trunc' or the shift amt in the non-truncated shift should be one-use. | ||||||||
3642 | if (!MaybeTruncation->hasOneUse() && | ||||||||
3643 | !NarrowestShift->getOperand(1)->hasOneUse()) | ||||||||
3644 | return nullptr; | ||||||||
3645 | } | ||||||||
3646 | } | ||||||||
3647 | |||||||||
3648 | // We have two shift amounts from two different shifts. The types of those | ||||||||
3649 | // shift amounts may not match. If that's the case let's bailout now. | ||||||||
3650 | if (XShAmt->getType() != YShAmt->getType()) | ||||||||
| |||||||||
3651 | return nullptr; | ||||||||
3652 | |||||||||
3653 | // As input, we have the following pattern: | ||||||||
3654 | // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0 | ||||||||
3655 | // We want to rewrite that as: | ||||||||
3656 | // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x) | ||||||||
3657 | // While we know that originally (Q+K) would not overflow | ||||||||
3658 | // (because 2 * (N-1) u<= iN -1), we have looked past extensions of | ||||||||
3659 | // shift amounts. so it may now overflow in smaller bitwidth. | ||||||||
3660 | // To ensure that does not happen, we need to ensure that the total maximal | ||||||||
3661 | // shift amount is still representable in that smaller bit width. | ||||||||
3662 | unsigned MaximalPossibleTotalShiftAmount = | ||||||||
3663 | (WidestTy->getScalarSizeInBits() - 1) + | ||||||||
3664 | (NarrowestTy->getScalarSizeInBits() - 1); | ||||||||
3665 | APInt MaximalRepresentableShiftAmount = | ||||||||
3666 | APInt::getAllOnesValue(XShAmt->getType()->getScalarSizeInBits()); | ||||||||
3667 | if (MaximalRepresentableShiftAmount.ult(MaximalPossibleTotalShiftAmount)) | ||||||||
3668 | return nullptr; | ||||||||
3669 | |||||||||
3670 | // Can we fold (XShAmt+YShAmt) ? | ||||||||
3671 | auto *NewShAmt = dyn_cast_or_null<Constant>( | ||||||||
3672 | SimplifyAddInst(XShAmt, YShAmt, /*isNSW=*/false, | ||||||||
3673 | /*isNUW=*/false, SQ.getWithInstruction(&I))); | ||||||||
3674 | if (!NewShAmt) | ||||||||
3675 | return nullptr; | ||||||||
3676 | NewShAmt = ConstantExpr::getZExtOrBitCast(NewShAmt, WidestTy); | ||||||||
3677 | unsigned WidestBitWidth = WidestTy->getScalarSizeInBits(); | ||||||||
3678 | |||||||||
3679 | // Is the new shift amount smaller than the bit width? | ||||||||
3680 | // FIXME: could also rely on ConstantRange. | ||||||||
3681 | if (!match(NewShAmt, | ||||||||
3682 | m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_ULT, | ||||||||
3683 | APInt(WidestBitWidth, WidestBitWidth)))) | ||||||||
3684 | return nullptr; | ||||||||
3685 | |||||||||
3686 | // An extra legality check is needed if we had trunc-of-lshr. | ||||||||
3687 | if (HadTrunc && match(WidestShift, m_LShr(m_Value(), m_Value()))) { | ||||||||
3688 | auto CanFold = [NewShAmt, WidestBitWidth, NarrowestShift, SQ, | ||||||||
3689 | WidestShift]() { | ||||||||
3690 | // It isn't obvious whether it's worth it to analyze non-constants here. | ||||||||
3691 | // Also, let's basically give up on non-splat cases, pessimizing vectors. | ||||||||
3692 | // If *any* of these preconditions matches we can perform the fold. | ||||||||
3693 | Constant *NewShAmtSplat = NewShAmt->getType()->isVectorTy() | ||||||||
3694 | ? NewShAmt->getSplatValue() | ||||||||
3695 | : NewShAmt; | ||||||||
3696 | // If it's edge-case shift (by 0 or by WidestBitWidth-1) we can fold. | ||||||||
3697 | if (NewShAmtSplat && | ||||||||
3698 | (NewShAmtSplat->isNullValue() || | ||||||||
3699 | NewShAmtSplat->getUniqueInteger() == WidestBitWidth - 1)) | ||||||||
3700 | return true; | ||||||||
3701 | // We consider *min* leading zeros so a single outlier | ||||||||
3702 | // blocks the transform as opposed to allowing it. | ||||||||
3703 | if (auto *C = dyn_cast<Constant>(NarrowestShift->getOperand(0))) { | ||||||||
3704 | KnownBits Known = computeKnownBits(C, SQ.DL); | ||||||||
3705 | unsigned MinLeadZero = Known.countMinLeadingZeros(); | ||||||||
3706 | // If the value being shifted has at most lowest bit set we can fold. | ||||||||
3707 | unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero; | ||||||||
3708 | if (MaxActiveBits <= 1) | ||||||||
3709 | return true; | ||||||||
3710 | // Precondition: NewShAmt u<= countLeadingZeros(C) | ||||||||
3711 | if (NewShAmtSplat && NewShAmtSplat->getUniqueInteger().ule(MinLeadZero)) | ||||||||
3712 | return true; | ||||||||
3713 | } | ||||||||
3714 | if (auto *C = dyn_cast<Constant>(WidestShift->getOperand(0))) { | ||||||||
3715 | KnownBits Known = computeKnownBits(C, SQ.DL); | ||||||||
3716 | unsigned MinLeadZero = Known.countMinLeadingZeros(); | ||||||||
3717 | // If the value being shifted has at most lowest bit set we can fold. | ||||||||
3718 | unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero; | ||||||||
3719 | if (MaxActiveBits <= 1) | ||||||||
3720 | return true; | ||||||||
3721 | // Precondition: ((WidestBitWidth-1)-NewShAmt) u<= countLeadingZeros(C) | ||||||||
3722 | if (NewShAmtSplat) { | ||||||||
3723 | APInt AdjNewShAmt = | ||||||||
3724 | (WidestBitWidth - 1) - NewShAmtSplat->getUniqueInteger(); | ||||||||
3725 | if (AdjNewShAmt.ule(MinLeadZero)) | ||||||||
3726 | return true; | ||||||||
3727 | } | ||||||||
3728 | } | ||||||||
3729 | return false; // Can't tell if it's ok. | ||||||||
3730 | }; | ||||||||
3731 | if (!CanFold()) | ||||||||
3732 | return nullptr; | ||||||||
3733 | } | ||||||||
3734 | |||||||||
3735 | // All good, we can do this fold. | ||||||||
3736 | X = Builder.CreateZExt(X, WidestTy); | ||||||||
3737 | Y = Builder.CreateZExt(Y, WidestTy); | ||||||||
3738 | // The shift is the same that was for X. | ||||||||
3739 | Value *T0 = XShiftOpcode == Instruction::BinaryOps::LShr | ||||||||
3740 | ? Builder.CreateLShr(X, NewShAmt) | ||||||||
3741 | : Builder.CreateShl(X, NewShAmt); | ||||||||
3742 | Value *T1 = Builder.CreateAnd(T0, Y); | ||||||||
3743 | return Builder.CreateICmp(I.getPredicate(), T1, | ||||||||
3744 | Constant::getNullValue(WidestTy)); | ||||||||
3745 | } | ||||||||
3746 | |||||||||
3747 | /// Fold | ||||||||
3748 | /// (-1 u/ x) u< y | ||||||||
3749 | /// ((x * y) u/ x) != y | ||||||||
3750 | /// to | ||||||||
3751 | /// @llvm.umul.with.overflow(x, y) plus extraction of overflow bit | ||||||||
3752 | /// Note that the comparison is commutative, while inverted (u>=, ==) predicate | ||||||||
3753 | /// will mean that we are looking for the opposite answer. | ||||||||
3754 | Value *InstCombinerImpl::foldUnsignedMultiplicationOverflowCheck(ICmpInst &I) { | ||||||||
3755 | ICmpInst::Predicate Pred; | ||||||||
3756 | Value *X, *Y; | ||||||||
3757 | Instruction *Mul; | ||||||||
3758 | bool NeedNegation; | ||||||||
3759 | // Look for: (-1 u/ x) u</u>= y | ||||||||
3760 | if (!I.isEquality() && | ||||||||
3761 | match(&I, m_c_ICmp(Pred, m_OneUse(m_UDiv(m_AllOnes(), m_Value(X))), | ||||||||
3762 | m_Value(Y)))) { | ||||||||
3763 | Mul = nullptr; | ||||||||
3764 | |||||||||
3765 | // Are we checking that overflow does not happen, or does happen? | ||||||||
3766 | switch (Pred) { | ||||||||
3767 | case ICmpInst::Predicate::ICMP_ULT: | ||||||||
3768 | NeedNegation = false; | ||||||||
3769 | break; // OK | ||||||||
3770 | case ICmpInst::Predicate::ICMP_UGE: | ||||||||
3771 | NeedNegation = true; | ||||||||
3772 | break; // OK | ||||||||
3773 | default: | ||||||||
3774 | return nullptr; // Wrong predicate. | ||||||||
3775 | } | ||||||||
3776 | } else // Look for: ((x * y) u/ x) !=/== y | ||||||||
3777 | if (I.isEquality() && | ||||||||
3778 | match(&I, m_c_ICmp(Pred, m_Value(Y), | ||||||||
3779 | m_OneUse(m_UDiv(m_CombineAnd(m_c_Mul(m_Deferred(Y), | ||||||||
3780 | m_Value(X)), | ||||||||
3781 | m_Instruction(Mul)), | ||||||||
3782 | m_Deferred(X)))))) { | ||||||||
3783 | NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ; | ||||||||
3784 | } else | ||||||||
3785 | return nullptr; | ||||||||
3786 | |||||||||
3787 | BuilderTy::InsertPointGuard Guard(Builder); | ||||||||
3788 | // If the pattern included (x * y), we'll want to insert new instructions | ||||||||
3789 | // right before that original multiplication so that we can replace it. | ||||||||
3790 | bool MulHadOtherUses = Mul && !Mul->hasOneUse(); | ||||||||
3791 | if (MulHadOtherUses) | ||||||||
3792 | Builder.SetInsertPoint(Mul); | ||||||||
3793 | |||||||||
3794 | Function *F = Intrinsic::getDeclaration( | ||||||||
3795 | I.getModule(), Intrinsic::umul_with_overflow, X->getType()); | ||||||||
3796 | CallInst *Call = Builder.CreateCall(F, {X, Y}, "umul"); | ||||||||
3797 | |||||||||
3798 | // If the multiplication was used elsewhere, to ensure that we don't leave | ||||||||
3799 | // "duplicate" instructions, replace uses of that original multiplication | ||||||||
3800 | // with the multiplication result from the with.overflow intrinsic. | ||||||||
3801 | if (MulHadOtherUses) | ||||||||
3802 | replaceInstUsesWith(*Mul, Builder.CreateExtractValue(Call, 0, "umul.val")); | ||||||||
3803 | |||||||||
3804 | Value *Res = Builder.CreateExtractValue(Call, 1, "umul.ov"); | ||||||||
3805 | if (NeedNegation) // This technically increases instruction count. | ||||||||
3806 | Res = Builder.CreateNot(Res, "umul.not.ov"); | ||||||||
3807 | |||||||||
3808 | // If we replaced the mul, erase it. Do this after all uses of Builder, | ||||||||
3809 | // as the mul is used as insertion point. | ||||||||
3810 | if (MulHadOtherUses) | ||||||||
3811 | eraseInstFromFunction(*Mul); | ||||||||
3812 | |||||||||
3813 | return Res; | ||||||||
3814 | } | ||||||||
3815 | |||||||||
3816 | static Instruction *foldICmpXNegX(ICmpInst &I) { | ||||||||
3817 | CmpInst::Predicate Pred; | ||||||||
3818 | Value *X; | ||||||||
3819 | if (!match(&I, m_c_ICmp(Pred, m_NSWNeg(m_Value(X)), m_Deferred(X)))) | ||||||||
3820 | return nullptr; | ||||||||
3821 | |||||||||
3822 | if (ICmpInst::isSigned(Pred)) | ||||||||
3823 | Pred = ICmpInst::getSwappedPredicate(Pred); | ||||||||
3824 | else if (ICmpInst::isUnsigned(Pred)) | ||||||||
3825 | Pred = ICmpInst::getSignedPredicate(Pred); | ||||||||
3826 | // else for equality-comparisons just keep the predicate. | ||||||||
3827 | |||||||||
3828 | return ICmpInst::Create(Instruction::ICmp, Pred, X, | ||||||||
3829 | Constant::getNullValue(X->getType()), I.getName()); | ||||||||
3830 | } | ||||||||
3831 | |||||||||
3832 | /// Try to fold icmp (binop), X or icmp X, (binop). | ||||||||
3833 | /// TODO: A large part of this logic is duplicated in InstSimplify's | ||||||||
3834 | /// simplifyICmpWithBinOp(). We should be able to share that and avoid the code | ||||||||
3835 | /// duplication. | ||||||||
3836 | Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I, | ||||||||
3837 | const SimplifyQuery &SQ) { | ||||||||
3838 | const SimplifyQuery Q = SQ.getWithInstruction(&I); | ||||||||
3839 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
3840 | |||||||||
3841 | // Special logic for binary operators. | ||||||||
3842 | BinaryOperator *BO0 = dyn_cast<BinaryOperator>(Op0); | ||||||||
| |||||||||
3843 | BinaryOperator *BO1 = dyn_cast<BinaryOperator>(Op1); | ||||||||
3844 | if (!BO0
| ||||||||
3845 | return nullptr; | ||||||||
3846 | |||||||||
3847 | if (Instruction *NewICmp
| ||||||||
3848 | return NewICmp; | ||||||||
3849 | |||||||||
3850 | const CmpInst::Predicate Pred = I.getPredicate(); | ||||||||
3851 | Value *X; | ||||||||
3852 | |||||||||
3853 | // Convert add-with-unsigned-overflow comparisons into a 'not' with compare. | ||||||||
3854 | // (Op1 + X) u</u>= Op1 --> ~Op1 u</u>= X | ||||||||
3855 | if (match(Op0, m_OneUse(m_c_Add(m_Specific(Op1), m_Value(X)))) && | ||||||||
3856 | (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) | ||||||||
3857 | return new ICmpInst(Pred, Builder.CreateNot(Op1), X); | ||||||||
3858 | // Op0 u>/u<= (Op0 + X) --> X u>/u<= ~Op0 | ||||||||
3859 | if (match(Op1, m_OneUse(m_c_Add(m_Specific(Op0), m_Value(X)))) && | ||||||||
3860 | (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) | ||||||||
3861 | return new ICmpInst(Pred, X, Builder.CreateNot(Op0)); | ||||||||
3862 | |||||||||
3863 | bool NoOp0WrapProblem = false, NoOp1WrapProblem = false; | ||||||||
3864 | if (BO0
| ||||||||
3865 | NoOp0WrapProblem = | ||||||||
3866 | ICmpInst::isEquality(Pred) || | ||||||||
3867 | (CmpInst::isUnsigned(Pred) && BO0->hasNoUnsignedWrap()) || | ||||||||
3868 | (CmpInst::isSigned(Pred) && BO0->hasNoSignedWrap()); | ||||||||
3869 | if (BO1
| ||||||||
3870 | NoOp1WrapProblem = | ||||||||
3871 | ICmpInst::isEquality(Pred) || | ||||||||
3872 | (CmpInst::isUnsigned(Pred) && BO1->hasNoUnsignedWrap()) || | ||||||||
3873 | (CmpInst::isSigned(Pred) && BO1->hasNoSignedWrap()); | ||||||||
3874 | |||||||||
3875 | // Analyze the case when either Op0 or Op1 is an add instruction. | ||||||||
3876 | // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null). | ||||||||
3877 | Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr; | ||||||||
3878 | if (BO0
| ||||||||
3879 | A = BO0->getOperand(0); | ||||||||
3880 | B = BO0->getOperand(1); | ||||||||
3881 | } | ||||||||
3882 | if (BO1
| ||||||||
3883 | C = BO1->getOperand(0); | ||||||||
3884 | D = BO1->getOperand(1); | ||||||||
3885 | } | ||||||||
3886 | |||||||||
3887 | // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow. | ||||||||
3888 | // icmp (A+B), B -> icmp A, 0 for equalities or if there is no overflow. | ||||||||
3889 | if ((A
| ||||||||
3890 | return new ICmpInst(Pred, A == Op1 ? B : A, | ||||||||
3891 | Constant::getNullValue(Op1->getType())); | ||||||||
3892 | |||||||||
3893 | // icmp C, (C+D) -> icmp 0, D for equalities or if there is no overflow. | ||||||||
3894 | // icmp D, (C+D) -> icmp 0, C for equalities or if there is no overflow. | ||||||||
3895 | if ((C
| ||||||||
3896 | return new ICmpInst(Pred, Constant::getNullValue(Op0->getType()), | ||||||||
3897 | C == Op0 ? D : C); | ||||||||
3898 | |||||||||
3899 | // icmp (A+B), (A+D) -> icmp B, D for equalities or if there is no overflow. | ||||||||
3900 | if (A
| ||||||||
3901 | NoOp1WrapProblem) { | ||||||||
3902 | // Determine Y and Z in the form icmp (X+Y), (X+Z). | ||||||||
3903 | Value *Y, *Z; | ||||||||
3904 | if (A == C) { | ||||||||
3905 | // C + B == C + D -> B == D | ||||||||
3906 | Y = B; | ||||||||
3907 | Z = D; | ||||||||
3908 | } else if (A == D) { | ||||||||
3909 | // D + B == C + D -> B == C | ||||||||
3910 | Y = B; | ||||||||
3911 | Z = C; | ||||||||
3912 | } else if (B == C) { | ||||||||
3913 | // A + C == C + D -> A == D | ||||||||
3914 | Y = A; | ||||||||
3915 | Z = D; | ||||||||
3916 | } else { | ||||||||
3917 | assert(B == D)(static_cast <bool> (B == D) ? void (0) : __assert_fail ("B == D", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3917, __extension__ __PRETTY_FUNCTION__)); | ||||||||
3918 | // A + D == C + D -> A == C | ||||||||
3919 | Y = A; | ||||||||
3920 | Z = C; | ||||||||
3921 | } | ||||||||
3922 | return new ICmpInst(Pred, Y, Z); | ||||||||
3923 | } | ||||||||
3924 | |||||||||
3925 | // icmp slt (A + -1), Op1 -> icmp sle A, Op1 | ||||||||
3926 | if (A
| ||||||||
3927 | match(B, m_AllOnes())) | ||||||||
3928 | return new ICmpInst(CmpInst::ICMP_SLE, A, Op1); | ||||||||
3929 | |||||||||
3930 | // icmp sge (A + -1), Op1 -> icmp sgt A, Op1 | ||||||||
3931 | if (A
| ||||||||
3932 | match(B, m_AllOnes())) | ||||||||
3933 | return new ICmpInst(CmpInst::ICMP_SGT, A, Op1); | ||||||||
3934 | |||||||||
3935 | // icmp sle (A + 1), Op1 -> icmp slt A, Op1 | ||||||||
3936 | if (A
| ||||||||
3937 | return new ICmpInst(CmpInst::ICMP_SLT, A, Op1); | ||||||||
3938 | |||||||||
3939 | // icmp sgt (A + 1), Op1 -> icmp sge A, Op1 | ||||||||
3940 | if (A
| ||||||||
3941 | return new ICmpInst(CmpInst::ICMP_SGE, A, Op1); | ||||||||
3942 | |||||||||
3943 | // icmp sgt Op0, (C + -1) -> icmp sge Op0, C | ||||||||
3944 | if (C
| ||||||||
3945 | match(D, m_AllOnes())) | ||||||||
3946 | return new ICmpInst(CmpInst::ICMP_SGE, Op0, C); | ||||||||
3947 | |||||||||
3948 | // icmp sle Op0, (C + -1) -> icmp slt Op0, C | ||||||||
3949 | if (C
| ||||||||
3950 | match(D, m_AllOnes())) | ||||||||
3951 | return new ICmpInst(CmpInst::ICMP_SLT, Op0, C); | ||||||||
3952 | |||||||||
3953 | // icmp sge Op0, (C + 1) -> icmp sgt Op0, C | ||||||||
3954 | if (C
| ||||||||
3955 | return new ICmpInst(CmpInst::ICMP_SGT, Op0, C); | ||||||||
3956 | |||||||||
3957 | // icmp slt Op0, (C + 1) -> icmp sle Op0, C | ||||||||
3958 | if (C
| ||||||||
3959 | return new ICmpInst(CmpInst::ICMP_SLE, Op0, C); | ||||||||
3960 | |||||||||
3961 | // TODO: The subtraction-related identities shown below also hold, but | ||||||||
3962 | // canonicalization from (X -nuw 1) to (X + -1) means that the combinations | ||||||||
3963 | // wouldn't happen even if they were implemented. | ||||||||
3964 | // | ||||||||
3965 | // icmp ult (A - 1), Op1 -> icmp ule A, Op1 | ||||||||
3966 | // icmp uge (A - 1), Op1 -> icmp ugt A, Op1 | ||||||||
3967 | // icmp ugt Op0, (C - 1) -> icmp uge Op0, C | ||||||||
3968 | // icmp ule Op0, (C - 1) -> icmp ult Op0, C | ||||||||
3969 | |||||||||
3970 | // icmp ule (A + 1), Op0 -> icmp ult A, Op1 | ||||||||
3971 | if (A
| ||||||||
3972 | return new ICmpInst(CmpInst::ICMP_ULT, A, Op1); | ||||||||
3973 | |||||||||
3974 | // icmp ugt (A + 1), Op0 -> icmp uge A, Op1 | ||||||||
3975 | if (A
| ||||||||
3976 | return new ICmpInst(CmpInst::ICMP_UGE, A, Op1); | ||||||||
3977 | |||||||||
3978 | // icmp uge Op0, (C + 1) -> icmp ugt Op0, C | ||||||||
3979 | if (C
| ||||||||
3980 | return new ICmpInst(CmpInst::ICMP_UGT, Op0, C); | ||||||||
3981 | |||||||||
3982 | // icmp ult Op0, (C + 1) -> icmp ule Op0, C | ||||||||
3983 | if (C
| ||||||||
3984 | return new ICmpInst(CmpInst::ICMP_ULE, Op0, C); | ||||||||
3985 | |||||||||
3986 | // if C1 has greater magnitude than C2: | ||||||||
3987 | // icmp (A + C1), (C + C2) -> icmp (A + C3), C | ||||||||
3988 | // s.t. C3 = C1 - C2 | ||||||||
3989 | // | ||||||||
3990 | // if C2 has greater magnitude than C1: | ||||||||
3991 | // icmp (A + C1), (C + C2) -> icmp A, (C + C3) | ||||||||
3992 | // s.t. C3 = C2 - C1 | ||||||||
3993 | if (A
| ||||||||
3994 | (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned()) | ||||||||
3995 | if (ConstantInt *C1 = dyn_cast<ConstantInt>(B)) | ||||||||
3996 | if (ConstantInt *C2 = dyn_cast<ConstantInt>(D)) { | ||||||||
3997 | const APInt &AP1 = C1->getValue(); | ||||||||
3998 | const APInt &AP2 = C2->getValue(); | ||||||||
3999 | if (AP1.isNegative() == AP2.isNegative()) { | ||||||||
4000 | APInt AP1Abs = C1->getValue().abs(); | ||||||||
4001 | APInt AP2Abs = C2->getValue().abs(); | ||||||||
4002 | if (AP1Abs.uge(AP2Abs)) { | ||||||||
4003 | ConstantInt *C3 = Builder.getInt(AP1 - AP2); | ||||||||
4004 | bool HasNUW = BO0->hasNoUnsignedWrap() && C3->getValue().ule(AP1); | ||||||||
4005 | bool HasNSW = BO0->hasNoSignedWrap(); | ||||||||
4006 | Value *NewAdd = Builder.CreateAdd(A, C3, "", HasNUW, HasNSW); | ||||||||
4007 | return new ICmpInst(Pred, NewAdd, C); | ||||||||
4008 | } else { | ||||||||
4009 | ConstantInt *C3 = Builder.getInt(AP2 - AP1); | ||||||||
4010 | bool HasNUW = BO1->hasNoUnsignedWrap() && C3->getValue().ule(AP2); | ||||||||
4011 | bool HasNSW = BO1->hasNoSignedWrap(); | ||||||||
4012 | Value *NewAdd = Builder.CreateAdd(C, C3, "", HasNUW, HasNSW); | ||||||||
4013 | return new ICmpInst(Pred, A, NewAdd); | ||||||||
4014 | } | ||||||||
4015 | } | ||||||||
4016 | } | ||||||||
4017 | |||||||||
4018 | // Analyze the case when either Op0 or Op1 is a sub instruction. | ||||||||
4019 | // Op0 = A - B (or A and B are null); Op1 = C - D (or C and D are null). | ||||||||
4020 | A = nullptr; | ||||||||
4021 | B = nullptr; | ||||||||
4022 | C = nullptr; | ||||||||
4023 | D = nullptr; | ||||||||
4024 | if (BO0
| ||||||||
4025 | A = BO0->getOperand(0); | ||||||||
4026 | B = BO0->getOperand(1); | ||||||||
4027 | } | ||||||||
4028 | if (BO1
| ||||||||
4029 | C = BO1->getOperand(0); | ||||||||
4030 | D = BO1->getOperand(1); | ||||||||
4031 | } | ||||||||
4032 | |||||||||
4033 | // icmp (A-B), A -> icmp 0, B for equalities or if there is no overflow. | ||||||||
4034 | if (A
| ||||||||
4035 | return new ICmpInst(Pred, Constant::getNullValue(Op1->getType()), B); | ||||||||
4036 | // icmp C, (C-D) -> icmp D, 0 for equalities or if there is no overflow. | ||||||||
4037 | if (C
| ||||||||
4038 | return new ICmpInst(Pred, D, Constant::getNullValue(Op0->getType())); | ||||||||
4039 | |||||||||
4040 | // Convert sub-with-unsigned-overflow comparisons into a comparison of args. | ||||||||
4041 | // (A - B) u>/u<= A --> B u>/u<= A | ||||||||
4042 | if (A
| ||||||||
4043 | return new ICmpInst(Pred, B, A); | ||||||||
4044 | // C u</u>= (C - D) --> C u</u>= D | ||||||||
4045 | if (C
| ||||||||
4046 | return new ICmpInst(Pred, C, D); | ||||||||
4047 | // (A - B) u>=/u< A --> B u>/u<= A iff B != 0 | ||||||||
4048 | if (A
| ||||||||
4049 | isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) | ||||||||
4050 | return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), B, A); | ||||||||
4051 | // C u<=/u> (C - D) --> C u</u>= D iff B != 0 | ||||||||
4052 | if (C
| ||||||||
4053 | isKnownNonZero(D, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) | ||||||||
4054 | return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), C, D); | ||||||||
4055 | |||||||||
4056 | // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow. | ||||||||
4057 | if (B
| ||||||||
4058 | return new ICmpInst(Pred, A, C); | ||||||||
4059 | |||||||||
4060 | // icmp (A-B), (A-D) -> icmp D, B for equalities or if there is no overflow. | ||||||||
4061 | if (A
| ||||||||
4062 | return new ICmpInst(Pred, D, B); | ||||||||
4063 | |||||||||
4064 | // icmp (0-X) < cst --> x > -cst | ||||||||
4065 | if (NoOp0WrapProblem
| ||||||||
4066 | Value *X; | ||||||||
4067 | if (match(BO0, m_Neg(m_Value(X)))) | ||||||||
4068 | if (Constant *RHSC = dyn_cast<Constant>(Op1)) | ||||||||
4069 | if (RHSC->isNotMinSignedValue()) | ||||||||
4070 | return new ICmpInst(I.getSwappedPredicate(), X, | ||||||||
4071 | ConstantExpr::getNeg(RHSC)); | ||||||||
4072 | } | ||||||||
4073 | |||||||||
4074 | { | ||||||||
4075 | // Try to remove shared constant multiplier from equality comparison: | ||||||||
4076 | // X * C == Y * C (with no overflowing/aliasing) --> X == Y | ||||||||
4077 | Value *X, *Y; | ||||||||
4078 | const APInt *C; | ||||||||
4079 | if (match(Op0, m_Mul(m_Value(X), m_APInt(C))) && *C != 0 && | ||||||||
4080 | match(Op1, m_Mul(m_Value(Y), m_SpecificInt(*C))) && I.isEquality()) | ||||||||
4081 | if (!C->countTrailingZeros() || | ||||||||
4082 | (BO0->hasNoSignedWrap() && BO1->hasNoSignedWrap()) || | ||||||||
4083 | (BO0->hasNoUnsignedWrap() && BO1->hasNoUnsignedWrap())) | ||||||||
4084 | return new ICmpInst(Pred, X, Y); | ||||||||
4085 | } | ||||||||
4086 | |||||||||
4087 | BinaryOperator *SRem = nullptr; | ||||||||
4088 | // icmp (srem X, Y), Y | ||||||||
4089 | if (BO0
| ||||||||
4090 | SRem = BO0; | ||||||||
4091 | // icmp Y, (srem X, Y) | ||||||||
4092 | else if (BO1
| ||||||||
4093 | Op0 == BO1->getOperand(1)) | ||||||||
4094 | SRem = BO1; | ||||||||
4095 | if (SRem
| ||||||||
4096 | // We don't check hasOneUse to avoid increasing register pressure because | ||||||||
4097 | // the value we use is the same value this instruction was already using. | ||||||||
4098 | switch (SRem == BO0 ? ICmpInst::getSwappedPredicate(Pred) : Pred) { | ||||||||
4099 | default: | ||||||||
4100 | break; | ||||||||
4101 | case ICmpInst::ICMP_EQ: | ||||||||
4102 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
4103 | case ICmpInst::ICMP_NE: | ||||||||
4104 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
4105 | case ICmpInst::ICMP_SGT: | ||||||||
4106 | case ICmpInst::ICMP_SGE: | ||||||||
4107 | return new ICmpInst(ICmpInst::ICMP_SGT, SRem->getOperand(1), | ||||||||
4108 | Constant::getAllOnesValue(SRem->getType())); | ||||||||
4109 | case ICmpInst::ICMP_SLT: | ||||||||
4110 | case ICmpInst::ICMP_SLE: | ||||||||
4111 | return new ICmpInst(ICmpInst::ICMP_SLT, SRem->getOperand(1), | ||||||||
4112 | Constant::getNullValue(SRem->getType())); | ||||||||
4113 | } | ||||||||
4114 | } | ||||||||
4115 | |||||||||
4116 | if (BO0
| ||||||||
4117 | BO1->hasOneUse() && BO0->getOperand(1) == BO1->getOperand(1)) { | ||||||||
4118 | switch (BO0->getOpcode()) { | ||||||||
4119 | default: | ||||||||
4120 | break; | ||||||||
4121 | case Instruction::Add: | ||||||||
4122 | case Instruction::Sub: | ||||||||
4123 | case Instruction::Xor: { | ||||||||
4124 | if (I.isEquality()) // a+x icmp eq/ne b+x --> a icmp b | ||||||||
4125 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
4126 | |||||||||
4127 | const APInt *C; | ||||||||
4128 | if (match(BO0->getOperand(1), m_APInt(C))) { | ||||||||
4129 | // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b | ||||||||
4130 | if (C->isSignMask()) { | ||||||||
4131 | ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate(); | ||||||||
4132 | return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
4133 | } | ||||||||
4134 | |||||||||
4135 | // icmp u/s (a ^ maxsignval), (b ^ maxsignval) --> icmp s/u' a, b | ||||||||
4136 | if (BO0->getOpcode() == Instruction::Xor && C->isMaxSignedValue()) { | ||||||||
4137 | ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate(); | ||||||||
4138 | NewPred = I.getSwappedPredicate(NewPred); | ||||||||
4139 | return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
4140 | } | ||||||||
4141 | } | ||||||||
4142 | break; | ||||||||
4143 | } | ||||||||
4144 | case Instruction::Mul: { | ||||||||
4145 | if (!I.isEquality()) | ||||||||
4146 | break; | ||||||||
4147 | |||||||||
4148 | const APInt *C; | ||||||||
4149 | if (match(BO0->getOperand(1), m_APInt(C)) && !C->isNullValue() && | ||||||||
4150 | !C->isOneValue()) { | ||||||||
4151 | // icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask) | ||||||||
4152 | // Mask = -1 >> count-trailing-zeros(C). | ||||||||
4153 | if (unsigned TZs = C->countTrailingZeros()) { | ||||||||
4154 | Constant *Mask = ConstantInt::get( | ||||||||
4155 | BO0->getType(), | ||||||||
4156 | APInt::getLowBitsSet(C->getBitWidth(), C->getBitWidth() - TZs)); | ||||||||
4157 | Value *And1 = Builder.CreateAnd(BO0->getOperand(0), Mask); | ||||||||
4158 | Value *And2 = Builder.CreateAnd(BO1->getOperand(0), Mask); | ||||||||
4159 | return new ICmpInst(Pred, And1, And2); | ||||||||
4160 | } | ||||||||
4161 | } | ||||||||
4162 | break; | ||||||||
4163 | } | ||||||||
4164 | case Instruction::UDiv: | ||||||||
4165 | case Instruction::LShr: | ||||||||
4166 | if (I.isSigned() || !BO0->isExact() || !BO1->isExact()) | ||||||||
4167 | break; | ||||||||
4168 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
4169 | |||||||||
4170 | case Instruction::SDiv: | ||||||||
4171 | if (!I.isEquality() || !BO0->isExact() || !BO1->isExact()) | ||||||||
4172 | break; | ||||||||
4173 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
4174 | |||||||||
4175 | case Instruction::AShr: | ||||||||
4176 | if (!BO0->isExact() || !BO1->isExact()) | ||||||||
4177 | break; | ||||||||
4178 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
4179 | |||||||||
4180 | case Instruction::Shl: { | ||||||||
4181 | bool NUW = BO0->hasNoUnsignedWrap() && BO1->hasNoUnsignedWrap(); | ||||||||
4182 | bool NSW = BO0->hasNoSignedWrap() && BO1->hasNoSignedWrap(); | ||||||||
4183 | if (!NUW && !NSW) | ||||||||
4184 | break; | ||||||||
4185 | if (!NSW && I.isSigned()) | ||||||||
4186 | break; | ||||||||
4187 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
4188 | } | ||||||||
4189 | } | ||||||||
4190 | } | ||||||||
4191 | |||||||||
4192 | if (BO0
| ||||||||
4193 | // Transform A & (L - 1) `ult` L --> L != 0 | ||||||||
4194 | auto LSubOne = m_Add(m_Specific(Op1), m_AllOnes()); | ||||||||
4195 | auto BitwiseAnd = m_c_And(m_Value(), LSubOne); | ||||||||
4196 | |||||||||
4197 | if (match(BO0, BitwiseAnd) && Pred == ICmpInst::ICMP_ULT) { | ||||||||
4198 | auto *Zero = Constant::getNullValue(BO0->getType()); | ||||||||
4199 | return new ICmpInst(ICmpInst::ICMP_NE, Op1, Zero); | ||||||||
4200 | } | ||||||||
4201 | } | ||||||||
4202 | |||||||||
4203 | if (Value *V
| ||||||||
4204 | return replaceInstUsesWith(I, V); | ||||||||
4205 | |||||||||
4206 | if (Value *V
| ||||||||
4207 | return replaceInstUsesWith(I, V); | ||||||||
4208 | |||||||||
4209 | if (Value *V
| ||||||||
4210 | return replaceInstUsesWith(I, V); | ||||||||
4211 | |||||||||
4212 | if (Value *V = foldShiftIntoShiftInAnotherHandOfAndInICmp(I, SQ, Builder)) | ||||||||
4213 | return replaceInstUsesWith(I, V); | ||||||||
4214 | |||||||||
4215 | return nullptr; | ||||||||
4216 | } | ||||||||
4217 | |||||||||
4218 | /// Fold icmp Pred min|max(X, Y), X. | ||||||||
4219 | static Instruction *foldICmpWithMinMax(ICmpInst &Cmp) { | ||||||||
4220 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
4221 | Value *Op0 = Cmp.getOperand(0); | ||||||||
4222 | Value *X = Cmp.getOperand(1); | ||||||||
4223 | |||||||||
4224 | // Canonicalize minimum or maximum operand to LHS of the icmp. | ||||||||
4225 | if (match(X, m_c_SMin(m_Specific(Op0), m_Value())) || | ||||||||
4226 | match(X, m_c_SMax(m_Specific(Op0), m_Value())) || | ||||||||
4227 | match(X, m_c_UMin(m_Specific(Op0), m_Value())) || | ||||||||
4228 | match(X, m_c_UMax(m_Specific(Op0), m_Value()))) { | ||||||||
4229 | std::swap(Op0, X); | ||||||||
4230 | Pred = Cmp.getSwappedPredicate(); | ||||||||
4231 | } | ||||||||
4232 | |||||||||
4233 | Value *Y; | ||||||||
4234 | if (match(Op0, m_c_SMin(m_Specific(X), m_Value(Y)))) { | ||||||||
4235 | // smin(X, Y) == X --> X s<= Y | ||||||||
4236 | // smin(X, Y) s>= X --> X s<= Y | ||||||||
4237 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SGE) | ||||||||
4238 | return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); | ||||||||
4239 | |||||||||
4240 | // smin(X, Y) != X --> X s> Y | ||||||||
4241 | // smin(X, Y) s< X --> X s> Y | ||||||||
4242 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SLT) | ||||||||
4243 | return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); | ||||||||
4244 | |||||||||
4245 | // These cases should be handled in InstSimplify: | ||||||||
4246 | // smin(X, Y) s<= X --> true | ||||||||
4247 | // smin(X, Y) s> X --> false | ||||||||
4248 | return nullptr; | ||||||||
4249 | } | ||||||||
4250 | |||||||||
4251 | if (match(Op0, m_c_SMax(m_Specific(X), m_Value(Y)))) { | ||||||||
4252 | // smax(X, Y) == X --> X s>= Y | ||||||||
4253 | // smax(X, Y) s<= X --> X s>= Y | ||||||||
4254 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SLE) | ||||||||
4255 | return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); | ||||||||
4256 | |||||||||
4257 | // smax(X, Y) != X --> X s< Y | ||||||||
4258 | // smax(X, Y) s> X --> X s< Y | ||||||||
4259 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SGT) | ||||||||
4260 | return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); | ||||||||
4261 | |||||||||
4262 | // These cases should be handled in InstSimplify: | ||||||||
4263 | // smax(X, Y) s>= X --> true | ||||||||
4264 | // smax(X, Y) s< X --> false | ||||||||
4265 | return nullptr; | ||||||||
4266 | } | ||||||||
4267 | |||||||||
4268 | if (match(Op0, m_c_UMin(m_Specific(X), m_Value(Y)))) { | ||||||||
4269 | // umin(X, Y) == X --> X u<= Y | ||||||||
4270 | // umin(X, Y) u>= X --> X u<= Y | ||||||||
4271 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_UGE) | ||||||||
4272 | return new ICmpInst(ICmpInst::ICMP_ULE, X, Y); | ||||||||
4273 | |||||||||
4274 | // umin(X, Y) != X --> X u> Y | ||||||||
4275 | // umin(X, Y) u< X --> X u> Y | ||||||||
4276 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_ULT) | ||||||||
4277 | return new ICmpInst(ICmpInst::ICMP_UGT, X, Y); | ||||||||
4278 | |||||||||
4279 | // These cases should be handled in InstSimplify: | ||||||||
4280 | // umin(X, Y) u<= X --> true | ||||||||
4281 | // umin(X, Y) u> X --> false | ||||||||
4282 | return nullptr; | ||||||||
4283 | } | ||||||||
4284 | |||||||||
4285 | if (match(Op0, m_c_UMax(m_Specific(X), m_Value(Y)))) { | ||||||||
4286 | // umax(X, Y) == X --> X u>= Y | ||||||||
4287 | // umax(X, Y) u<= X --> X u>= Y | ||||||||
4288 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_ULE) | ||||||||
4289 | return new ICmpInst(ICmpInst::ICMP_UGE, X, Y); | ||||||||
4290 | |||||||||
4291 | // umax(X, Y) != X --> X u< Y | ||||||||
4292 | // umax(X, Y) u> X --> X u< Y | ||||||||
4293 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_UGT) | ||||||||
4294 | return new ICmpInst(ICmpInst::ICMP_ULT, X, Y); | ||||||||
4295 | |||||||||
4296 | // These cases should be handled in InstSimplify: | ||||||||
4297 | // umax(X, Y) u>= X --> true | ||||||||
4298 | // umax(X, Y) u< X --> false | ||||||||
4299 | return nullptr; | ||||||||
4300 | } | ||||||||
4301 | |||||||||
4302 | return nullptr; | ||||||||
4303 | } | ||||||||
4304 | |||||||||
4305 | Instruction *InstCombinerImpl::foldICmpEquality(ICmpInst &I) { | ||||||||
4306 | if (!I.isEquality()) | ||||||||
4307 | return nullptr; | ||||||||
4308 | |||||||||
4309 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
4310 | const CmpInst::Predicate Pred = I.getPredicate(); | ||||||||
4311 | Value *A, *B, *C, *D; | ||||||||
4312 | if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) { | ||||||||
4313 | if (A == Op1 || B == Op1) { // (A^B) == A -> B == 0 | ||||||||
4314 | Value *OtherVal = A == Op1 ? B : A; | ||||||||
4315 | return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType())); | ||||||||
4316 | } | ||||||||
4317 | |||||||||
4318 | if (match(Op1, m_Xor(m_Value(C), m_Value(D)))) { | ||||||||
4319 | // A^c1 == C^c2 --> A == C^(c1^c2) | ||||||||
4320 | ConstantInt *C1, *C2; | ||||||||
4321 | if (match(B, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2)) && | ||||||||
4322 | Op1->hasOneUse()) { | ||||||||
4323 | Constant *NC = Builder.getInt(C1->getValue() ^ C2->getValue()); | ||||||||
4324 | Value *Xor = Builder.CreateXor(C, NC); | ||||||||
4325 | return new ICmpInst(Pred, A, Xor); | ||||||||
4326 | } | ||||||||
4327 | |||||||||
4328 | // A^B == A^D -> B == D | ||||||||
4329 | if (A == C) | ||||||||
4330 | return new ICmpInst(Pred, B, D); | ||||||||
4331 | if (A == D) | ||||||||
4332 | return new ICmpInst(Pred, B, C); | ||||||||
4333 | if (B == C) | ||||||||
4334 | return new ICmpInst(Pred, A, D); | ||||||||
4335 | if (B == D) | ||||||||
4336 | return new ICmpInst(Pred, A, C); | ||||||||
4337 | } | ||||||||
4338 | } | ||||||||
4339 | |||||||||
4340 | if (match(Op1, m_Xor(m_Value(A), m_Value(B))) && (A == Op0 || B == Op0)) { | ||||||||
4341 | // A == (A^B) -> B == 0 | ||||||||
4342 | Value *OtherVal = A == Op0 ? B : A; | ||||||||
4343 | return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType())); | ||||||||
4344 | } | ||||||||
4345 | |||||||||
4346 | // (X&Z) == (Y&Z) -> (X^Y) & Z == 0 | ||||||||
4347 | if (match(Op0, m_OneUse(m_And(m_Value(A), m_Value(B)))) && | ||||||||
4348 | match(Op1, m_OneUse(m_And(m_Value(C), m_Value(D))))) { | ||||||||
4349 | Value *X = nullptr, *Y = nullptr, *Z = nullptr; | ||||||||
4350 | |||||||||
4351 | if (A == C) { | ||||||||
4352 | X = B; | ||||||||
4353 | Y = D; | ||||||||
4354 | Z = A; | ||||||||
4355 | } else if (A == D) { | ||||||||
4356 | X = B; | ||||||||
4357 | Y = C; | ||||||||
4358 | Z = A; | ||||||||
4359 | } else if (B == C) { | ||||||||
4360 | X = A; | ||||||||
4361 | Y = D; | ||||||||
4362 | Z = B; | ||||||||
4363 | } else if (B == D) { | ||||||||
4364 | X = A; | ||||||||
4365 | Y = C; | ||||||||
4366 | Z = B; | ||||||||
4367 | } | ||||||||
4368 | |||||||||
4369 | if (X) { // Build (X^Y) & Z | ||||||||
4370 | Op1 = Builder.CreateXor(X, Y); | ||||||||
4371 | Op1 = Builder.CreateAnd(Op1, Z); | ||||||||
4372 | return new ICmpInst(Pred, Op1, Constant::getNullValue(Op1->getType())); | ||||||||
4373 | } | ||||||||
4374 | } | ||||||||
4375 | |||||||||
4376 | // Transform (zext A) == (B & (1<<X)-1) --> A == (trunc B) | ||||||||
4377 | // and (B & (1<<X)-1) == (zext A) --> A == (trunc B) | ||||||||
4378 | ConstantInt *Cst1; | ||||||||
4379 | if ((Op0->hasOneUse() && match(Op0, m_ZExt(m_Value(A))) && | ||||||||
4380 | match(Op1, m_And(m_Value(B), m_ConstantInt(Cst1)))) || | ||||||||
4381 | (Op1->hasOneUse() && match(Op0, m_And(m_Value(B), m_ConstantInt(Cst1))) && | ||||||||
4382 | match(Op1, m_ZExt(m_Value(A))))) { | ||||||||
4383 | APInt Pow2 = Cst1->getValue() + 1; | ||||||||
4384 | if (Pow2.isPowerOf2() && isa<IntegerType>(A->getType()) && | ||||||||
4385 | Pow2.logBase2() == cast<IntegerType>(A->getType())->getBitWidth()) | ||||||||
4386 | return new ICmpInst(Pred, A, Builder.CreateTrunc(B, A->getType())); | ||||||||
4387 | } | ||||||||
4388 | |||||||||
4389 | // (A >> C) == (B >> C) --> (A^B) u< (1 << C) | ||||||||
4390 | // For lshr and ashr pairs. | ||||||||
4391 | if ((match(Op0, m_OneUse(m_LShr(m_Value(A), m_ConstantInt(Cst1)))) && | ||||||||
4392 | match(Op1, m_OneUse(m_LShr(m_Value(B), m_Specific(Cst1))))) || | ||||||||
4393 | (match(Op0, m_OneUse(m_AShr(m_Value(A), m_ConstantInt(Cst1)))) && | ||||||||
4394 | match(Op1, m_OneUse(m_AShr(m_Value(B), m_Specific(Cst1)))))) { | ||||||||
4395 | unsigned TypeBits = Cst1->getBitWidth(); | ||||||||
4396 | unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits); | ||||||||
4397 | if (ShAmt < TypeBits && ShAmt != 0) { | ||||||||
4398 | ICmpInst::Predicate NewPred = | ||||||||
4399 | Pred == ICmpInst::ICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; | ||||||||
4400 | Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted"); | ||||||||
4401 | APInt CmpVal = APInt::getOneBitSet(TypeBits, ShAmt); | ||||||||
4402 | return new ICmpInst(NewPred, Xor, Builder.getInt(CmpVal)); | ||||||||
4403 | } | ||||||||
4404 | } | ||||||||
4405 | |||||||||
4406 | // (A << C) == (B << C) --> ((A^B) & (~0U >> C)) == 0 | ||||||||
4407 | if (match(Op0, m_OneUse(m_Shl(m_Value(A), m_ConstantInt(Cst1)))) && | ||||||||
4408 | match(Op1, m_OneUse(m_Shl(m_Value(B), m_Specific(Cst1))))) { | ||||||||
4409 | unsigned TypeBits = Cst1->getBitWidth(); | ||||||||
4410 | unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits); | ||||||||
4411 | if (ShAmt < TypeBits && ShAmt != 0) { | ||||||||
4412 | Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted"); | ||||||||
4413 | APInt AndVal = APInt::getLowBitsSet(TypeBits, TypeBits - ShAmt); | ||||||||
4414 | Value *And = Builder.CreateAnd(Xor, Builder.getInt(AndVal), | ||||||||
4415 | I.getName() + ".mask"); | ||||||||
4416 | return new ICmpInst(Pred, And, Constant::getNullValue(Cst1->getType())); | ||||||||
4417 | } | ||||||||
4418 | } | ||||||||
4419 | |||||||||
4420 | // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to | ||||||||
4421 | // "icmp (and X, mask), cst" | ||||||||
4422 | uint64_t ShAmt = 0; | ||||||||
4423 | if (Op0->hasOneUse() && | ||||||||
4424 | match(Op0, m_Trunc(m_OneUse(m_LShr(m_Value(A), m_ConstantInt(ShAmt))))) && | ||||||||
4425 | match(Op1, m_ConstantInt(Cst1)) && | ||||||||
4426 | // Only do this when A has multiple uses. This is most important to do | ||||||||
4427 | // when it exposes other optimizations. | ||||||||
4428 | !A->hasOneUse()) { | ||||||||
4429 | unsigned ASize = cast<IntegerType>(A->getType())->getPrimitiveSizeInBits(); | ||||||||
4430 | |||||||||
4431 | if (ShAmt < ASize) { | ||||||||
4432 | APInt MaskV = | ||||||||
4433 | APInt::getLowBitsSet(ASize, Op0->getType()->getPrimitiveSizeInBits()); | ||||||||
4434 | MaskV <<= ShAmt; | ||||||||
4435 | |||||||||
4436 | APInt CmpV = Cst1->getValue().zext(ASize); | ||||||||
4437 | CmpV <<= ShAmt; | ||||||||
4438 | |||||||||
4439 | Value *Mask = Builder.CreateAnd(A, Builder.getInt(MaskV)); | ||||||||
4440 | return new ICmpInst(Pred, Mask, Builder.getInt(CmpV)); | ||||||||
4441 | } | ||||||||
4442 | } | ||||||||
4443 | |||||||||
4444 | // If both operands are byte-swapped or bit-reversed, just compare the | ||||||||
4445 | // original values. | ||||||||
4446 | // TODO: Move this to a function similar to foldICmpIntrinsicWithConstant() | ||||||||
4447 | // and handle more intrinsics. | ||||||||
4448 | if ((match(Op0, m_BSwap(m_Value(A))) && match(Op1, m_BSwap(m_Value(B)))) || | ||||||||
4449 | (match(Op0, m_BitReverse(m_Value(A))) && | ||||||||
4450 | match(Op1, m_BitReverse(m_Value(B))))) | ||||||||
4451 | return new ICmpInst(Pred, A, B); | ||||||||
4452 | |||||||||
4453 | // Canonicalize checking for a power-of-2-or-zero value: | ||||||||
4454 | // (A & (A-1)) == 0 --> ctpop(A) < 2 (two commuted variants) | ||||||||
4455 | // ((A-1) & A) != 0 --> ctpop(A) > 1 (two commuted variants) | ||||||||
4456 | if (!match(Op0, m_OneUse(m_c_And(m_Add(m_Value(A), m_AllOnes()), | ||||||||
4457 | m_Deferred(A)))) || | ||||||||
4458 | !match(Op1, m_ZeroInt())) | ||||||||
4459 | A = nullptr; | ||||||||
4460 | |||||||||
4461 | // (A & -A) == A --> ctpop(A) < 2 (four commuted variants) | ||||||||
4462 | // (-A & A) != A --> ctpop(A) > 1 (four commuted variants) | ||||||||
4463 | if (match(Op0, m_OneUse(m_c_And(m_Neg(m_Specific(Op1)), m_Specific(Op1))))) | ||||||||
4464 | A = Op1; | ||||||||
4465 | else if (match(Op1, | ||||||||
4466 | m_OneUse(m_c_And(m_Neg(m_Specific(Op0)), m_Specific(Op0))))) | ||||||||
4467 | A = Op0; | ||||||||
4468 | |||||||||
4469 | if (A) { | ||||||||
4470 | Type *Ty = A->getType(); | ||||||||
4471 | CallInst *CtPop = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, A); | ||||||||
4472 | return Pred == ICmpInst::ICMP_EQ | ||||||||
4473 | ? new ICmpInst(ICmpInst::ICMP_ULT, CtPop, ConstantInt::get(Ty, 2)) | ||||||||
4474 | : new ICmpInst(ICmpInst::ICMP_UGT, CtPop, ConstantInt::get(Ty, 1)); | ||||||||
4475 | } | ||||||||
4476 | |||||||||
4477 | return nullptr; | ||||||||
4478 | } | ||||||||
4479 | |||||||||
4480 | static Instruction *foldICmpWithZextOrSext(ICmpInst &ICmp, | ||||||||
4481 | InstCombiner::BuilderTy &Builder) { | ||||||||
4482 | assert(isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0")(static_cast <bool> (isa<CastInst>(ICmp.getOperand (0)) && "Expected cast for operand 0") ? void (0) : __assert_fail ("isa<CastInst>(ICmp.getOperand(0)) && \"Expected cast for operand 0\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4482, __extension__ __PRETTY_FUNCTION__)); | ||||||||
4483 | auto *CastOp0 = cast<CastInst>(ICmp.getOperand(0)); | ||||||||
4484 | Value *X; | ||||||||
4485 | if (!match(CastOp0, m_ZExtOrSExt(m_Value(X)))) | ||||||||
4486 | return nullptr; | ||||||||
4487 | |||||||||
4488 | bool IsSignedExt = CastOp0->getOpcode() == Instruction::SExt; | ||||||||
4489 | bool IsSignedCmp = ICmp.isSigned(); | ||||||||
4490 | if (auto *CastOp1 = dyn_cast<CastInst>(ICmp.getOperand(1))) { | ||||||||
4491 | // If the signedness of the two casts doesn't agree (i.e. one is a sext | ||||||||
4492 | // and the other is a zext), then we can't handle this. | ||||||||
4493 | // TODO: This is too strict. We can handle some predicates (equality?). | ||||||||
4494 | if (CastOp0->getOpcode() != CastOp1->getOpcode()) | ||||||||
4495 | return nullptr; | ||||||||
4496 | |||||||||
4497 | // Not an extension from the same type? | ||||||||
4498 | Value *Y = CastOp1->getOperand(0); | ||||||||
4499 | Type *XTy = X->getType(), *YTy = Y->getType(); | ||||||||
4500 | if (XTy != YTy) { | ||||||||
4501 | // One of the casts must have one use because we are creating a new cast. | ||||||||
4502 | if (!CastOp0->hasOneUse() && !CastOp1->hasOneUse()) | ||||||||
4503 | return nullptr; | ||||||||
4504 | // Extend the narrower operand to the type of the wider operand. | ||||||||
4505 | if (XTy->getScalarSizeInBits() < YTy->getScalarSizeInBits()) | ||||||||
4506 | X = Builder.CreateCast(CastOp0->getOpcode(), X, YTy); | ||||||||
4507 | else if (YTy->getScalarSizeInBits() < XTy->getScalarSizeInBits()) | ||||||||
4508 | Y = Builder.CreateCast(CastOp0->getOpcode(), Y, XTy); | ||||||||
4509 | else | ||||||||
4510 | return nullptr; | ||||||||
4511 | } | ||||||||
4512 | |||||||||
4513 | // (zext X) == (zext Y) --> X == Y | ||||||||
4514 | // (sext X) == (sext Y) --> X == Y | ||||||||
4515 | if (ICmp.isEquality()) | ||||||||
4516 | return new ICmpInst(ICmp.getPredicate(), X, Y); | ||||||||
4517 | |||||||||
4518 | // A signed comparison of sign extended values simplifies into a | ||||||||
4519 | // signed comparison. | ||||||||
4520 | if (IsSignedCmp && IsSignedExt) | ||||||||
4521 | return new ICmpInst(ICmp.getPredicate(), X, Y); | ||||||||
4522 | |||||||||
4523 | // The other three cases all fold into an unsigned comparison. | ||||||||
4524 | return new ICmpInst(ICmp.getUnsignedPredicate(), X, Y); | ||||||||
4525 | } | ||||||||
4526 | |||||||||
4527 | // Below here, we are only folding a compare with constant. | ||||||||
4528 | auto *C = dyn_cast<Constant>(ICmp.getOperand(1)); | ||||||||
4529 | if (!C) | ||||||||
4530 | return nullptr; | ||||||||
4531 | |||||||||
4532 | // Compute the constant that would happen if we truncated to SrcTy then | ||||||||
4533 | // re-extended to DestTy. | ||||||||
4534 | Type *SrcTy = CastOp0->getSrcTy(); | ||||||||
4535 | Type *DestTy = CastOp0->getDestTy(); | ||||||||
4536 | Constant *Res1 = ConstantExpr::getTrunc(C, SrcTy); | ||||||||
4537 | Constant *Res2 = ConstantExpr::getCast(CastOp0->getOpcode(), Res1, DestTy); | ||||||||
4538 | |||||||||
4539 | // If the re-extended constant didn't change... | ||||||||
4540 | if (Res2 == C) { | ||||||||
4541 | if (ICmp.isEquality()) | ||||||||
4542 | return new ICmpInst(ICmp.getPredicate(), X, Res1); | ||||||||
4543 | |||||||||
4544 | // A signed comparison of sign extended values simplifies into a | ||||||||
4545 | // signed comparison. | ||||||||
4546 | if (IsSignedExt && IsSignedCmp) | ||||||||
4547 | return new ICmpInst(ICmp.getPredicate(), X, Res1); | ||||||||
4548 | |||||||||
4549 | // The other three cases all fold into an unsigned comparison. | ||||||||
4550 | return new ICmpInst(ICmp.getUnsignedPredicate(), X, Res1); | ||||||||
4551 | } | ||||||||
4552 | |||||||||
4553 | // The re-extended constant changed, partly changed (in the case of a vector), | ||||||||
4554 | // or could not be determined to be equal (in the case of a constant | ||||||||
4555 | // expression), so the constant cannot be represented in the shorter type. | ||||||||
4556 | // All the cases that fold to true or false will have already been handled | ||||||||
4557 | // by SimplifyICmpInst, so only deal with the tricky case. | ||||||||
4558 | if (IsSignedCmp || !IsSignedExt || !isa<ConstantInt>(C)) | ||||||||
4559 | return nullptr; | ||||||||
4560 | |||||||||
4561 | // Is source op positive? | ||||||||
4562 | // icmp ult (sext X), C --> icmp sgt X, -1 | ||||||||
4563 | if (ICmp.getPredicate() == ICmpInst::ICMP_ULT) | ||||||||
4564 | return new ICmpInst(CmpInst::ICMP_SGT, X, Constant::getAllOnesValue(SrcTy)); | ||||||||
4565 | |||||||||
4566 | // Is source op negative? | ||||||||
4567 | // icmp ugt (sext X), C --> icmp slt X, 0 | ||||||||
4568 | assert(ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!")(static_cast <bool> (ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!") ? void (0) : __assert_fail ("ICmp.getPredicate() == ICmpInst::ICMP_UGT && \"ICmp should be folded!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4568, __extension__ __PRETTY_FUNCTION__)); | ||||||||
4569 | return new ICmpInst(CmpInst::ICMP_SLT, X, Constant::getNullValue(SrcTy)); | ||||||||
4570 | } | ||||||||
4571 | |||||||||
4572 | /// Handle icmp (cast x), (cast or constant). | ||||||||
4573 | Instruction *InstCombinerImpl::foldICmpWithCastOp(ICmpInst &ICmp) { | ||||||||
4574 | // If any operand of ICmp is a inttoptr roundtrip cast then remove it as | ||||||||
4575 | // icmp compares only pointer's value. | ||||||||
4576 | // icmp (inttoptr (ptrtoint p1)), p2 --> icmp p1, p2. | ||||||||
4577 | Value *SimplifiedOp0 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(0)); | ||||||||
4578 | Value *SimplifiedOp1 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(1)); | ||||||||
4579 | if (SimplifiedOp0 || SimplifiedOp1) | ||||||||
4580 | return new ICmpInst(ICmp.getPredicate(), | ||||||||
4581 | SimplifiedOp0 ? SimplifiedOp0 : ICmp.getOperand(0), | ||||||||
4582 | SimplifiedOp1 ? SimplifiedOp1 : ICmp.getOperand(1)); | ||||||||
4583 | |||||||||
4584 | auto *CastOp0 = dyn_cast<CastInst>(ICmp.getOperand(0)); | ||||||||
4585 | if (!CastOp0) | ||||||||
4586 | return nullptr; | ||||||||
4587 | if (!isa<Constant>(ICmp.getOperand(1)) && !isa<CastInst>(ICmp.getOperand(1))) | ||||||||
4588 | return nullptr; | ||||||||
4589 | |||||||||
4590 | Value *Op0Src = CastOp0->getOperand(0); | ||||||||
4591 | Type *SrcTy = CastOp0->getSrcTy(); | ||||||||
4592 | Type *DestTy = CastOp0->getDestTy(); | ||||||||
4593 | |||||||||
4594 | // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the | ||||||||
4595 | // integer type is the same size as the pointer type. | ||||||||
4596 | auto CompatibleSizes = [&](Type *SrcTy, Type *DestTy) { | ||||||||
4597 | if (isa<VectorType>(SrcTy)) { | ||||||||
4598 | SrcTy = cast<VectorType>(SrcTy)->getElementType(); | ||||||||
4599 | DestTy = cast<VectorType>(DestTy)->getElementType(); | ||||||||
4600 | } | ||||||||
4601 | return DL.getPointerTypeSizeInBits(SrcTy) == DestTy->getIntegerBitWidth(); | ||||||||
4602 | }; | ||||||||
4603 | if (CastOp0->getOpcode() == Instruction::PtrToInt && | ||||||||
4604 | CompatibleSizes(SrcTy, DestTy)) { | ||||||||
4605 | Value *NewOp1 = nullptr; | ||||||||
4606 | if (auto *PtrToIntOp1 = dyn_cast<PtrToIntOperator>(ICmp.getOperand(1))) { | ||||||||
4607 | Value *PtrSrc = PtrToIntOp1->getOperand(0); | ||||||||
4608 | if (PtrSrc->getType()->getPointerAddressSpace() == | ||||||||
4609 | Op0Src->getType()->getPointerAddressSpace()) { | ||||||||
4610 | NewOp1 = PtrToIntOp1->getOperand(0); | ||||||||
4611 | // If the pointer types don't match, insert a bitcast. | ||||||||
4612 | if (Op0Src->getType() != NewOp1->getType()) | ||||||||
4613 | NewOp1 = Builder.CreateBitCast(NewOp1, Op0Src->getType()); | ||||||||
4614 | } | ||||||||
4615 | } else if (auto *RHSC = dyn_cast<Constant>(ICmp.getOperand(1))) { | ||||||||
4616 | NewOp1 = ConstantExpr::getIntToPtr(RHSC, SrcTy); | ||||||||
4617 | } | ||||||||
4618 | |||||||||
4619 | if (NewOp1) | ||||||||
4620 | return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1); | ||||||||
4621 | } | ||||||||
4622 | |||||||||
4623 | return foldICmpWithZextOrSext(ICmp, Builder); | ||||||||
4624 | } | ||||||||
4625 | |||||||||
4626 | static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS) { | ||||||||
4627 | switch (BinaryOp) { | ||||||||
4628 | default: | ||||||||
4629 | llvm_unreachable("Unsupported binary op")::llvm::llvm_unreachable_internal("Unsupported binary op", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4629); | ||||||||
4630 | case Instruction::Add: | ||||||||
4631 | case Instruction::Sub: | ||||||||
4632 | return match(RHS, m_Zero()); | ||||||||
4633 | case Instruction::Mul: | ||||||||
4634 | return match(RHS, m_One()); | ||||||||
4635 | } | ||||||||
4636 | } | ||||||||
4637 | |||||||||
4638 | OverflowResult | ||||||||
4639 | InstCombinerImpl::computeOverflow(Instruction::BinaryOps BinaryOp, | ||||||||
4640 | bool IsSigned, Value *LHS, Value *RHS, | ||||||||
4641 | Instruction *CxtI) const { | ||||||||
4642 | switch (BinaryOp) { | ||||||||
4643 | default: | ||||||||
4644 | llvm_unreachable("Unsupported binary op")::llvm::llvm_unreachable_internal("Unsupported binary op", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4644); | ||||||||
4645 | case Instruction::Add: | ||||||||
4646 | if (IsSigned) | ||||||||
4647 | return computeOverflowForSignedAdd(LHS, RHS, CxtI); | ||||||||
4648 | else | ||||||||
4649 | return computeOverflowForUnsignedAdd(LHS, RHS, CxtI); | ||||||||
4650 | case Instruction::Sub: | ||||||||
4651 | if (IsSigned) | ||||||||
4652 | return computeOverflowForSignedSub(LHS, RHS, CxtI); | ||||||||
4653 | else | ||||||||
4654 | return computeOverflowForUnsignedSub(LHS, RHS, CxtI); | ||||||||
4655 | case Instruction::Mul: | ||||||||
4656 | if (IsSigned) | ||||||||
4657 | return computeOverflowForSignedMul(LHS, RHS, CxtI); | ||||||||
4658 | else | ||||||||
4659 | return computeOverflowForUnsignedMul(LHS, RHS, CxtI); | ||||||||
4660 | } | ||||||||
4661 | } | ||||||||
4662 | |||||||||
4663 | bool InstCombinerImpl::OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp, | ||||||||
4664 | bool IsSigned, Value *LHS, | ||||||||
4665 | Value *RHS, Instruction &OrigI, | ||||||||
4666 | Value *&Result, | ||||||||
4667 | Constant *&Overflow) { | ||||||||
4668 | if (OrigI.isCommutative() && isa<Constant>(LHS) && !isa<Constant>(RHS)) | ||||||||
4669 | std::swap(LHS, RHS); | ||||||||
4670 | |||||||||
4671 | // If the overflow check was an add followed by a compare, the insertion point | ||||||||
4672 | // may be pointing to the compare. We want to insert the new instructions | ||||||||
4673 | // before the add in case there are uses of the add between the add and the | ||||||||
4674 | // compare. | ||||||||
4675 | Builder.SetInsertPoint(&OrigI); | ||||||||
4676 | |||||||||
4677 | Type *OverflowTy = Type::getInt1Ty(LHS->getContext()); | ||||||||
4678 | if (auto *LHSTy = dyn_cast<VectorType>(LHS->getType())) | ||||||||
4679 | OverflowTy = VectorType::get(OverflowTy, LHSTy->getElementCount()); | ||||||||
4680 | |||||||||
4681 | if (isNeutralValue(BinaryOp, RHS)) { | ||||||||
4682 | Result = LHS; | ||||||||
4683 | Overflow = ConstantInt::getFalse(OverflowTy); | ||||||||
4684 | return true; | ||||||||
4685 | } | ||||||||
4686 | |||||||||
4687 | switch (computeOverflow(BinaryOp, IsSigned, LHS, RHS, &OrigI)) { | ||||||||
4688 | case OverflowResult::MayOverflow: | ||||||||
4689 | return false; | ||||||||
4690 | case OverflowResult::AlwaysOverflowsLow: | ||||||||
4691 | case OverflowResult::AlwaysOverflowsHigh: | ||||||||
4692 | Result = Builder.CreateBinOp(BinaryOp, LHS, RHS); | ||||||||
4693 | Result->takeName(&OrigI); | ||||||||
4694 | Overflow = ConstantInt::getTrue(OverflowTy); | ||||||||
4695 | return true; | ||||||||
4696 | case OverflowResult::NeverOverflows: | ||||||||
4697 | Result = Builder.CreateBinOp(BinaryOp, LHS, RHS); | ||||||||
4698 | Result->takeName(&OrigI); | ||||||||
4699 | Overflow = ConstantInt::getFalse(OverflowTy); | ||||||||
4700 | if (auto *Inst = dyn_cast<Instruction>(Result)) { | ||||||||
4701 | if (IsSigned) | ||||||||
4702 | Inst->setHasNoSignedWrap(); | ||||||||
4703 | else | ||||||||
4704 | Inst->setHasNoUnsignedWrap(); | ||||||||
4705 | } | ||||||||
4706 | return true; | ||||||||
4707 | } | ||||||||
4708 | |||||||||
4709 | llvm_unreachable("Unexpected overflow result")::llvm::llvm_unreachable_internal("Unexpected overflow result" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4709); | ||||||||
4710 | } | ||||||||
4711 | |||||||||
4712 | /// Recognize and process idiom involving test for multiplication | ||||||||
4713 | /// overflow. | ||||||||
4714 | /// | ||||||||
4715 | /// The caller has matched a pattern of the form: | ||||||||
4716 | /// I = cmp u (mul(zext A, zext B), V | ||||||||
4717 | /// The function checks if this is a test for overflow and if so replaces | ||||||||
4718 | /// multiplication with call to 'mul.with.overflow' intrinsic. | ||||||||
4719 | /// | ||||||||
4720 | /// \param I Compare instruction. | ||||||||
4721 | /// \param MulVal Result of 'mult' instruction. It is one of the arguments of | ||||||||
4722 | /// the compare instruction. Must be of integer type. | ||||||||
4723 | /// \param OtherVal The other argument of compare instruction. | ||||||||
4724 | /// \returns Instruction which must replace the compare instruction, NULL if no | ||||||||
4725 | /// replacement required. | ||||||||
4726 | static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal, | ||||||||
4727 | Value *OtherVal, | ||||||||
4728 | InstCombinerImpl &IC) { | ||||||||
4729 | // Don't bother doing this transformation for pointers, don't do it for | ||||||||
4730 | // vectors. | ||||||||
4731 | if (!isa<IntegerType>(MulVal->getType())) | ||||||||
4732 | return nullptr; | ||||||||
4733 | |||||||||
4734 | assert(I.getOperand(0) == MulVal || I.getOperand(1) == MulVal)(static_cast <bool> (I.getOperand(0) == MulVal || I.getOperand (1) == MulVal) ? void (0) : __assert_fail ("I.getOperand(0) == MulVal || I.getOperand(1) == MulVal" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4734, __extension__ __PRETTY_FUNCTION__)); | ||||||||
4735 | assert(I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal)(static_cast <bool> (I.getOperand(0) == OtherVal || I.getOperand (1) == OtherVal) ? void (0) : __assert_fail ("I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4735, __extension__ __PRETTY_FUNCTION__)); | ||||||||
4736 | auto *MulInstr = dyn_cast<Instruction>(MulVal); | ||||||||
4737 | if (!MulInstr) | ||||||||
4738 | return nullptr; | ||||||||
4739 | assert(MulInstr->getOpcode() == Instruction::Mul)(static_cast <bool> (MulInstr->getOpcode() == Instruction ::Mul) ? void (0) : __assert_fail ("MulInstr->getOpcode() == Instruction::Mul" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4739, __extension__ __PRETTY_FUNCTION__)); | ||||||||
4740 | |||||||||
4741 | auto *LHS = cast<ZExtOperator>(MulInstr->getOperand(0)), | ||||||||
4742 | *RHS = cast<ZExtOperator>(MulInstr->getOperand(1)); | ||||||||
4743 | assert(LHS->getOpcode() == Instruction::ZExt)(static_cast <bool> (LHS->getOpcode() == Instruction ::ZExt) ? void (0) : __assert_fail ("LHS->getOpcode() == Instruction::ZExt" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4743, __extension__ __PRETTY_FUNCTION__)); | ||||||||
4744 | assert(RHS->getOpcode() == Instruction::ZExt)(static_cast <bool> (RHS->getOpcode() == Instruction ::ZExt) ? void (0) : __assert_fail ("RHS->getOpcode() == Instruction::ZExt" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4744, __extension__ __PRETTY_FUNCTION__)); | ||||||||
4745 | Value *A = LHS->getOperand(0), *B = RHS->getOperand(0); | ||||||||
4746 | |||||||||
4747 | // Calculate type and width of the result produced by mul.with.overflow. | ||||||||
4748 | Type *TyA = A->getType(), *TyB = B->getType(); | ||||||||
4749 | unsigned WidthA = TyA->getPrimitiveSizeInBits(), | ||||||||
4750 | WidthB = TyB->getPrimitiveSizeInBits(); | ||||||||
4751 | unsigned MulWidth; | ||||||||
4752 | Type *MulType; | ||||||||
4753 | if (WidthB > WidthA) { | ||||||||
4754 | MulWidth = WidthB; | ||||||||
4755 | MulType = TyB; | ||||||||
4756 | } else { | ||||||||
4757 | MulWidth = WidthA; | ||||||||
4758 | MulType = TyA; | ||||||||
4759 | } | ||||||||
4760 | |||||||||
4761 | // In order to replace the original mul with a narrower mul.with.overflow, | ||||||||
4762 | // all uses must ignore upper bits of the product. The number of used low | ||||||||
4763 | // bits must be not greater than the width of mul.with.overflow. | ||||||||
4764 | if (MulVal->hasNUsesOrMore(2)) | ||||||||
4765 | for (User *U : MulVal->users()) { | ||||||||
4766 | if (U == &I) | ||||||||
4767 | continue; | ||||||||
4768 | if (TruncInst *TI = dyn_cast<TruncInst>(U)) { | ||||||||
4769 | // Check if truncation ignores bits above MulWidth. | ||||||||
4770 | unsigned TruncWidth = TI->getType()->getPrimitiveSizeInBits(); | ||||||||
4771 | if (TruncWidth > MulWidth) | ||||||||
4772 | return nullptr; | ||||||||
4773 | } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) { | ||||||||
4774 | // Check if AND ignores bits above MulWidth. | ||||||||
4775 | if (BO->getOpcode() != Instruction::And) | ||||||||
4776 | return nullptr; | ||||||||
4777 | if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->getOperand(1))) { | ||||||||
4778 | const APInt &CVal = CI->getValue(); | ||||||||
4779 | if (CVal.getBitWidth() - CVal.countLeadingZeros() > MulWidth) | ||||||||
4780 | return nullptr; | ||||||||
4781 | } else { | ||||||||
4782 | // In this case we could have the operand of the binary operation | ||||||||
4783 | // being defined in another block, and performing the replacement | ||||||||
4784 | // could break the dominance relation. | ||||||||
4785 | return nullptr; | ||||||||
4786 | } | ||||||||
4787 | } else { | ||||||||
4788 | // Other uses prohibit this transformation. | ||||||||
4789 | return nullptr; | ||||||||
4790 | } | ||||||||
4791 | } | ||||||||
4792 | |||||||||
4793 | // Recognize patterns | ||||||||
4794 | switch (I.getPredicate()) { | ||||||||
4795 | case ICmpInst::ICMP_EQ: | ||||||||
4796 | case ICmpInst::ICMP_NE: | ||||||||
4797 | // Recognize pattern: | ||||||||
4798 | // mulval = mul(zext A, zext B) | ||||||||
4799 | // cmp eq/neq mulval, and(mulval, mask), mask selects low MulWidth bits. | ||||||||
4800 | ConstantInt *CI; | ||||||||
4801 | Value *ValToMask; | ||||||||
4802 | if (match(OtherVal, m_And(m_Value(ValToMask), m_ConstantInt(CI)))) { | ||||||||
4803 | if (ValToMask != MulVal) | ||||||||
4804 | return nullptr; | ||||||||
4805 | const APInt &CVal = CI->getValue() + 1; | ||||||||
4806 | if (CVal.isPowerOf2()) { | ||||||||
4807 | unsigned MaskWidth = CVal.logBase2(); | ||||||||
4808 | if (MaskWidth == MulWidth) | ||||||||
4809 | break; // Recognized | ||||||||
4810 | } | ||||||||
4811 | } | ||||||||
4812 | return nullptr; | ||||||||
4813 | |||||||||
4814 | case ICmpInst::ICMP_UGT: | ||||||||
4815 | // Recognize pattern: | ||||||||
4816 | // mulval = mul(zext A, zext B) | ||||||||
4817 | // cmp ugt mulval, max | ||||||||
4818 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | ||||||||
4819 | APInt MaxVal = APInt::getMaxValue(MulWidth); | ||||||||
4820 | MaxVal = MaxVal.zext(CI->getBitWidth()); | ||||||||
4821 | if (MaxVal.eq(CI->getValue())) | ||||||||
4822 | break; // Recognized | ||||||||
4823 | } | ||||||||
4824 | return nullptr; | ||||||||
4825 | |||||||||
4826 | case ICmpInst::ICMP_UGE: | ||||||||
4827 | // Recognize pattern: | ||||||||
4828 | // mulval = mul(zext A, zext B) | ||||||||
4829 | // cmp uge mulval, max+1 | ||||||||
4830 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | ||||||||
4831 | APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth); | ||||||||
4832 | if (MaxVal.eq(CI->getValue())) | ||||||||
4833 | break; // Recognized | ||||||||
4834 | } | ||||||||
4835 | return nullptr; | ||||||||
4836 | |||||||||
4837 | case ICmpInst::ICMP_ULE: | ||||||||
4838 | // Recognize pattern: | ||||||||
4839 | // mulval = mul(zext A, zext B) | ||||||||
4840 | // cmp ule mulval, max | ||||||||
4841 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | ||||||||
4842 | APInt MaxVal = APInt::getMaxValue(MulWidth); | ||||||||
4843 | MaxVal = MaxVal.zext(CI->getBitWidth()); | ||||||||
4844 | if (MaxVal.eq(CI->getValue())) | ||||||||
4845 | break; // Recognized | ||||||||
4846 | } | ||||||||
4847 | return nullptr; | ||||||||
4848 | |||||||||
4849 | case ICmpInst::ICMP_ULT: | ||||||||
4850 | // Recognize pattern: | ||||||||
4851 | // mulval = mul(zext A, zext B) | ||||||||
4852 | // cmp ule mulval, max + 1 | ||||||||
4853 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | ||||||||
4854 | APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth); | ||||||||
4855 | if (MaxVal.eq(CI->getValue())) | ||||||||
4856 | break; // Recognized | ||||||||
4857 | } | ||||||||
4858 | return nullptr; | ||||||||
4859 | |||||||||
4860 | default: | ||||||||
4861 | return nullptr; | ||||||||
4862 | } | ||||||||
4863 | |||||||||
4864 | InstCombiner::BuilderTy &Builder = IC.Builder; | ||||||||
4865 | Builder.SetInsertPoint(MulInstr); | ||||||||
4866 | |||||||||
4867 | // Replace: mul(zext A, zext B) --> mul.with.overflow(A, B) | ||||||||
4868 | Value *MulA = A, *MulB = B; | ||||||||
4869 | if (WidthA < MulWidth) | ||||||||
4870 | MulA = Builder.CreateZExt(A, MulType); | ||||||||
4871 | if (WidthB < MulWidth) | ||||||||
4872 | MulB = Builder.CreateZExt(B, MulType); | ||||||||
4873 | Function *F = Intrinsic::getDeclaration( | ||||||||
4874 | I.getModule(), Intrinsic::umul_with_overflow, MulType); | ||||||||
4875 | CallInst *Call = Builder.CreateCall(F, {MulA, MulB}, "umul"); | ||||||||
4876 | IC.addToWorklist(MulInstr); | ||||||||
4877 | |||||||||
4878 | // If there are uses of mul result other than the comparison, we know that | ||||||||
4879 | // they are truncation or binary AND. Change them to use result of | ||||||||
4880 | // mul.with.overflow and adjust properly mask/size. | ||||||||
4881 | if (MulVal->hasNUsesOrMore(2)) { | ||||||||
4882 | Value *Mul = Builder.CreateExtractValue(Call, 0, "umul.value"); | ||||||||
4883 | for (User *U : make_early_inc_range(MulVal->users())) { | ||||||||
4884 | if (U == &I || U == OtherVal) | ||||||||
4885 | continue; | ||||||||
4886 | if (TruncInst *TI = dyn_cast<TruncInst>(U)) { | ||||||||
4887 | if (TI->getType()->getPrimitiveSizeInBits() == MulWidth) | ||||||||
4888 | IC.replaceInstUsesWith(*TI, Mul); | ||||||||
4889 | else | ||||||||
4890 | TI->setOperand(0, Mul); | ||||||||
4891 | } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) { | ||||||||
4892 | assert(BO->getOpcode() == Instruction::And)(static_cast <bool> (BO->getOpcode() == Instruction:: And) ? void (0) : __assert_fail ("BO->getOpcode() == Instruction::And" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4892, __extension__ __PRETTY_FUNCTION__)); | ||||||||
4893 | // Replace (mul & mask) --> zext (mul.with.overflow & short_mask) | ||||||||
4894 | ConstantInt *CI = cast<ConstantInt>(BO->getOperand(1)); | ||||||||
4895 | APInt ShortMask = CI->getValue().trunc(MulWidth); | ||||||||
4896 | Value *ShortAnd = Builder.CreateAnd(Mul, ShortMask); | ||||||||
4897 | Value *Zext = Builder.CreateZExt(ShortAnd, BO->getType()); | ||||||||
4898 | IC.replaceInstUsesWith(*BO, Zext); | ||||||||
4899 | } else { | ||||||||
4900 | llvm_unreachable("Unexpected Binary operation")::llvm::llvm_unreachable_internal("Unexpected Binary operation" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4900); | ||||||||
4901 | } | ||||||||
4902 | IC.addToWorklist(cast<Instruction>(U)); | ||||||||
4903 | } | ||||||||
4904 | } | ||||||||
4905 | if (isa<Instruction>(OtherVal)) | ||||||||
4906 | IC.addToWorklist(cast<Instruction>(OtherVal)); | ||||||||
4907 | |||||||||
4908 | // The original icmp gets replaced with the overflow value, maybe inverted | ||||||||
4909 | // depending on predicate. | ||||||||
4910 | bool Inverse = false; | ||||||||
4911 | switch (I.getPredicate()) { | ||||||||
4912 | case ICmpInst::ICMP_NE: | ||||||||
4913 | break; | ||||||||
4914 | case ICmpInst::ICMP_EQ: | ||||||||
4915 | Inverse = true; | ||||||||
4916 | break; | ||||||||
4917 | case ICmpInst::ICMP_UGT: | ||||||||
4918 | case ICmpInst::ICMP_UGE: | ||||||||
4919 | if (I.getOperand(0) == MulVal) | ||||||||
4920 | break; | ||||||||
4921 | Inverse = true; | ||||||||
4922 | break; | ||||||||
4923 | case ICmpInst::ICMP_ULT: | ||||||||
4924 | case ICmpInst::ICMP_ULE: | ||||||||
4925 | if (I.getOperand(1) == MulVal) | ||||||||
4926 | break; | ||||||||
4927 | Inverse = true; | ||||||||
4928 | break; | ||||||||
4929 | default: | ||||||||
4930 | llvm_unreachable("Unexpected predicate")::llvm::llvm_unreachable_internal("Unexpected predicate", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4930); | ||||||||
4931 | } | ||||||||
4932 | if (Inverse) { | ||||||||
4933 | Value *Res = Builder.CreateExtractValue(Call, 1); | ||||||||
4934 | return BinaryOperator::CreateNot(Res); | ||||||||
4935 | } | ||||||||
4936 | |||||||||
4937 | return ExtractValueInst::Create(Call, 1); | ||||||||
4938 | } | ||||||||
4939 | |||||||||
4940 | /// When performing a comparison against a constant, it is possible that not all | ||||||||
4941 | /// the bits in the LHS are demanded. This helper method computes the mask that | ||||||||
4942 | /// IS demanded. | ||||||||
4943 | static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth) { | ||||||||
4944 | const APInt *RHS; | ||||||||
4945 | if (!match(I.getOperand(1), m_APInt(RHS))) | ||||||||
4946 | return APInt::getAllOnesValue(BitWidth); | ||||||||
4947 | |||||||||
4948 | // If this is a normal comparison, it demands all bits. If it is a sign bit | ||||||||
4949 | // comparison, it only demands the sign bit. | ||||||||
4950 | bool UnusedBit; | ||||||||
4951 | if (InstCombiner::isSignBitCheck(I.getPredicate(), *RHS, UnusedBit)) | ||||||||
4952 | return APInt::getSignMask(BitWidth); | ||||||||
4953 | |||||||||
4954 | switch (I.getPredicate()) { | ||||||||
4955 | // For a UGT comparison, we don't care about any bits that | ||||||||
4956 | // correspond to the trailing ones of the comparand. The value of these | ||||||||
4957 | // bits doesn't impact the outcome of the comparison, because any value | ||||||||
4958 | // greater than the RHS must differ in a bit higher than these due to carry. | ||||||||
4959 | case ICmpInst::ICMP_UGT: | ||||||||
4960 | return APInt::getBitsSetFrom(BitWidth, RHS->countTrailingOnes()); | ||||||||
4961 | |||||||||
4962 | // Similarly, for a ULT comparison, we don't care about the trailing zeros. | ||||||||
4963 | // Any value less than the RHS must differ in a higher bit because of carries. | ||||||||
4964 | case ICmpInst::ICMP_ULT: | ||||||||
4965 | return APInt::getBitsSetFrom(BitWidth, RHS->countTrailingZeros()); | ||||||||
4966 | |||||||||
4967 | default: | ||||||||
4968 | return APInt::getAllOnesValue(BitWidth); | ||||||||
4969 | } | ||||||||
4970 | } | ||||||||
4971 | |||||||||
4972 | /// Check if the order of \p Op0 and \p Op1 as operands in an ICmpInst | ||||||||
4973 | /// should be swapped. | ||||||||
4974 | /// The decision is based on how many times these two operands are reused | ||||||||
4975 | /// as subtract operands and their positions in those instructions. | ||||||||
4976 | /// The rationale is that several architectures use the same instruction for | ||||||||
4977 | /// both subtract and cmp. Thus, it is better if the order of those operands | ||||||||
4978 | /// match. | ||||||||
4979 | /// \return true if Op0 and Op1 should be swapped. | ||||||||
4980 | static bool swapMayExposeCSEOpportunities(const Value *Op0, const Value *Op1) { | ||||||||
4981 | // Filter out pointer values as those cannot appear directly in subtract. | ||||||||
4982 | // FIXME: we may want to go through inttoptrs or bitcasts. | ||||||||
4983 | if (Op0->getType()->isPointerTy()) | ||||||||
4984 | return false; | ||||||||
4985 | // If a subtract already has the same operands as a compare, swapping would be | ||||||||
4986 | // bad. If a subtract has the same operands as a compare but in reverse order, | ||||||||
4987 | // then swapping is good. | ||||||||
4988 | int GoodToSwap = 0; | ||||||||
4989 | for (const User *U : Op0->users()) { | ||||||||
4990 | if (match(U, m_Sub(m_Specific(Op1), m_Specific(Op0)))) | ||||||||
4991 | GoodToSwap++; | ||||||||
4992 | else if (match(U, m_Sub(m_Specific(Op0), m_Specific(Op1)))) | ||||||||
4993 | GoodToSwap--; | ||||||||
4994 | } | ||||||||
4995 | return GoodToSwap > 0; | ||||||||
4996 | } | ||||||||
4997 | |||||||||
4998 | /// Check that one use is in the same block as the definition and all | ||||||||
4999 | /// other uses are in blocks dominated by a given block. | ||||||||
5000 | /// | ||||||||
5001 | /// \param DI Definition | ||||||||
5002 | /// \param UI Use | ||||||||
5003 | /// \param DB Block that must dominate all uses of \p DI outside | ||||||||
5004 | /// the parent block | ||||||||
5005 | /// \return true when \p UI is the only use of \p DI in the parent block | ||||||||
5006 | /// and all other uses of \p DI are in blocks dominated by \p DB. | ||||||||
5007 | /// | ||||||||
5008 | bool InstCombinerImpl::dominatesAllUses(const Instruction *DI, | ||||||||
5009 | const Instruction *UI, | ||||||||
5010 | const BasicBlock *DB) const { | ||||||||
5011 | assert(DI && UI && "Instruction not defined\n")(static_cast <bool> (DI && UI && "Instruction not defined\n" ) ? void (0) : __assert_fail ("DI && UI && \"Instruction not defined\\n\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5011, __extension__ __PRETTY_FUNCTION__)); | ||||||||
5012 | // Ignore incomplete definitions. | ||||||||
5013 | if (!DI->getParent()) | ||||||||
5014 | return false; | ||||||||
5015 | // DI and UI must be in the same block. | ||||||||
5016 | if (DI->getParent() != UI->getParent()) | ||||||||
5017 | return false; | ||||||||
5018 | // Protect from self-referencing blocks. | ||||||||
5019 | if (DI->getParent() == DB) | ||||||||
5020 | return false; | ||||||||
5021 | for (const User *U : DI->users()) { | ||||||||
5022 | auto *Usr = cast<Instruction>(U); | ||||||||
5023 | if (Usr != UI && !DT.dominates(DB, Usr->getParent())) | ||||||||
5024 | return false; | ||||||||
5025 | } | ||||||||
5026 | return true; | ||||||||
5027 | } | ||||||||
5028 | |||||||||
5029 | /// Return true when the instruction sequence within a block is select-cmp-br. | ||||||||
5030 | static bool isChainSelectCmpBranch(const SelectInst *SI) { | ||||||||
5031 | const BasicBlock *BB = SI->getParent(); | ||||||||
5032 | if (!BB) | ||||||||
5033 | return false; | ||||||||
5034 | auto *BI = dyn_cast_or_null<BranchInst>(BB->getTerminator()); | ||||||||
5035 | if (!BI || BI->getNumSuccessors() != 2) | ||||||||
5036 | return false; | ||||||||
5037 | auto *IC = dyn_cast<ICmpInst>(BI->getCondition()); | ||||||||
5038 | if (!IC || (IC->getOperand(0) != SI && IC->getOperand(1) != SI)) | ||||||||
5039 | return false; | ||||||||
5040 | return true; | ||||||||
5041 | } | ||||||||
5042 | |||||||||
5043 | /// True when a select result is replaced by one of its operands | ||||||||
5044 | /// in select-icmp sequence. This will eventually result in the elimination | ||||||||
5045 | /// of the select. | ||||||||
5046 | /// | ||||||||
5047 | /// \param SI Select instruction | ||||||||
5048 | /// \param Icmp Compare instruction | ||||||||
5049 | /// \param SIOpd Operand that replaces the select | ||||||||
5050 | /// | ||||||||
5051 | /// Notes: | ||||||||
5052 | /// - The replacement is global and requires dominator information | ||||||||
5053 | /// - The caller is responsible for the actual replacement | ||||||||
5054 | /// | ||||||||
5055 | /// Example: | ||||||||
5056 | /// | ||||||||
5057 | /// entry: | ||||||||
5058 | /// %4 = select i1 %3, %C* %0, %C* null | ||||||||
5059 | /// %5 = icmp eq %C* %4, null | ||||||||
5060 | /// br i1 %5, label %9, label %7 | ||||||||
5061 | /// ... | ||||||||
5062 | /// ; <label>:7 ; preds = %entry | ||||||||
5063 | /// %8 = getelementptr inbounds %C* %4, i64 0, i32 0 | ||||||||
5064 | /// ... | ||||||||
5065 | /// | ||||||||
5066 | /// can be transformed to | ||||||||
5067 | /// | ||||||||
5068 | /// %5 = icmp eq %C* %0, null | ||||||||
5069 | /// %6 = select i1 %3, i1 %5, i1 true | ||||||||
5070 | /// br i1 %6, label %9, label %7 | ||||||||
5071 | /// ... | ||||||||
5072 | /// ; <label>:7 ; preds = %entry | ||||||||
5073 | /// %8 = getelementptr inbounds %C* %0, i64 0, i32 0 // replace by %0! | ||||||||
5074 | /// | ||||||||
5075 | /// Similar when the first operand of the select is a constant or/and | ||||||||
5076 | /// the compare is for not equal rather than equal. | ||||||||
5077 | /// | ||||||||
5078 | /// NOTE: The function is only called when the select and compare constants | ||||||||
5079 | /// are equal, the optimization can work only for EQ predicates. This is not a | ||||||||
5080 | /// major restriction since a NE compare should be 'normalized' to an equal | ||||||||
5081 | /// compare, which usually happens in the combiner and test case | ||||||||
5082 | /// select-cmp-br.ll checks for it. | ||||||||
5083 | bool InstCombinerImpl::replacedSelectWithOperand(SelectInst *SI, | ||||||||
5084 | const ICmpInst *Icmp, | ||||||||
5085 | const unsigned SIOpd) { | ||||||||
5086 | assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!")(static_cast <bool> ((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!") ? void (0) : __assert_fail ("(SIOpd == 1 || SIOpd == 2) && \"Invalid select operand!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5086, __extension__ __PRETTY_FUNCTION__)); | ||||||||
5087 | if (isChainSelectCmpBranch(SI) && Icmp->getPredicate() == ICmpInst::ICMP_EQ) { | ||||||||
5088 | BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(1); | ||||||||
5089 | // The check for the single predecessor is not the best that can be | ||||||||
5090 | // done. But it protects efficiently against cases like when SI's | ||||||||
5091 | // home block has two successors, Succ and Succ1, and Succ1 predecessor | ||||||||
5092 | // of Succ. Then SI can't be replaced by SIOpd because the use that gets | ||||||||
5093 | // replaced can be reached on either path. So the uniqueness check | ||||||||
5094 | // guarantees that the path all uses of SI (outside SI's parent) are on | ||||||||
5095 | // is disjoint from all other paths out of SI. But that information | ||||||||
5096 | // is more expensive to compute, and the trade-off here is in favor | ||||||||
5097 | // of compile-time. It should also be noticed that we check for a single | ||||||||
5098 | // predecessor and not only uniqueness. This to handle the situation when | ||||||||
5099 | // Succ and Succ1 points to the same basic block. | ||||||||
5100 | if (Succ->getSinglePredecessor() && dominatesAllUses(SI, Icmp, Succ)) { | ||||||||
5101 | NumSel++; | ||||||||
5102 | SI->replaceUsesOutsideBlock(SI->getOperand(SIOpd), SI->getParent()); | ||||||||
5103 | return true; | ||||||||
5104 | } | ||||||||
5105 | } | ||||||||
5106 | return false; | ||||||||
5107 | } | ||||||||
5108 | |||||||||
5109 | /// Try to fold the comparison based on range information we can get by checking | ||||||||
5110 | /// whether bits are known to be zero or one in the inputs. | ||||||||
5111 | Instruction *InstCombinerImpl::foldICmpUsingKnownBits(ICmpInst &I) { | ||||||||
5112 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
5113 | Type *Ty = Op0->getType(); | ||||||||
5114 | ICmpInst::Predicate Pred = I.getPredicate(); | ||||||||
5115 | |||||||||
5116 | // Get scalar or pointer size. | ||||||||
5117 | unsigned BitWidth = Ty->isIntOrIntVectorTy() | ||||||||
5118 | ? Ty->getScalarSizeInBits() | ||||||||
5119 | : DL.getPointerTypeSizeInBits(Ty->getScalarType()); | ||||||||
5120 | |||||||||
5121 | if (!BitWidth) | ||||||||
5122 | return nullptr; | ||||||||
5123 | |||||||||
5124 | KnownBits Op0Known(BitWidth); | ||||||||
5125 | KnownBits Op1Known(BitWidth); | ||||||||
5126 | |||||||||
5127 | if (SimplifyDemandedBits(&I, 0, | ||||||||
5128 | getDemandedBitsLHSMask(I, BitWidth), | ||||||||
5129 | Op0Known, 0)) | ||||||||
5130 | return &I; | ||||||||
5131 | |||||||||
5132 | if (SimplifyDemandedBits(&I, 1, APInt::getAllOnesValue(BitWidth), | ||||||||
5133 | Op1Known, 0)) | ||||||||
5134 | return &I; | ||||||||
5135 | |||||||||
5136 | // Given the known and unknown bits, compute a range that the LHS could be | ||||||||
5137 | // in. Compute the Min, Max and RHS values based on the known bits. For the | ||||||||
5138 | // EQ and NE we use unsigned values. | ||||||||
5139 | APInt Op0Min(BitWidth, 0), Op0Max(BitWidth, 0); | ||||||||
5140 | APInt Op1Min(BitWidth, 0), Op1Max(BitWidth, 0); | ||||||||
5141 | if (I.isSigned()) { | ||||||||
5142 | Op0Min = Op0Known.getSignedMinValue(); | ||||||||
5143 | Op0Max = Op0Known.getSignedMaxValue(); | ||||||||
5144 | Op1Min = Op1Known.getSignedMinValue(); | ||||||||
5145 | Op1Max = Op1Known.getSignedMaxValue(); | ||||||||
5146 | } else { | ||||||||
5147 | Op0Min = Op0Known.getMinValue(); | ||||||||
5148 | Op0Max = Op0Known.getMaxValue(); | ||||||||
5149 | Op1Min = Op1Known.getMinValue(); | ||||||||
5150 | Op1Max = Op1Known.getMaxValue(); | ||||||||
5151 | } | ||||||||
5152 | |||||||||
5153 | // If Min and Max are known to be the same, then SimplifyDemandedBits figured | ||||||||
5154 | // out that the LHS or RHS is a constant. Constant fold this now, so that | ||||||||
5155 | // code below can assume that Min != Max. | ||||||||
5156 | if (!isa<Constant>(Op0) && Op0Min == Op0Max) | ||||||||
5157 | return new ICmpInst(Pred, ConstantExpr::getIntegerValue(Ty, Op0Min), Op1); | ||||||||
5158 | if (!isa<Constant>(Op1) && Op1Min == Op1Max) | ||||||||
5159 | return new ICmpInst(Pred, Op0, ConstantExpr::getIntegerValue(Ty, Op1Min)); | ||||||||
5160 | |||||||||
5161 | // Based on the range information we know about the LHS, see if we can | ||||||||
5162 | // simplify this comparison. For example, (x&4) < 8 is always true. | ||||||||
5163 | switch (Pred) { | ||||||||
5164 | default: | ||||||||
5165 | llvm_unreachable("Unknown icmp opcode!")::llvm::llvm_unreachable_internal("Unknown icmp opcode!", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5165); | ||||||||
5166 | case ICmpInst::ICMP_EQ: | ||||||||
5167 | case ICmpInst::ICMP_NE: { | ||||||||
5168 | if (Op0Max.ult(Op1Min) || Op0Min.ugt(Op1Max)) | ||||||||
5169 | return replaceInstUsesWith( | ||||||||
5170 | I, ConstantInt::getBool(I.getType(), Pred == CmpInst::ICMP_NE)); | ||||||||
5171 | |||||||||
5172 | // If all bits are known zero except for one, then we know at most one bit | ||||||||
5173 | // is set. If the comparison is against zero, then this is a check to see if | ||||||||
5174 | // *that* bit is set. | ||||||||
5175 | APInt Op0KnownZeroInverted = ~Op0Known.Zero; | ||||||||
5176 | if (Op1Known.isZero()) { | ||||||||
5177 | // If the LHS is an AND with the same constant, look through it. | ||||||||
5178 | Value *LHS = nullptr; | ||||||||
5179 | const APInt *LHSC; | ||||||||
5180 | if (!match(Op0, m_And(m_Value(LHS), m_APInt(LHSC))) || | ||||||||
5181 | *LHSC != Op0KnownZeroInverted) | ||||||||
5182 | LHS = Op0; | ||||||||
5183 | |||||||||
5184 | Value *X; | ||||||||
5185 | if (match(LHS, m_Shl(m_One(), m_Value(X)))) { | ||||||||
5186 | APInt ValToCheck = Op0KnownZeroInverted; | ||||||||
5187 | Type *XTy = X->getType(); | ||||||||
5188 | if (ValToCheck.isPowerOf2()) { | ||||||||
5189 | // ((1 << X) & 8) == 0 -> X != 3 | ||||||||
5190 | // ((1 << X) & 8) != 0 -> X == 3 | ||||||||
5191 | auto *CmpC = ConstantInt::get(XTy, ValToCheck.countTrailingZeros()); | ||||||||
5192 | auto NewPred = ICmpInst::getInversePredicate(Pred); | ||||||||
5193 | return new ICmpInst(NewPred, X, CmpC); | ||||||||
5194 | } else if ((++ValToCheck).isPowerOf2()) { | ||||||||
5195 | // ((1 << X) & 7) == 0 -> X >= 3 | ||||||||
5196 | // ((1 << X) & 7) != 0 -> X < 3 | ||||||||
5197 | auto *CmpC = ConstantInt::get(XTy, ValToCheck.countTrailingZeros()); | ||||||||
5198 | auto NewPred = | ||||||||
5199 | Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGE : CmpInst::ICMP_ULT; | ||||||||
5200 | return new ICmpInst(NewPred, X, CmpC); | ||||||||
5201 | } | ||||||||
5202 | } | ||||||||
5203 | |||||||||
5204 | // Check if the LHS is 8 >>u x and the result is a power of 2 like 1. | ||||||||
5205 | const APInt *CI; | ||||||||
5206 | if (Op0KnownZeroInverted.isOneValue() && | ||||||||
5207 | match(LHS, m_LShr(m_Power2(CI), m_Value(X)))) { | ||||||||
5208 | // ((8 >>u X) & 1) == 0 -> X != 3 | ||||||||
5209 | // ((8 >>u X) & 1) != 0 -> X == 3 | ||||||||
5210 | unsigned CmpVal = CI->countTrailingZeros(); | ||||||||
5211 | auto NewPred = ICmpInst::getInversePredicate(Pred); | ||||||||
5212 | return new ICmpInst(NewPred, X, ConstantInt::get(X->getType(), CmpVal)); | ||||||||
5213 | } | ||||||||
5214 | } | ||||||||
5215 | break; | ||||||||
5216 | } | ||||||||
5217 | case ICmpInst::ICMP_ULT: { | ||||||||
5218 | if (Op0Max.ult(Op1Min)) // A <u B -> true if max(A) < min(B) | ||||||||
5219 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
5220 | if (Op0Min.uge(Op1Max)) // A <u B -> false if min(A) >= max(B) | ||||||||
5221 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
5222 | if (Op1Min == Op0Max) // A <u B -> A != B if max(A) == min(B) | ||||||||
5223 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | ||||||||
5224 | |||||||||
5225 | const APInt *CmpC; | ||||||||
5226 | if (match(Op1, m_APInt(CmpC))) { | ||||||||
5227 | // A <u C -> A == C-1 if min(A)+1 == C | ||||||||
5228 | if (*CmpC == Op0Min + 1) | ||||||||
5229 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||||||
5230 | ConstantInt::get(Op1->getType(), *CmpC - 1)); | ||||||||
5231 | // X <u C --> X == 0, if the number of zero bits in the bottom of X | ||||||||
5232 | // exceeds the log2 of C. | ||||||||
5233 | if (Op0Known.countMinTrailingZeros() >= CmpC->ceilLogBase2()) | ||||||||
5234 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||||||
5235 | Constant::getNullValue(Op1->getType())); | ||||||||
5236 | } | ||||||||
5237 | break; | ||||||||
5238 | } | ||||||||
5239 | case ICmpInst::ICMP_UGT: { | ||||||||
5240 | if (Op0Min.ugt(Op1Max)) // A >u B -> true if min(A) > max(B) | ||||||||
5241 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
5242 | if (Op0Max.ule(Op1Min)) // A >u B -> false if max(A) <= max(B) | ||||||||
5243 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
5244 | if (Op1Max == Op0Min) // A >u B -> A != B if min(A) == max(B) | ||||||||
5245 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | ||||||||
5246 | |||||||||
5247 | const APInt *CmpC; | ||||||||
5248 | if (match(Op1, m_APInt(CmpC))) { | ||||||||
5249 | // A >u C -> A == C+1 if max(a)-1 == C | ||||||||
5250 | if (*CmpC == Op0Max - 1) | ||||||||
5251 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||||||
5252 | ConstantInt::get(Op1->getType(), *CmpC + 1)); | ||||||||
5253 | // X >u C --> X != 0, if the number of zero bits in the bottom of X | ||||||||
5254 | // exceeds the log2 of C. | ||||||||
5255 | if (Op0Known.countMinTrailingZeros() >= CmpC->getActiveBits()) | ||||||||
5256 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, | ||||||||
5257 | Constant::getNullValue(Op1->getType())); | ||||||||
5258 | } | ||||||||
5259 | break; | ||||||||
5260 | } | ||||||||
5261 | case ICmpInst::ICMP_SLT: { | ||||||||
5262 | if (Op0Max.slt(Op1Min)) // A <s B -> true if max(A) < min(C) | ||||||||
5263 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
5264 | if (Op0Min.sge(Op1Max)) // A <s B -> false if min(A) >= max(C) | ||||||||
5265 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
5266 | if (Op1Min == Op0Max) // A <s B -> A != B if max(A) == min(B) | ||||||||
5267 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | ||||||||
5268 | const APInt *CmpC; | ||||||||
5269 | if (match(Op1, m_APInt(CmpC))) { | ||||||||
5270 | if (*CmpC == Op0Min + 1) // A <s C -> A == C-1 if min(A)+1 == C | ||||||||
5271 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||||||
5272 | ConstantInt::get(Op1->getType(), *CmpC - 1)); | ||||||||
5273 | } | ||||||||
5274 | break; | ||||||||
5275 | } | ||||||||
5276 | case ICmpInst::ICMP_SGT: { | ||||||||
5277 | if (Op0Min.sgt(Op1Max)) // A >s B -> true if min(A) > max(B) | ||||||||
5278 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
5279 | if (Op0Max.sle(Op1Min)) // A >s B -> false if max(A) <= min(B) | ||||||||
5280 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
5281 | if (Op1Max == Op0Min) // A >s B -> A != B if min(A) == max(B) | ||||||||
5282 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | ||||||||
5283 | const APInt *CmpC; | ||||||||
5284 | if (match(Op1, m_APInt(CmpC))) { | ||||||||
5285 | if (*CmpC == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C | ||||||||
5286 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||||||
5287 | ConstantInt::get(Op1->getType(), *CmpC + 1)); | ||||||||
5288 | } | ||||||||
5289 | break; | ||||||||
5290 | } | ||||||||
5291 | case ICmpInst::ICMP_SGE: | ||||||||
5292 | assert(!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!")(static_cast <bool> (!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!") ? void (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_SGE with ConstantInt not folded!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5292, __extension__ __PRETTY_FUNCTION__)); | ||||||||
5293 | if (Op0Min.sge(Op1Max)) // A >=s B -> true if min(A) >= max(B) | ||||||||
5294 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
5295 | if (Op0Max.slt(Op1Min)) // A >=s B -> false if max(A) < min(B) | ||||||||
5296 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
5297 | if (Op1Min == Op0Max) // A >=s B -> A == B if max(A) == min(B) | ||||||||
5298 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); | ||||||||
5299 | break; | ||||||||
5300 | case ICmpInst::ICMP_SLE: | ||||||||
5301 | assert(!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!")(static_cast <bool> (!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!") ? void (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_SLE with ConstantInt not folded!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5301, __extension__ __PRETTY_FUNCTION__)); | ||||||||
5302 | if (Op0Max.sle(Op1Min)) // A <=s B -> true if max(A) <= min(B) | ||||||||
5303 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
5304 | if (Op0Min.sgt(Op1Max)) // A <=s B -> false if min(A) > max(B) | ||||||||
5305 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
5306 | if (Op1Max == Op0Min) // A <=s B -> A == B if min(A) == max(B) | ||||||||
5307 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); | ||||||||
5308 | break; | ||||||||
5309 | case ICmpInst::ICMP_UGE: | ||||||||
5310 | assert(!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!")(static_cast <bool> (!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!") ? void (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_UGE with ConstantInt not folded!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5310, __extension__ __PRETTY_FUNCTION__)); | ||||||||
5311 | if (Op0Min.uge(Op1Max)) // A >=u B -> true if min(A) >= max(B) | ||||||||
5312 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
5313 | if (Op0Max.ult(Op1Min)) // A >=u B -> false if max(A) < min(B) | ||||||||
5314 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
5315 | if (Op1Min == Op0Max) // A >=u B -> A == B if max(A) == min(B) | ||||||||
5316 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); | ||||||||
5317 | break; | ||||||||
5318 | case ICmpInst::ICMP_ULE: | ||||||||
5319 | assert(!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!")(static_cast <bool> (!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!") ? void (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_ULE with ConstantInt not folded!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5319, __extension__ __PRETTY_FUNCTION__)); | ||||||||
5320 | if (Op0Max.ule(Op1Min)) // A <=u B -> true if max(A) <= min(B) | ||||||||
5321 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
5322 | if (Op0Min.ugt(Op1Max)) // A <=u B -> false if min(A) > max(B) | ||||||||
5323 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
5324 | if (Op1Max == Op0Min) // A <=u B -> A == B if min(A) == max(B) | ||||||||
5325 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); | ||||||||
5326 | break; | ||||||||
5327 | } | ||||||||
5328 | |||||||||
5329 | // Turn a signed comparison into an unsigned one if both operands are known to | ||||||||
5330 | // have the same sign. | ||||||||
5331 | if (I.isSigned() && | ||||||||
5332 | ((Op0Known.Zero.isNegative() && Op1Known.Zero.isNegative()) || | ||||||||
5333 | (Op0Known.One.isNegative() && Op1Known.One.isNegative()))) | ||||||||
5334 | return new ICmpInst(I.getUnsignedPredicate(), Op0, Op1); | ||||||||
5335 | |||||||||
5336 | return nullptr; | ||||||||
5337 | } | ||||||||
5338 | |||||||||
5339 | llvm::Optional<std::pair<CmpInst::Predicate, Constant *>> | ||||||||
5340 | InstCombiner::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred, | ||||||||
5341 | Constant *C) { | ||||||||
5342 | assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&(static_cast <bool> (ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) && "Only for relational integer predicates." ) ? void (0) : __assert_fail ("ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) && \"Only for relational integer predicates.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5343, __extension__ __PRETTY_FUNCTION__)) | ||||||||
5343 | "Only for relational integer predicates.")(static_cast <bool> (ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) && "Only for relational integer predicates." ) ? void (0) : __assert_fail ("ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) && \"Only for relational integer predicates.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5343, __extension__ __PRETTY_FUNCTION__)); | ||||||||
5344 | |||||||||
5345 | Type *Type = C->getType(); | ||||||||
5346 | bool IsSigned = ICmpInst::isSigned(Pred); | ||||||||
5347 | |||||||||
5348 | CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(Pred); | ||||||||
5349 | bool WillIncrement = | ||||||||
5350 | UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT; | ||||||||
5351 | |||||||||
5352 | // Check if the constant operand can be safely incremented/decremented | ||||||||
5353 | // without overflowing/underflowing. | ||||||||
5354 | auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) { | ||||||||
5355 | return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned); | ||||||||
5356 | }; | ||||||||
5357 | |||||||||
5358 | Constant *SafeReplacementConstant = nullptr; | ||||||||
5359 | if (auto *CI = dyn_cast<ConstantInt>(C)) { | ||||||||
5360 | // Bail out if the constant can't be safely incremented/decremented. | ||||||||
5361 | if (!ConstantIsOk(CI)) | ||||||||
5362 | return llvm::None; | ||||||||
5363 | } else if (auto *FVTy = dyn_cast<FixedVectorType>(Type)) { | ||||||||
5364 | unsigned NumElts = FVTy->getNumElements(); | ||||||||
5365 | for (unsigned i = 0; i != NumElts; ++i) { | ||||||||
5366 | Constant *Elt = C->getAggregateElement(i); | ||||||||
5367 | if (!Elt) | ||||||||
5368 | return llvm::None; | ||||||||
5369 | |||||||||
5370 | if (isa<UndefValue>(Elt)) | ||||||||
5371 | continue; | ||||||||
5372 | |||||||||
5373 | // Bail out if we can't determine if this constant is min/max or if we | ||||||||
5374 | // know that this constant is min/max. | ||||||||
5375 | auto *CI = dyn_cast<ConstantInt>(Elt); | ||||||||
5376 | if (!CI || !ConstantIsOk(CI)) | ||||||||
5377 | return llvm::None; | ||||||||
5378 | |||||||||
5379 | if (!SafeReplacementConstant) | ||||||||
5380 | SafeReplacementConstant = CI; | ||||||||
5381 | } | ||||||||
5382 | } else { | ||||||||
5383 | // ConstantExpr? | ||||||||
5384 | return llvm::None; | ||||||||
5385 | } | ||||||||
5386 | |||||||||
5387 | // It may not be safe to change a compare predicate in the presence of | ||||||||
5388 | // undefined elements, so replace those elements with the first safe constant | ||||||||
5389 | // that we found. | ||||||||
5390 | // TODO: in case of poison, it is safe; let's replace undefs only. | ||||||||
5391 | if (C->containsUndefOrPoisonElement()) { | ||||||||
5392 | assert(SafeReplacementConstant && "Replacement constant not set")(static_cast <bool> (SafeReplacementConstant && "Replacement constant not set") ? void (0) : __assert_fail ( "SafeReplacementConstant && \"Replacement constant not set\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5392, __extension__ __PRETTY_FUNCTION__)); | ||||||||
5393 | C = Constant::replaceUndefsWith(C, SafeReplacementConstant); | ||||||||
5394 | } | ||||||||
5395 | |||||||||
5396 | CmpInst::Predicate NewPred = CmpInst::getFlippedStrictnessPredicate(Pred); | ||||||||
5397 | |||||||||
5398 | // Increment or decrement the constant. | ||||||||
5399 | Constant *OneOrNegOne = ConstantInt::get(Type, WillIncrement ? 1 : -1, true); | ||||||||
5400 | Constant *NewC = ConstantExpr::getAdd(C, OneOrNegOne); | ||||||||
5401 | |||||||||
5402 | return std::make_pair(NewPred, NewC); | ||||||||
5403 | } | ||||||||
5404 | |||||||||
5405 | /// If we have an icmp le or icmp ge instruction with a constant operand, turn | ||||||||
5406 | /// it into the appropriate icmp lt or icmp gt instruction. This transform | ||||||||
5407 | /// allows them to be folded in visitICmpInst. | ||||||||
5408 | static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) { | ||||||||
5409 | ICmpInst::Predicate Pred = I.getPredicate(); | ||||||||
5410 | if (ICmpInst::isEquality(Pred) || !ICmpInst::isIntPredicate(Pred) || | ||||||||
5411 | InstCombiner::isCanonicalPredicate(Pred)) | ||||||||
5412 | return nullptr; | ||||||||
5413 | |||||||||
5414 | Value *Op0 = I.getOperand(0); | ||||||||
5415 | Value *Op1 = I.getOperand(1); | ||||||||
5416 | auto *Op1C = dyn_cast<Constant>(Op1); | ||||||||
5417 | if (!Op1C) | ||||||||
5418 | return nullptr; | ||||||||
5419 | |||||||||
5420 | auto FlippedStrictness = | ||||||||
5421 | InstCombiner::getFlippedStrictnessPredicateAndConstant(Pred, Op1C); | ||||||||
5422 | if (!FlippedStrictness) | ||||||||
5423 | return nullptr; | ||||||||
5424 | |||||||||
5425 | return new ICmpInst(FlippedStrictness->first, Op0, FlippedStrictness->second); | ||||||||
5426 | } | ||||||||
5427 | |||||||||
5428 | /// If we have a comparison with a non-canonical predicate, if we can update | ||||||||
5429 | /// all the users, invert the predicate and adjust all the users. | ||||||||
5430 | CmpInst *InstCombinerImpl::canonicalizeICmpPredicate(CmpInst &I) { | ||||||||
5431 | // Is the predicate already canonical? | ||||||||
5432 | CmpInst::Predicate Pred = I.getPredicate(); | ||||||||
5433 | if (InstCombiner::isCanonicalPredicate(Pred)) | ||||||||
5434 | return nullptr; | ||||||||
5435 | |||||||||
5436 | // Can all users be adjusted to predicate inversion? | ||||||||
5437 | if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr)) | ||||||||
5438 | return nullptr; | ||||||||
5439 | |||||||||
5440 | // Ok, we can canonicalize comparison! | ||||||||
5441 | // Let's first invert the comparison's predicate. | ||||||||
5442 | I.setPredicate(CmpInst::getInversePredicate(Pred)); | ||||||||
5443 | I.setName(I.getName() + ".not"); | ||||||||
5444 | |||||||||
5445 | // And, adapt users. | ||||||||
5446 | freelyInvertAllUsersOf(&I); | ||||||||
5447 | |||||||||
5448 | return &I; | ||||||||
5449 | } | ||||||||
5450 | |||||||||
5451 | /// Integer compare with boolean values can always be turned into bitwise ops. | ||||||||
5452 | static Instruction *canonicalizeICmpBool(ICmpInst &I, | ||||||||
5453 | InstCombiner::BuilderTy &Builder) { | ||||||||
5454 | Value *A = I.getOperand(0), *B = I.getOperand(1); | ||||||||
5455 | assert(A->getType()->isIntOrIntVectorTy(1) && "Bools only")(static_cast <bool> (A->getType()->isIntOrIntVectorTy (1) && "Bools only") ? void (0) : __assert_fail ("A->getType()->isIntOrIntVectorTy(1) && \"Bools only\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5455, __extension__ __PRETTY_FUNCTION__)); | ||||||||
5456 | |||||||||
5457 | // A boolean compared to true/false can be simplified to Op0/true/false in | ||||||||
5458 | // 14 out of the 20 (10 predicates * 2 constants) possible combinations. | ||||||||
5459 | // Cases not handled by InstSimplify are always 'not' of Op0. | ||||||||
5460 | if (match(B, m_Zero())) { | ||||||||
5461 | switch (I.getPredicate()) { | ||||||||
5462 | case CmpInst::ICMP_EQ: // A == 0 -> !A | ||||||||
5463 | case CmpInst::ICMP_ULE: // A <=u 0 -> !A | ||||||||
5464 | case CmpInst::ICMP_SGE: // A >=s 0 -> !A | ||||||||
5465 | return BinaryOperator::CreateNot(A); | ||||||||
5466 | default: | ||||||||
5467 | llvm_unreachable("ICmp i1 X, C not simplified as expected.")::llvm::llvm_unreachable_internal("ICmp i1 X, C not simplified as expected." , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5467); | ||||||||
5468 | } | ||||||||
5469 | } else if (match(B, m_One())) { | ||||||||
5470 | switch (I.getPredicate()) { | ||||||||
5471 | case CmpInst::ICMP_NE: // A != 1 -> !A | ||||||||
5472 | case CmpInst::ICMP_ULT: // A <u 1 -> !A | ||||||||
5473 | case CmpInst::ICMP_SGT: // A >s -1 -> !A | ||||||||
5474 | return BinaryOperator::CreateNot(A); | ||||||||
5475 | default: | ||||||||
5476 | llvm_unreachable("ICmp i1 X, C not simplified as expected.")::llvm::llvm_unreachable_internal("ICmp i1 X, C not simplified as expected." , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5476); | ||||||||
5477 | } | ||||||||
5478 | } | ||||||||
5479 | |||||||||
5480 | switch (I.getPredicate()) { | ||||||||
5481 | default: | ||||||||
5482 | llvm_unreachable("Invalid icmp instruction!")::llvm::llvm_unreachable_internal("Invalid icmp instruction!" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5482); | ||||||||
5483 | case ICmpInst::ICMP_EQ: | ||||||||
5484 | // icmp eq i1 A, B -> ~(A ^ B) | ||||||||
5485 | return BinaryOperator::CreateNot(Builder.CreateXor(A, B)); | ||||||||
5486 | |||||||||
5487 | case ICmpInst::ICMP_NE: | ||||||||
5488 | // icmp ne i1 A, B -> A ^ B | ||||||||
5489 | return BinaryOperator::CreateXor(A, B); | ||||||||
5490 | |||||||||
5491 | case ICmpInst::ICMP_UGT: | ||||||||
5492 | // icmp ugt -> icmp ult | ||||||||
5493 | std::swap(A, B); | ||||||||
5494 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||||
5495 | case ICmpInst::ICMP_ULT: | ||||||||
5496 | // icmp ult i1 A, B -> ~A & B | ||||||||
5497 | return BinaryOperator::CreateAnd(Builder.CreateNot(A), B); | ||||||||
5498 | |||||||||
5499 | case ICmpInst::ICMP_SGT: | ||||||||
5500 | // icmp sgt -> icmp slt | ||||||||
5501 | std::swap(A, B); | ||||||||
5502 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||||
5503 | case ICmpInst::ICMP_SLT: | ||||||||
5504 | // icmp slt i1 A, B -> A & ~B | ||||||||
5505 | return BinaryOperator::CreateAnd(Builder.CreateNot(B), A); | ||||||||
5506 | |||||||||
5507 | case ICmpInst::ICMP_UGE: | ||||||||
5508 | // icmp uge -> icmp ule | ||||||||
5509 | std::swap(A, B); | ||||||||
5510 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||||
5511 | case ICmpInst::ICMP_ULE: | ||||||||
5512 | // icmp ule i1 A, B -> ~A | B | ||||||||
5513 | return BinaryOperator::CreateOr(Builder.CreateNot(A), B); | ||||||||
5514 | |||||||||
5515 | case ICmpInst::ICMP_SGE: | ||||||||
5516 | // icmp sge -> icmp sle | ||||||||
5517 | std::swap(A, B); | ||||||||
5518 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||||
5519 | case ICmpInst::ICMP_SLE: | ||||||||
5520 | // icmp sle i1 A, B -> A | ~B | ||||||||
5521 | return BinaryOperator::CreateOr(Builder.CreateNot(B), A); | ||||||||
5522 | } | ||||||||
5523 | } | ||||||||
5524 | |||||||||
5525 | // Transform pattern like: | ||||||||
5526 | // (1 << Y) u<= X or ~(-1 << Y) u< X or ((1 << Y)+(-1)) u< X | ||||||||
5527 | // (1 << Y) u> X or ~(-1 << Y) u>= X or ((1 << Y)+(-1)) u>= X | ||||||||
5528 | // Into: | ||||||||
5529 | // (X l>> Y) != 0 | ||||||||
5530 | // (X l>> Y) == 0 | ||||||||
5531 | static Instruction *foldICmpWithHighBitMask(ICmpInst &Cmp, | ||||||||
5532 | InstCombiner::BuilderTy &Builder) { | ||||||||
5533 | ICmpInst::Predicate Pred, NewPred; | ||||||||
5534 | Value *X, *Y; | ||||||||
5535 | if (match(&Cmp, | ||||||||
5536 | m_c_ICmp(Pred, m_OneUse(m_Shl(m_One(), m_Value(Y))), m_Value(X)))) { | ||||||||
5537 | switch (Pred) { | ||||||||
5538 | case ICmpInst::ICMP_ULE: | ||||||||
5539 | NewPred = ICmpInst::ICMP_NE; | ||||||||
5540 | break; | ||||||||
5541 | case ICmpInst::ICMP_UGT: | ||||||||
5542 | NewPred = ICmpInst::ICMP_EQ; | ||||||||
5543 | break; | ||||||||
5544 | default: | ||||||||
5545 | return nullptr; | ||||||||
5546 | } | ||||||||
5547 | } else if (match(&Cmp, m_c_ICmp(Pred, | ||||||||
5548 | m_OneUse(m_CombineOr( | ||||||||
5549 | m_Not(m_Shl(m_AllOnes(), m_Value(Y))), | ||||||||
5550 | m_Add(m_Shl(m_One(), m_Value(Y)), | ||||||||
5551 | m_AllOnes()))), | ||||||||
5552 | m_Value(X)))) { | ||||||||
5553 | // The variant with 'add' is not canonical, (the variant with 'not' is) | ||||||||
5554 | // we only get it because it has extra uses, and can't be canonicalized, | ||||||||
5555 | |||||||||
5556 | switch (Pred) { | ||||||||
5557 | case ICmpInst::ICMP_ULT: | ||||||||
5558 | NewPred = ICmpInst::ICMP_NE; | ||||||||
5559 | break; | ||||||||
5560 | case ICmpInst::ICMP_UGE: | ||||||||
5561 | NewPred = ICmpInst::ICMP_EQ; | ||||||||
5562 | break; | ||||||||
5563 | default: | ||||||||
5564 | return nullptr; | ||||||||
5565 | } | ||||||||
5566 | } else | ||||||||
5567 | return nullptr; | ||||||||
5568 | |||||||||
5569 | Value *NewX = Builder.CreateLShr(X, Y, X->getName() + ".highbits"); | ||||||||
5570 | Constant *Zero = Constant::getNullValue(NewX->getType()); | ||||||||
5571 | return CmpInst::Create(Instruction::ICmp, NewPred, NewX, Zero); | ||||||||
5572 | } | ||||||||
5573 | |||||||||
5574 | static Instruction *foldVectorCmp(CmpInst &Cmp, | ||||||||
5575 | InstCombiner::BuilderTy &Builder) { | ||||||||
5576 | const CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
5577 | Value *LHS = Cmp.getOperand(0), *RHS = Cmp.getOperand(1); | ||||||||
5578 | Value *V1, *V2; | ||||||||
5579 | ArrayRef<int> M; | ||||||||
5580 | if (!match(LHS, m_Shuffle(m_Value(V1), m_Undef(), m_Mask(M)))) | ||||||||
5581 | return nullptr; | ||||||||
5582 | |||||||||
5583 | // If both arguments of the cmp are shuffles that use the same mask and | ||||||||
5584 | // shuffle within a single vector, move the shuffle after the cmp: | ||||||||
5585 | // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M | ||||||||
5586 | Type *V1Ty = V1->getType(); | ||||||||
5587 | if (match(RHS, m_Shuffle(m_Value(V2), m_Undef(), m_SpecificMask(M))) && | ||||||||
5588 | V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) { | ||||||||
5589 | Value *NewCmp = Builder.CreateCmp(Pred, V1, V2); | ||||||||
5590 | return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), M); | ||||||||
5591 | } | ||||||||
5592 | |||||||||
5593 | // Try to canonicalize compare with splatted operand and splat constant. | ||||||||
5594 | // TODO: We could generalize this for more than splats. See/use the code in | ||||||||
5595 | // InstCombiner::foldVectorBinop(). | ||||||||
5596 | Constant *C; | ||||||||
5597 | if (!LHS->hasOneUse() || !match(RHS, m_Constant(C))) | ||||||||
5598 | return nullptr; | ||||||||
5599 | |||||||||
5600 | // Length-changing splats are ok, so adjust the constants as needed: | ||||||||
5601 | // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M | ||||||||
5602 | Constant *ScalarC = C->getSplatValue(/* AllowUndefs */ true); | ||||||||
5603 | int MaskSplatIndex; | ||||||||
5604 | if (ScalarC && match(M, m_SplatOrUndefMask(MaskSplatIndex))) { | ||||||||
5605 | // We allow undefs in matching, but this transform removes those for safety. | ||||||||
5606 | // Demanded elements analysis should be able to recover some/all of that. | ||||||||
5607 | C = ConstantVector::getSplat(cast<VectorType>(V1Ty)->getElementCount(), | ||||||||
5608 | ScalarC); | ||||||||
5609 | SmallVector<int, 8> NewM(M.size(), MaskSplatIndex); | ||||||||
5610 | Value *NewCmp = Builder.CreateCmp(Pred, V1, C); | ||||||||
5611 | return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), | ||||||||
5612 | NewM); | ||||||||
5613 | } | ||||||||
5614 | |||||||||
5615 | return nullptr; | ||||||||
5616 | } | ||||||||
5617 | |||||||||
5618 | // extract(uadd.with.overflow(A, B), 0) ult A | ||||||||
5619 | // -> extract(uadd.with.overflow(A, B), 1) | ||||||||
5620 | static Instruction *foldICmpOfUAddOv(ICmpInst &I) { | ||||||||
5621 | CmpInst::Predicate Pred = I.getPredicate(); | ||||||||
5622 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
5623 | |||||||||
5624 | Value *UAddOv; | ||||||||
5625 | Value *A, *B; | ||||||||
5626 | auto UAddOvResultPat = m_ExtractValue<0>( | ||||||||
5627 | m_Intrinsic<Intrinsic::uadd_with_overflow>(m_Value(A), m_Value(B))); | ||||||||
5628 | if (match(Op0, UAddOvResultPat) && | ||||||||
5629 | ((Pred == ICmpInst::ICMP_ULT && (Op1 == A || Op1 == B)) || | ||||||||
5630 | (Pred == ICmpInst::ICMP_EQ && match(Op1, m_ZeroInt()) && | ||||||||
5631 | (match(A, m_One()) || match(B, m_One()))) || | ||||||||
5632 | (Pred == ICmpInst::ICMP_NE && match(Op1, m_AllOnes()) && | ||||||||
5633 | (match(A, m_AllOnes()) || match(B, m_AllOnes()))))) | ||||||||
5634 | // extract(uadd.with.overflow(A, B), 0) < A | ||||||||
5635 | // extract(uadd.with.overflow(A, 1), 0) == 0 | ||||||||
5636 | // extract(uadd.with.overflow(A, -1), 0) != -1 | ||||||||
5637 | UAddOv = cast<ExtractValueInst>(Op0)->getAggregateOperand(); | ||||||||
5638 | else if (match(Op1, UAddOvResultPat) && | ||||||||
5639 | Pred == ICmpInst::ICMP_UGT && (Op0 == A || Op0 == B)) | ||||||||
5640 | // A > extract(uadd.with.overflow(A, B), 0) | ||||||||
5641 | UAddOv = cast<ExtractValueInst>(Op1)->getAggregateOperand(); | ||||||||
5642 | else | ||||||||
5643 | return nullptr; | ||||||||
5644 | |||||||||
5645 | return ExtractValueInst::Create(UAddOv, 1); | ||||||||
5646 | } | ||||||||
5647 | |||||||||
5648 | Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) { | ||||||||
5649 | bool Changed = false; | ||||||||
5650 | const SimplifyQuery Q = SQ.getWithInstruction(&I); | ||||||||
5651 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
5652 | unsigned Op0Cplxity = getComplexity(Op0); | ||||||||
5653 | unsigned Op1Cplxity = getComplexity(Op1); | ||||||||
5654 | |||||||||
5655 | /// Orders the operands of the compare so that they are listed from most | ||||||||
5656 | /// complex to least complex. This puts constants before unary operators, | ||||||||
5657 | /// before binary operators. | ||||||||
5658 | if (Op0Cplxity < Op1Cplxity || | ||||||||
5659 | (Op0Cplxity == Op1Cplxity && swapMayExposeCSEOpportunities(Op0, Op1))) { | ||||||||
5660 | I.swapOperands(); | ||||||||
5661 | std::swap(Op0, Op1); | ||||||||
5662 | Changed = true; | ||||||||
5663 | } | ||||||||
5664 | |||||||||
5665 | if (Value *V = SimplifyICmpInst(I.getPredicate(), Op0, Op1, Q)) | ||||||||
5666 | return replaceInstUsesWith(I, V); | ||||||||
5667 | |||||||||
5668 | // Comparing -val or val with non-zero is the same as just comparing val | ||||||||
5669 | // ie, abs(val) != 0 -> val != 0 | ||||||||
5670 | if (I.getPredicate() == ICmpInst::ICMP_NE && match(Op1, m_Zero())) { | ||||||||
5671 | Value *Cond, *SelectTrue, *SelectFalse; | ||||||||
5672 | if (match(Op0, m_Select(m_Value(Cond), m_Value(SelectTrue), | ||||||||
5673 | m_Value(SelectFalse)))) { | ||||||||
5674 | if (Value *V = dyn_castNegVal(SelectTrue)) { | ||||||||
5675 | if (V == SelectFalse) | ||||||||
5676 | return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1); | ||||||||
5677 | } | ||||||||
5678 | else if (Value *V = dyn_castNegVal(SelectFalse)) { | ||||||||
5679 | if (V == SelectTrue) | ||||||||
5680 | return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1); | ||||||||
5681 | } | ||||||||
5682 | } | ||||||||
5683 | } | ||||||||
5684 | |||||||||
5685 | if (Op0->getType()->isIntOrIntVectorTy(1)) | ||||||||
5686 | if (Instruction *Res = canonicalizeICmpBool(I, Builder)) | ||||||||
5687 | return Res; | ||||||||
5688 | |||||||||
5689 | if (Instruction *Res = canonicalizeCmpWithConstant(I)) | ||||||||
5690 | return Res; | ||||||||
5691 | |||||||||
5692 | if (Instruction *Res = canonicalizeICmpPredicate(I)) | ||||||||
5693 | return Res; | ||||||||
5694 | |||||||||
5695 | if (Instruction *Res = foldICmpWithConstant(I)) | ||||||||
5696 | return Res; | ||||||||
5697 | |||||||||
5698 | if (Instruction *Res = foldICmpWithDominatingICmp(I)) | ||||||||
5699 | return Res; | ||||||||
5700 | |||||||||
5701 | if (Instruction *Res = foldICmpBinOp(I, Q)) | ||||||||
5702 | return Res; | ||||||||
5703 | |||||||||
5704 | if (Instruction *Res = foldICmpUsingKnownBits(I)) | ||||||||
5705 | return Res; | ||||||||
5706 | |||||||||
5707 | // Test if the ICmpInst instruction is used exclusively by a select as | ||||||||
5708 | // part of a minimum or maximum operation. If so, refrain from doing | ||||||||
5709 | // any other folding. This helps out other analyses which understand | ||||||||
5710 | // non-obfuscated minimum and maximum idioms, such as ScalarEvolution | ||||||||
5711 | // and CodeGen. And in this case, at least one of the comparison | ||||||||
5712 | // operands has at least one user besides the compare (the select), | ||||||||
5713 | // which would often largely negate the benefit of folding anyway. | ||||||||
5714 | // | ||||||||
5715 | // Do the same for the other patterns recognized by matchSelectPattern. | ||||||||
5716 | if (I.hasOneUse()) | ||||||||
5717 | if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) { | ||||||||
5718 | Value *A, *B; | ||||||||
5719 | SelectPatternResult SPR = matchSelectPattern(SI, A, B); | ||||||||
5720 | if (SPR.Flavor != SPF_UNKNOWN) | ||||||||
5721 | return nullptr; | ||||||||
5722 | } | ||||||||
5723 | |||||||||
5724 | // Do this after checking for min/max to prevent infinite looping. | ||||||||
5725 | if (Instruction *Res = foldICmpWithZero(I)) | ||||||||
5726 | return Res; | ||||||||
5727 | |||||||||
5728 | // FIXME: We only do this after checking for min/max to prevent infinite | ||||||||
5729 | // looping caused by a reverse canonicalization of these patterns for min/max. | ||||||||
5730 | // FIXME: The organization of folds is a mess. These would naturally go into | ||||||||
5731 | // canonicalizeCmpWithConstant(), but we can't move all of the above folds | ||||||||
5732 | // down here after the min/max restriction. | ||||||||
5733 | ICmpInst::Predicate Pred = I.getPredicate(); | ||||||||
5734 | const APInt *C; | ||||||||
5735 | if (match(Op1, m_APInt(C))) { | ||||||||
5736 | // For i32: x >u 2147483647 -> x <s 0 -> true if sign bit set | ||||||||
5737 | if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) { | ||||||||
5738 | Constant *Zero = Constant::getNullValue(Op0->getType()); | ||||||||
5739 | return new ICmpInst(ICmpInst::ICMP_SLT, Op0, Zero); | ||||||||
5740 | } | ||||||||
5741 | |||||||||
5742 | // For i32: x <u 2147483648 -> x >s -1 -> true if sign bit clear | ||||||||
5743 | if (Pred == ICmpInst::ICMP_ULT && C->isMinSignedValue()) { | ||||||||
5744 | Constant *AllOnes = Constant::getAllOnesValue(Op0->getType()); | ||||||||
5745 | return new ICmpInst(ICmpInst::ICMP_SGT, Op0, AllOnes); | ||||||||
5746 | } | ||||||||
5747 | } | ||||||||
5748 | |||||||||
5749 | if (Instruction *Res = foldICmpInstWithConstant(I)) | ||||||||
5750 | return Res; | ||||||||
5751 | |||||||||
5752 | // Try to match comparison as a sign bit test. Intentionally do this after | ||||||||
5753 | // foldICmpInstWithConstant() to potentially let other folds to happen first. | ||||||||
5754 | if (Instruction *New = foldSignBitTest(I)) | ||||||||
5755 | return New; | ||||||||
5756 | |||||||||
5757 | if (Instruction *Res = foldICmpInstWithConstantNotInt(I)) | ||||||||
5758 | return Res; | ||||||||
5759 | |||||||||
5760 | // If we can optimize a 'icmp GEP, P' or 'icmp P, GEP', do so now. | ||||||||
5761 | if (GEPOperator *GEP = dyn_cast<GEPOperator>(Op0)) | ||||||||
5762 | if (Instruction *NI = foldGEPICmp(GEP, Op1, I.getPredicate(), I)) | ||||||||
5763 | return NI; | ||||||||
5764 | if (GEPOperator *GEP = dyn_cast<GEPOperator>(Op1)) | ||||||||
5765 | if (Instruction *NI = foldGEPICmp(GEP, Op0, | ||||||||
5766 | ICmpInst::getSwappedPredicate(I.getPredicate()), I)) | ||||||||
5767 | return NI; | ||||||||
5768 | |||||||||
5769 | // Try to optimize equality comparisons against alloca-based pointers. | ||||||||
5770 | if (Op0->getType()->isPointerTy() && I.isEquality()) { | ||||||||
5771 | assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?")(static_cast <bool> (Op1->getType()->isPointerTy( ) && "Comparing pointer with non-pointer?") ? void (0 ) : __assert_fail ("Op1->getType()->isPointerTy() && \"Comparing pointer with non-pointer?\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5771, __extension__ __PRETTY_FUNCTION__)); | ||||||||
5772 | if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0))) | ||||||||
5773 | if (Instruction *New = foldAllocaCmp(I, Alloca, Op1)) | ||||||||
5774 | return New; | ||||||||
5775 | if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op1))) | ||||||||
5776 | if (Instruction *New = foldAllocaCmp(I, Alloca, Op0)) | ||||||||
5777 | return New; | ||||||||
5778 | } | ||||||||
5779 | |||||||||
5780 | if (Instruction *Res = foldICmpBitCast(I, Builder)) | ||||||||
5781 | return Res; | ||||||||
5782 | |||||||||
5783 | // TODO: Hoist this above the min/max bailout. | ||||||||
5784 | if (Instruction *R = foldICmpWithCastOp(I)) | ||||||||
5785 | return R; | ||||||||
5786 | |||||||||
5787 | if (Instruction *Res = foldICmpWithMinMax(I)) | ||||||||
5788 | return Res; | ||||||||
5789 | |||||||||
5790 | { | ||||||||
5791 | Value *A, *B; | ||||||||
5792 | // Transform (A & ~B) == 0 --> (A & B) != 0 | ||||||||
5793 | // and (A & ~B) != 0 --> (A & B) == 0 | ||||||||
5794 | // if A is a power of 2. | ||||||||
5795 | if (match(Op0, m_And(m_Value(A), m_Not(m_Value(B)))) && | ||||||||
5796 | match(Op1, m_Zero()) && | ||||||||
5797 | isKnownToBeAPowerOfTwo(A, false, 0, &I) && I.isEquality()) | ||||||||
5798 | return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(A, B), | ||||||||
5799 | Op1); | ||||||||
5800 | |||||||||
5801 | // ~X < ~Y --> Y < X | ||||||||
5802 | // ~X < C --> X > ~C | ||||||||
5803 | if (match(Op0, m_Not(m_Value(A)))) { | ||||||||
5804 | if (match(Op1, m_Not(m_Value(B)))) | ||||||||
5805 | return new ICmpInst(I.getPredicate(), B, A); | ||||||||
5806 | |||||||||
5807 | const APInt *C; | ||||||||
5808 | if (match(Op1, m_APInt(C))) | ||||||||
5809 | return new ICmpInst(I.getSwappedPredicate(), A, | ||||||||
5810 | ConstantInt::get(Op1->getType(), ~(*C))); | ||||||||
5811 | } | ||||||||
5812 | |||||||||
5813 | Instruction *AddI = nullptr; | ||||||||
5814 | if (match(&I, m_UAddWithOverflow(m_Value(A), m_Value(B), | ||||||||
5815 | m_Instruction(AddI))) && | ||||||||
5816 | isa<IntegerType>(A->getType())) { | ||||||||
5817 | Value *Result; | ||||||||
5818 | Constant *Overflow; | ||||||||
5819 | // m_UAddWithOverflow can match patterns that do not include an explicit | ||||||||
5820 | // "add" instruction, so check the opcode of the matched op. | ||||||||
5821 | if (AddI->getOpcode() == Instruction::Add && | ||||||||
5822 | OptimizeOverflowCheck(Instruction::Add, /*Signed*/ false, A, B, *AddI, | ||||||||
5823 | Result, Overflow)) { | ||||||||
5824 | replaceInstUsesWith(*AddI, Result); | ||||||||
5825 | eraseInstFromFunction(*AddI); | ||||||||
5826 | return replaceInstUsesWith(I, Overflow); | ||||||||
5827 | } | ||||||||
5828 | } | ||||||||
5829 | |||||||||
5830 | // (zext a) * (zext b) --> llvm.umul.with.overflow. | ||||||||
5831 | if (match(Op0, m_Mul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) { | ||||||||
5832 | if (Instruction *R = processUMulZExtIdiom(I, Op0, Op1, *this)) | ||||||||
5833 | return R; | ||||||||
5834 | } | ||||||||
5835 | if (match(Op1, m_Mul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) { | ||||||||
5836 | if (Instruction *R = processUMulZExtIdiom(I, Op1, Op0, *this)) | ||||||||
5837 | return R; | ||||||||
5838 | } | ||||||||
5839 | } | ||||||||
5840 | |||||||||
5841 | if (Instruction *Res = foldICmpEquality(I)) | ||||||||
5842 | return Res; | ||||||||
5843 | |||||||||
5844 | if (Instruction *Res = foldICmpOfUAddOv(I)) | ||||||||
5845 | return Res; | ||||||||
5846 | |||||||||
5847 | // The 'cmpxchg' instruction returns an aggregate containing the old value and | ||||||||
5848 | // an i1 which indicates whether or not we successfully did the swap. | ||||||||
5849 | // | ||||||||
5850 | // Replace comparisons between the old value and the expected value with the | ||||||||
5851 | // indicator that 'cmpxchg' returns. | ||||||||
5852 | // | ||||||||
5853 | // N.B. This transform is only valid when the 'cmpxchg' is not permitted to | ||||||||
5854 | // spuriously fail. In those cases, the old value may equal the expected | ||||||||
5855 | // value but it is possible for the swap to not occur. | ||||||||
5856 | if (I.getPredicate() == ICmpInst::ICMP_EQ) | ||||||||
5857 | if (auto *EVI = dyn_cast<ExtractValueInst>(Op0)) | ||||||||
5858 | if (auto *ACXI = dyn_cast<AtomicCmpXchgInst>(EVI->getAggregateOperand())) | ||||||||
5859 | if (EVI->getIndices()[0] == 0 && ACXI->getCompareOperand() == Op1 && | ||||||||
5860 | !ACXI->isWeak()) | ||||||||
5861 | return ExtractValueInst::Create(ACXI, 1); | ||||||||
5862 | |||||||||
5863 | { | ||||||||
5864 | Value *X; | ||||||||
5865 | const APInt *C; | ||||||||
5866 | // icmp X+Cst, X | ||||||||
5867 | if (match(Op0, m_Add(m_Value(X), m_APInt(C))) && Op1 == X) | ||||||||
5868 | return foldICmpAddOpConst(X, *C, I.getPredicate()); | ||||||||
5869 | |||||||||
5870 | // icmp X, X+Cst | ||||||||
5871 | if (match(Op1, m_Add(m_Value(X), m_APInt(C))) && Op0 == X) | ||||||||
5872 | return foldICmpAddOpConst(X, *C, I.getSwappedPredicate()); | ||||||||
5873 | } | ||||||||
5874 | |||||||||
5875 | if (Instruction *Res = foldICmpWithHighBitMask(I, Builder)) | ||||||||
5876 | return Res; | ||||||||
5877 | |||||||||
5878 | if (I.getType()->isVectorTy()) | ||||||||
5879 | if (Instruction *Res = foldVectorCmp(I, Builder)) | ||||||||
5880 | return Res; | ||||||||
5881 | |||||||||
5882 | return Changed ? &I : nullptr; | ||||||||
5883 | } | ||||||||
5884 | |||||||||
5885 | /// Fold fcmp ([us]itofp x, cst) if possible. | ||||||||
5886 | Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, | ||||||||
5887 | Instruction *LHSI, | ||||||||
5888 | Constant *RHSC) { | ||||||||
5889 | if (!isa<ConstantFP>(RHSC)) return nullptr; | ||||||||
5890 | const APFloat &RHS = cast<ConstantFP>(RHSC)->getValueAPF(); | ||||||||
5891 | |||||||||
5892 | // Get the width of the mantissa. We don't want to hack on conversions that | ||||||||
5893 | // might lose information from the integer, e.g. "i64 -> float" | ||||||||
5894 | int MantissaWidth = LHSI->getType()->getFPMantissaWidth(); | ||||||||
5895 | if (MantissaWidth == -1) return nullptr; // Unknown. | ||||||||
5896 | |||||||||
5897 | IntegerType *IntTy = cast<IntegerType>(LHSI->getOperand(0)->getType()); | ||||||||
5898 | |||||||||
5899 | bool LHSUnsigned = isa<UIToFPInst>(LHSI); | ||||||||
5900 | |||||||||
5901 | if (I.isEquality()) { | ||||||||
5902 | FCmpInst::Predicate P = I.getPredicate(); | ||||||||
5903 | bool IsExact = false; | ||||||||
5904 | APSInt RHSCvt(IntTy->getBitWidth(), LHSUnsigned); | ||||||||
5905 | RHS.convertToInteger(RHSCvt, APFloat::rmNearestTiesToEven, &IsExact); | ||||||||
5906 | |||||||||
5907 | // If the floating point constant isn't an integer value, we know if we will | ||||||||
5908 | // ever compare equal / not equal to it. | ||||||||
5909 | if (!IsExact) { | ||||||||
5910 | // TODO: Can never be -0.0 and other non-representable values | ||||||||
5911 | APFloat RHSRoundInt(RHS); | ||||||||
5912 | RHSRoundInt.roundToIntegral(APFloat::rmNearestTiesToEven); | ||||||||
5913 | if (RHS != RHSRoundInt) { | ||||||||
5914 | if (P == FCmpInst::FCMP_OEQ || P == FCmpInst::FCMP_UEQ) | ||||||||
5915 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
5916 | |||||||||
5917 | assert(P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE)(static_cast <bool> (P == FCmpInst::FCMP_ONE || P == FCmpInst ::FCMP_UNE) ? void (0) : __assert_fail ("P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5917, __extension__ __PRETTY_FUNCTION__)); | ||||||||
5918 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
5919 | } | ||||||||
5920 | } | ||||||||
5921 | |||||||||
5922 | // TODO: If the constant is exactly representable, is it always OK to do | ||||||||
5923 | // equality compares as integer? | ||||||||
5924 | } | ||||||||
5925 | |||||||||
5926 | // Check to see that the input is converted from an integer type that is small | ||||||||
5927 | // enough that preserves all bits. TODO: check here for "known" sign bits. | ||||||||
5928 | // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e. | ||||||||
5929 | unsigned InputSize = IntTy->getScalarSizeInBits(); | ||||||||
5930 | |||||||||
5931 | // Following test does NOT adjust InputSize downwards for signed inputs, | ||||||||
5932 | // because the most negative value still requires all the mantissa bits | ||||||||
5933 | // to distinguish it from one less than that value. | ||||||||
5934 | if ((int)InputSize > MantissaWidth) { | ||||||||
5935 | // Conversion would lose accuracy. Check if loss can impact comparison. | ||||||||
5936 | int Exp = ilogb(RHS); | ||||||||
5937 | if (Exp == APFloat::IEK_Inf) { | ||||||||
5938 | int MaxExponent = ilogb(APFloat::getLargest(RHS.getSemantics())); | ||||||||
5939 | if (MaxExponent < (int)InputSize - !LHSUnsigned) | ||||||||
5940 | // Conversion could create infinity. | ||||||||
5941 | return nullptr; | ||||||||
5942 | } else { | ||||||||
5943 | // Note that if RHS is zero or NaN, then Exp is negative | ||||||||
5944 | // and first condition is trivially false. | ||||||||
5945 | if (MantissaWidth <= Exp && Exp <= (int)InputSize - !LHSUnsigned) | ||||||||
5946 | // Conversion could affect comparison. | ||||||||
5947 | return nullptr; | ||||||||
5948 | } | ||||||||
5949 | } | ||||||||
5950 | |||||||||
5951 | // Otherwise, we can potentially simplify the comparison. We know that it | ||||||||
5952 | // will always come through as an integer value and we know the constant is | ||||||||
5953 | // not a NAN (it would have been previously simplified). | ||||||||
5954 | assert(!RHS.isNaN() && "NaN comparison not already folded!")(static_cast <bool> (!RHS.isNaN() && "NaN comparison not already folded!" ) ? void (0) : __assert_fail ("!RHS.isNaN() && \"NaN comparison not already folded!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5954, __extension__ __PRETTY_FUNCTION__)); | ||||||||
5955 | |||||||||
5956 | ICmpInst::Predicate Pred; | ||||||||
5957 | switch (I.getPredicate()) { | ||||||||
5958 | default: llvm_unreachable("Unexpected predicate!")::llvm::llvm_unreachable_internal("Unexpected predicate!", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5958); | ||||||||
5959 | case FCmpInst::FCMP_UEQ: | ||||||||
5960 | case FCmpInst::FCMP_OEQ: | ||||||||
5961 | Pred = ICmpInst::ICMP_EQ; | ||||||||
5962 | break; | ||||||||
5963 | case FCmpInst::FCMP_UGT: | ||||||||
5964 | case FCmpInst::FCMP_OGT: | ||||||||
5965 | Pred = LHSUnsigned ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_SGT; | ||||||||
5966 | break; | ||||||||
5967 | case FCmpInst::FCMP_UGE: | ||||||||
5968 | case FCmpInst::FCMP_OGE: | ||||||||
5969 | Pred = LHSUnsigned ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_SGE; | ||||||||
5970 | break; | ||||||||
5971 | case FCmpInst::FCMP_ULT: | ||||||||
5972 | case FCmpInst::FCMP_OLT: | ||||||||
5973 | Pred = LHSUnsigned ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_SLT; | ||||||||
5974 | break; | ||||||||
5975 | case FCmpInst::FCMP_ULE: | ||||||||
5976 | case FCmpInst::FCMP_OLE: | ||||||||
5977 | Pred = LHSUnsigned ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_SLE; | ||||||||
5978 | break; | ||||||||
5979 | case FCmpInst::FCMP_UNE: | ||||||||
5980 | case FCmpInst::FCMP_ONE: | ||||||||
5981 | Pred = ICmpInst::ICMP_NE; | ||||||||
5982 | break; | ||||||||
5983 | case FCmpInst::FCMP_ORD: | ||||||||
5984 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
5985 | case FCmpInst::FCMP_UNO: | ||||||||
5986 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
5987 | } | ||||||||
5988 | |||||||||
5989 | // Now we know that the APFloat is a normal number, zero or inf. | ||||||||
5990 | |||||||||
5991 | // See if the FP constant is too large for the integer. For example, | ||||||||
5992 | // comparing an i8 to 300.0. | ||||||||
5993 | unsigned IntWidth = IntTy->getScalarSizeInBits(); | ||||||||
5994 | |||||||||
5995 | if (!LHSUnsigned) { | ||||||||
5996 | // If the RHS value is > SignedMax, fold the comparison. This handles +INF | ||||||||
5997 | // and large values. | ||||||||
5998 | APFloat SMax(RHS.getSemantics()); | ||||||||
5999 | SMax.convertFromAPInt(APInt::getSignedMaxValue(IntWidth), true, | ||||||||
6000 | APFloat::rmNearestTiesToEven); | ||||||||
6001 | if (SMax < RHS) { // smax < 13123.0 | ||||||||
6002 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT || | ||||||||
6003 | Pred == ICmpInst::ICMP_SLE) | ||||||||
6004 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
6005 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
6006 | } | ||||||||
6007 | } else { | ||||||||
6008 | // If the RHS value is > UnsignedMax, fold the comparison. This handles | ||||||||
6009 | // +INF and large values. | ||||||||
6010 | APFloat UMax(RHS.getSemantics()); | ||||||||
6011 | UMax.convertFromAPInt(APInt::getMaxValue(IntWidth), false, | ||||||||
6012 | APFloat::rmNearestTiesToEven); | ||||||||
6013 | if (UMax < RHS) { // umax < 13123.0 | ||||||||
6014 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT || | ||||||||
6015 | Pred == ICmpInst::ICMP_ULE) | ||||||||
6016 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
6017 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
6018 | } | ||||||||
6019 | } | ||||||||
6020 | |||||||||
6021 | if (!LHSUnsigned) { | ||||||||
6022 | // See if the RHS value is < SignedMin. | ||||||||
6023 | APFloat SMin(RHS.getSemantics()); | ||||||||
6024 | SMin.convertFromAPInt(APInt::getSignedMinValue(IntWidth), true, | ||||||||
6025 | APFloat::rmNearestTiesToEven); | ||||||||
6026 | if (SMin > RHS) { // smin > 12312.0 | ||||||||
6027 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT || | ||||||||
6028 | Pred == ICmpInst::ICMP_SGE) | ||||||||
6029 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
6030 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
6031 | } | ||||||||
6032 | } else { | ||||||||
6033 | // See if the RHS value is < UnsignedMin. | ||||||||
6034 | APFloat UMin(RHS.getSemantics()); | ||||||||
6035 | UMin.convertFromAPInt(APInt::getMinValue(IntWidth), false, | ||||||||
6036 | APFloat::rmNearestTiesToEven); | ||||||||
6037 | if (UMin > RHS) { // umin > 12312.0 | ||||||||
6038 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT || | ||||||||
6039 | Pred == ICmpInst::ICMP_UGE) | ||||||||
6040 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
6041 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
6042 | } | ||||||||
6043 | } | ||||||||
6044 | |||||||||
6045 | // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or | ||||||||
6046 | // [0, UMAX], but it may still be fractional. See if it is fractional by | ||||||||
6047 | // casting the FP value to the integer value and back, checking for equality. | ||||||||
6048 | // Don't do this for zero, because -0.0 is not fractional. | ||||||||
6049 | Constant *RHSInt = LHSUnsigned | ||||||||
6050 | ? ConstantExpr::getFPToUI(RHSC, IntTy) | ||||||||
6051 | : ConstantExpr::getFPToSI(RHSC, IntTy); | ||||||||
6052 | if (!RHS.isZero()) { | ||||||||
6053 | bool Equal = LHSUnsigned | ||||||||
6054 | ? ConstantExpr::getUIToFP(RHSInt, RHSC->getType()) == RHSC | ||||||||
6055 | : ConstantExpr::getSIToFP(RHSInt, RHSC->getType()) == RHSC; | ||||||||
6056 | if (!Equal) { | ||||||||
6057 | // If we had a comparison against a fractional value, we have to adjust | ||||||||
6058 | // the compare predicate and sometimes the value. RHSC is rounded towards | ||||||||
6059 | // zero at this point. | ||||||||
6060 | switch (Pred) { | ||||||||
6061 | default: llvm_unreachable("Unexpected integer comparison!")::llvm::llvm_unreachable_internal("Unexpected integer comparison!" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 6061); | ||||||||
6062 | case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true | ||||||||
6063 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
6064 | case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false | ||||||||
6065 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
6066 | case ICmpInst::ICMP_ULE: | ||||||||
6067 | // (float)int <= 4.4 --> int <= 4 | ||||||||
6068 | // (float)int <= -4.4 --> false | ||||||||
6069 | if (RHS.isNegative()) | ||||||||
6070 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
6071 | break; | ||||||||
6072 | case ICmpInst::ICMP_SLE: | ||||||||
6073 | // (float)int <= 4.4 --> int <= 4 | ||||||||
6074 | // (float)int <= -4.4 --> int < -4 | ||||||||
6075 | if (RHS.isNegative()) | ||||||||
6076 | Pred = ICmpInst::ICMP_SLT; | ||||||||
6077 | break; | ||||||||
6078 | case ICmpInst::ICMP_ULT: | ||||||||
6079 | // (float)int < -4.4 --> false | ||||||||
6080 | // (float)int < 4.4 --> int <= 4 | ||||||||
6081 | if (RHS.isNegative()) | ||||||||
6082 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
6083 | Pred = ICmpInst::ICMP_ULE; | ||||||||
6084 | break; | ||||||||
6085 | case ICmpInst::ICMP_SLT: | ||||||||
6086 | // (float)int < -4.4 --> int < -4 | ||||||||
6087 | // (float)int < 4.4 --> int <= 4 | ||||||||
6088 | if (!RHS.isNegative()) | ||||||||
6089 | Pred = ICmpInst::ICMP_SLE; | ||||||||
6090 | break; | ||||||||
6091 | case ICmpInst::ICMP_UGT: | ||||||||
6092 | // (float)int > 4.4 --> int > 4 | ||||||||
6093 | // (float)int > -4.4 --> true | ||||||||
6094 | if (RHS.isNegative()) | ||||||||
6095 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
6096 | break; | ||||||||
6097 | case ICmpInst::ICMP_SGT: | ||||||||
6098 | // (float)int > 4.4 --> int > 4 | ||||||||
6099 | // (float)int > -4.4 --> int >= -4 | ||||||||
6100 | if (RHS.isNegative()) | ||||||||
6101 | Pred = ICmpInst::ICMP_SGE; | ||||||||
6102 | break; | ||||||||
6103 | case ICmpInst::ICMP_UGE: | ||||||||
6104 | // (float)int >= -4.4 --> true | ||||||||
6105 | // (float)int >= 4.4 --> int > 4 | ||||||||
6106 | if (RHS.isNegative()) | ||||||||
6107 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
6108 | Pred = ICmpInst::ICMP_UGT; | ||||||||
6109 | break; | ||||||||
6110 | case ICmpInst::ICMP_SGE: | ||||||||
6111 | // (float)int >= -4.4 --> int >= -4 | ||||||||
6112 | // (float)int >= 4.4 --> int > 4 | ||||||||
6113 | if (!RHS.isNegative()) | ||||||||
6114 | Pred = ICmpInst::ICMP_SGT; | ||||||||
6115 | break; | ||||||||
6116 | } | ||||||||
6117 | } | ||||||||
6118 | } | ||||||||
6119 | |||||||||
6120 | // Lower this FP comparison into an appropriate integer version of the | ||||||||
6121 | // comparison. | ||||||||
6122 | return new ICmpInst(Pred, LHSI->getOperand(0), RHSInt); | ||||||||
6123 | } | ||||||||
6124 | |||||||||
6125 | /// Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary. | ||||||||
6126 | static Instruction *foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI, | ||||||||
6127 | Constant *RHSC) { | ||||||||
6128 | // When C is not 0.0 and infinities are not allowed: | ||||||||
6129 | // (C / X) < 0.0 is a sign-bit test of X | ||||||||
6130 | // (C / X) < 0.0 --> X < 0.0 (if C is positive) | ||||||||
6131 | // (C / X) < 0.0 --> X > 0.0 (if C is negative, swap the predicate) | ||||||||
6132 | // | ||||||||
6133 | // Proof: | ||||||||
6134 | // Multiply (C / X) < 0.0 by X * X / C. | ||||||||
6135 | // - X is non zero, if it is the flag 'ninf' is violated. | ||||||||
6136 | // - C defines the sign of X * X * C. Thus it also defines whether to swap | ||||||||
6137 | // the predicate. C is also non zero by definition. | ||||||||
6138 | // | ||||||||
6139 | // Thus X * X / C is non zero and the transformation is valid. [qed] | ||||||||
6140 | |||||||||
6141 | FCmpInst::Predicate Pred = I.getPredicate(); | ||||||||
6142 | |||||||||
6143 | // Check that predicates are valid. | ||||||||
6144 | if ((Pred != FCmpInst::FCMP_OGT) && (Pred != FCmpInst::FCMP_OLT) && | ||||||||
6145 | (Pred != FCmpInst::FCMP_OGE) && (Pred != FCmpInst::FCMP_OLE)) | ||||||||
6146 | return nullptr; | ||||||||
6147 | |||||||||
6148 | // Check that RHS operand is zero. | ||||||||
6149 | if (!match(RHSC, m_AnyZeroFP())) | ||||||||
6150 | return nullptr; | ||||||||
6151 | |||||||||
6152 | // Check fastmath flags ('ninf'). | ||||||||
6153 | if (!LHSI->hasNoInfs() || !I.hasNoInfs()) | ||||||||
6154 | return nullptr; | ||||||||
6155 | |||||||||
6156 | // Check the properties of the dividend. It must not be zero to avoid a | ||||||||
6157 | // division by zero (see Proof). | ||||||||
6158 | const APFloat *C; | ||||||||
6159 | if (!match(LHSI->getOperand(0), m_APFloat(C))) | ||||||||
6160 | return nullptr; | ||||||||
6161 | |||||||||
6162 | if (C->isZero()) | ||||||||
6163 | return nullptr; | ||||||||
6164 | |||||||||
6165 | // Get swapped predicate if necessary. | ||||||||
6166 | if (C->isNegative()) | ||||||||
6167 | Pred = I.getSwappedPredicate(); | ||||||||
6168 | |||||||||
6169 | return new FCmpInst(Pred, LHSI->getOperand(1), RHSC, "", &I); | ||||||||
6170 | } | ||||||||
6171 | |||||||||
6172 | /// Optimize fabs(X) compared with zero. | ||||||||
6173 | static Instruction *foldFabsWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC) { | ||||||||
6174 | Value *X; | ||||||||
6175 | if (!match(I.getOperand(0), m_FAbs(m_Value(X))) || | ||||||||
6176 | !match(I.getOperand(1), m_PosZeroFP())) | ||||||||
6177 | return nullptr; | ||||||||
6178 | |||||||||
6179 | auto replacePredAndOp0 = [&IC](FCmpInst *I, FCmpInst::Predicate P, Value *X) { | ||||||||
6180 | I->setPredicate(P); | ||||||||
6181 | return IC.replaceOperand(*I, 0, X); | ||||||||
6182 | }; | ||||||||
6183 | |||||||||
6184 | switch (I.getPredicate()) { | ||||||||
6185 | case FCmpInst::FCMP_UGE: | ||||||||
6186 | case FCmpInst::FCMP_OLT: | ||||||||
6187 | // fabs(X) >= 0.0 --> true | ||||||||
6188 | // fabs(X) < 0.0 --> false | ||||||||
6189 | llvm_unreachable("fcmp should have simplified")::llvm::llvm_unreachable_internal("fcmp should have simplified" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 6189); | ||||||||
6190 | |||||||||
6191 | case FCmpInst::FCMP_OGT: | ||||||||
6192 | // fabs(X) > 0.0 --> X != 0.0 | ||||||||
6193 | return replacePredAndOp0(&I, FCmpInst::FCMP_ONE, X); | ||||||||
6194 | |||||||||
6195 | case FCmpInst::FCMP_UGT: | ||||||||
6196 | // fabs(X) u> 0.0 --> X u!= 0.0 | ||||||||
6197 | return replacePredAndOp0(&I, FCmpInst::FCMP_UNE, X); | ||||||||
6198 | |||||||||
6199 | case FCmpInst::FCMP_OLE: | ||||||||
6200 | // fabs(X) <= 0.0 --> X == 0.0 | ||||||||
6201 | return replacePredAndOp0(&I, FCmpInst::FCMP_OEQ, X); | ||||||||
6202 | |||||||||
6203 | case FCmpInst::FCMP_ULE: | ||||||||
6204 | // fabs(X) u<= 0.0 --> X u== 0.0 | ||||||||
6205 | return replacePredAndOp0(&I, FCmpInst::FCMP_UEQ, X); | ||||||||
6206 | |||||||||
6207 | case FCmpInst::FCMP_OGE: | ||||||||
6208 | // fabs(X) >= 0.0 --> !isnan(X) | ||||||||
6209 | assert(!I.hasNoNaNs() && "fcmp should have simplified")(static_cast <bool> (!I.hasNoNaNs() && "fcmp should have simplified" ) ? void (0) : __assert_fail ("!I.hasNoNaNs() && \"fcmp should have simplified\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 6209, __extension__ __PRETTY_FUNCTION__)); | ||||||||
6210 | return replacePredAndOp0(&I, FCmpInst::FCMP_ORD, X); | ||||||||
6211 | |||||||||
6212 | case FCmpInst::FCMP_ULT: | ||||||||
6213 | // fabs(X) u< 0.0 --> isnan(X) | ||||||||
6214 | assert(!I.hasNoNaNs() && "fcmp should have simplified")(static_cast <bool> (!I.hasNoNaNs() && "fcmp should have simplified" ) ? void (0) : __assert_fail ("!I.hasNoNaNs() && \"fcmp should have simplified\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 6214, __extension__ __PRETTY_FUNCTION__)); | ||||||||
6215 | return replacePredAndOp0(&I, FCmpInst::FCMP_UNO, X); | ||||||||
6216 | |||||||||
6217 | case FCmpInst::FCMP_OEQ: | ||||||||
6218 | case FCmpInst::FCMP_UEQ: | ||||||||
6219 | case FCmpInst::FCMP_ONE: | ||||||||
6220 | case FCmpInst::FCMP_UNE: | ||||||||
6221 | case FCmpInst::FCMP_ORD: | ||||||||
6222 | case FCmpInst::FCMP_UNO: | ||||||||
6223 | // Look through the fabs() because it doesn't change anything but the sign. | ||||||||
6224 | // fabs(X) == 0.0 --> X == 0.0, | ||||||||
6225 | // fabs(X) != 0.0 --> X != 0.0 | ||||||||
6226 | // isnan(fabs(X)) --> isnan(X) | ||||||||
6227 | // !isnan(fabs(X) --> !isnan(X) | ||||||||
6228 | return replacePredAndOp0(&I, I.getPredicate(), X); | ||||||||
6229 | |||||||||
6230 | default: | ||||||||
6231 | return nullptr; | ||||||||
6232 | } | ||||||||
6233 | } | ||||||||
6234 | |||||||||
6235 | Instruction *InstCombinerImpl::visitFCmpInst(FCmpInst &I) { | ||||||||
6236 | bool Changed = false; | ||||||||
6237 | |||||||||
6238 | /// Orders the operands of the compare so that they are listed from most | ||||||||
6239 | /// complex to least complex. This puts constants before unary operators, | ||||||||
6240 | /// before binary operators. | ||||||||
6241 | if (getComplexity(I.getOperand(0)) < getComplexity(I.getOperand(1))) { | ||||||||
6242 | I.swapOperands(); | ||||||||
6243 | Changed = true; | ||||||||
6244 | } | ||||||||
6245 | |||||||||
6246 | const CmpInst::Predicate Pred = I.getPredicate(); | ||||||||
6247 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
6248 | if (Value *V = SimplifyFCmpInst(Pred, Op0, Op1, I.getFastMathFlags(), | ||||||||
6249 | SQ.getWithInstruction(&I))) | ||||||||
6250 | return replaceInstUsesWith(I, V); | ||||||||
6251 | |||||||||
6252 | // Simplify 'fcmp pred X, X' | ||||||||
6253 | Type *OpType = Op0->getType(); | ||||||||
6254 | assert(OpType == Op1->getType() && "fcmp with different-typed operands?")(static_cast <bool> (OpType == Op1->getType() && "fcmp with different-typed operands?") ? void (0) : __assert_fail ("OpType == Op1->getType() && \"fcmp with different-typed operands?\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 6254, __extension__ __PRETTY_FUNCTION__)); | ||||||||
6255 | if (Op0 == Op1) { | ||||||||
6256 | switch (Pred) { | ||||||||
6257 | default: break; | ||||||||
6258 | case FCmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y) | ||||||||
6259 | case FCmpInst::FCMP_ULT: // True if unordered or less than | ||||||||
6260 | case FCmpInst::FCMP_UGT: // True if unordered or greater than | ||||||||
6261 | case FCmpInst::FCMP_UNE: // True if unordered or not equal | ||||||||
6262 | // Canonicalize these to be 'fcmp uno %X, 0.0'. | ||||||||
6263 | I.setPredicate(FCmpInst::FCMP_UNO); | ||||||||
6264 | I.setOperand(1, Constant::getNullValue(OpType)); | ||||||||
6265 | return &I; | ||||||||
6266 | |||||||||
6267 | case FCmpInst::FCMP_ORD: // True if ordered (no nans) | ||||||||
6268 | case FCmpInst::FCMP_OEQ: // True if ordered and equal | ||||||||
6269 | case FCmpInst::FCMP_OGE: // True if ordered and greater than or equal | ||||||||
6270 | case FCmpInst::FCMP_OLE: // True if ordered and less than or equal | ||||||||
6271 | // Canonicalize these to be 'fcmp ord %X, 0.0'. | ||||||||
6272 | I.setPredicate(FCmpInst::FCMP_ORD); | ||||||||
6273 | I.setOperand(1, Constant::getNullValue(OpType)); | ||||||||
6274 | return &I; | ||||||||
6275 | } | ||||||||
6276 | } | ||||||||
6277 | |||||||||
6278 | // If we're just checking for a NaN (ORD/UNO) and have a non-NaN operand, | ||||||||
6279 | // then canonicalize the operand to 0.0. | ||||||||
6280 | if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) { | ||||||||
6281 | if (!match(Op0, m_PosZeroFP()) && isKnownNeverNaN(Op0, &TLI)) | ||||||||
6282 | return replaceOperand(I, 0, ConstantFP::getNullValue(OpType)); | ||||||||
6283 | |||||||||
6284 | if (!match(Op1, m_PosZeroFP()) && isKnownNeverNaN(Op1, &TLI)) | ||||||||
6285 | return replaceOperand(I, 1, ConstantFP::getNullValue(OpType)); | ||||||||
6286 | } | ||||||||
6287 | |||||||||
6288 | // fcmp pred (fneg X), (fneg Y) -> fcmp swap(pred) X, Y | ||||||||
6289 | Value *X, *Y; | ||||||||
6290 | if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_FNeg(m_Value(Y)))) | ||||||||
6291 | return new FCmpInst(I.getSwappedPredicate(), X, Y, "", &I); | ||||||||
6292 | |||||||||
6293 | // Test if the FCmpInst instruction is used exclusively by a select as | ||||||||
6294 | // part of a minimum or maximum operation. If so, refrain from doing | ||||||||
6295 | // any other folding. This helps out other analyses which understand | ||||||||
6296 | // non-obfuscated minimum and maximum idioms, such as ScalarEvolution | ||||||||
6297 | // and CodeGen. And in this case, at least one of the comparison | ||||||||
6298 | // operands has at least one user besides the compare (the select), | ||||||||
6299 | // which would often largely negate the benefit of folding anyway. | ||||||||
6300 | if (I.hasOneUse()) | ||||||||
6301 | if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) { | ||||||||
6302 | Value *A, *B; | ||||||||
6303 | SelectPatternResult SPR = matchSelectPattern(SI, A, B); | ||||||||
6304 | if (SPR.Flavor != SPF_UNKNOWN) | ||||||||
6305 | return nullptr; | ||||||||
6306 | } | ||||||||
6307 | |||||||||
6308 | // The sign of 0.0 is ignored by fcmp, so canonicalize to +0.0: | ||||||||
6309 | // fcmp Pred X, -0.0 --> fcmp Pred X, 0.0 | ||||||||
6310 | if (match(Op1, m_AnyZeroFP()) && !match(Op1, m_PosZeroFP())) | ||||||||
6311 | return replaceOperand(I, 1, ConstantFP::getNullValue(OpType)); | ||||||||
6312 | |||||||||
6313 | // Handle fcmp with instruction LHS and constant RHS. | ||||||||
6314 | Instruction *LHSI; | ||||||||
6315 | Constant *RHSC; | ||||||||
6316 | if (match(Op0, m_Instruction(LHSI)) && match(Op1, m_Constant(RHSC))) { | ||||||||
6317 | switch (LHSI->getOpcode()) { | ||||||||
6318 | case Instruction::PHI: | ||||||||
6319 | // Only fold fcmp into the PHI if the phi and fcmp are in the same | ||||||||
6320 | // block. If in the same block, we're encouraging jump threading. If | ||||||||
6321 | // not, we are just pessimizing the code by making an i1 phi. | ||||||||
6322 | if (LHSI->getParent() == I.getParent()) | ||||||||
6323 | if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI))) | ||||||||
6324 | return NV; | ||||||||
6325 | break; | ||||||||
6326 | case Instruction::SIToFP: | ||||||||
6327 | case Instruction::UIToFP: | ||||||||
6328 | if (Instruction *NV = foldFCmpIntToFPConst(I, LHSI, RHSC)) | ||||||||
6329 | return NV; | ||||||||
6330 | break; | ||||||||
6331 | case Instruction::FDiv: | ||||||||
6332 | if (Instruction *NV = foldFCmpReciprocalAndZero(I, LHSI, RHSC)) | ||||||||
6333 | return NV; | ||||||||
6334 | break; | ||||||||
6335 | case Instruction::Load: | ||||||||
6336 | if (auto *GEP = dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) | ||||||||
6337 | if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) | ||||||||
6338 | if (GV->isConstant() && GV->hasDefinitiveInitializer() && | ||||||||
6339 | !cast<LoadInst>(LHSI)->isVolatile()) | ||||||||
6340 | if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, I)) | ||||||||
6341 | return Res; | ||||||||
6342 | break; | ||||||||
6343 | } | ||||||||
6344 | } | ||||||||
6345 | |||||||||
6346 | if (Instruction *R = foldFabsWithFcmpZero(I, *this)) | ||||||||
6347 | return R; | ||||||||
6348 | |||||||||
6349 | if (match(Op0, m_FNeg(m_Value(X)))) { | ||||||||
6350 | // fcmp pred (fneg X), C --> fcmp swap(pred) X, -C | ||||||||
6351 | Constant *C; | ||||||||
6352 | if (match(Op1, m_Constant(C))) { | ||||||||
6353 | Constant *NegC = ConstantExpr::getFNeg(C); | ||||||||
6354 | return new FCmpInst(I.getSwappedPredicate(), X, NegC, "", &I); | ||||||||
6355 | } | ||||||||
6356 | } | ||||||||
6357 | |||||||||
6358 | if (match(Op0, m_FPExt(m_Value(X)))) { | ||||||||
6359 | // fcmp (fpext X), (fpext Y) -> fcmp X, Y | ||||||||
6360 | if (match(Op1, m_FPExt(m_Value(Y))) && X->getType() == Y->getType()) | ||||||||
6361 | return new FCmpInst(Pred, X, Y, "", &I); | ||||||||
6362 | |||||||||
6363 | // fcmp (fpext X), C -> fcmp X, (fptrunc C) if fptrunc is lossless | ||||||||
6364 | const APFloat *C; | ||||||||
6365 | if (match(Op1, m_APFloat(C))) { | ||||||||
6366 | const fltSemantics &FPSem = | ||||||||
6367 | X->getType()->getScalarType()->getFltSemantics(); | ||||||||
6368 | bool Lossy; | ||||||||
6369 | APFloat TruncC = *C; | ||||||||
6370 | TruncC.convert(FPSem, APFloat::rmNearestTiesToEven, &Lossy); | ||||||||
6371 | |||||||||
6372 | // Avoid lossy conversions and denormals. | ||||||||
6373 | // Zero is a special case that's OK to convert. | ||||||||
6374 | APFloat Fabs = TruncC; | ||||||||
6375 | Fabs.clearSign(); | ||||||||
6376 | if (!Lossy && | ||||||||
6377 | (!(Fabs < APFloat::getSmallestNormalized(FPSem)) || Fabs.isZero())) { | ||||||||
6378 | Constant *NewC = ConstantFP::get(X->getType(), TruncC); | ||||||||
6379 | return new FCmpInst(Pred, X, NewC, "", &I); | ||||||||
6380 | } | ||||||||
6381 | } | ||||||||
6382 | } | ||||||||
6383 | |||||||||
6384 | // Convert a sign-bit test of an FP value into a cast and integer compare. | ||||||||
6385 | // TODO: Simplify if the copysign constant is 0.0 or NaN. | ||||||||
6386 | // TODO: Handle non-zero compare constants. | ||||||||
6387 | // TODO: Handle other predicates. | ||||||||
6388 | const APFloat *C; | ||||||||
6389 | if (match(Op0, m_OneUse(m_Intrinsic<Intrinsic::copysign>(m_APFloat(C), | ||||||||
6390 | m_Value(X)))) && | ||||||||
6391 | match(Op1, m_AnyZeroFP()) && !C->isZero() && !C->isNaN()) { | ||||||||
6392 | Type *IntType = Builder.getIntNTy(X->getType()->getScalarSizeInBits()); | ||||||||
6393 | if (auto *VecTy = dyn_cast<VectorType>(OpType)) | ||||||||
6394 | IntType = VectorType::get(IntType, VecTy->getElementCount()); | ||||||||
6395 | |||||||||
6396 | // copysign(non-zero constant, X) < 0.0 --> (bitcast X) < 0 | ||||||||
6397 | if (Pred == FCmpInst::FCMP_OLT) { | ||||||||
6398 | Value *IntX = Builder.CreateBitCast(X, IntType); | ||||||||
6399 | return new ICmpInst(ICmpInst::ICMP_SLT, IntX, | ||||||||
6400 | ConstantInt::getNullValue(IntType)); | ||||||||
6401 | } | ||||||||
6402 | } | ||||||||
6403 | |||||||||
6404 | if (I.getType()->isVectorTy()) | ||||||||
6405 | if (Instruction *Res = foldVectorCmp(I, Builder)) | ||||||||
6406 | return Res; | ||||||||
6407 | |||||||||
6408 | return Changed ? &I : nullptr; | ||||||||
6409 | } |
1 | //===- PatternMatch.h - Match on the LLVM IR --------------------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file provides a simple and efficient mechanism for performing general |
10 | // tree-based pattern matches on the LLVM IR. The power of these routines is |
11 | // that it allows you to write concise patterns that are expressive and easy to |
12 | // understand. The other major advantage of this is that it allows you to |
13 | // trivially capture/bind elements in the pattern to variables. For example, |
14 | // you can do something like this: |
15 | // |
16 | // Value *Exp = ... |
17 | // Value *X, *Y; ConstantInt *C1, *C2; // (X & C1) | (Y & C2) |
18 | // if (match(Exp, m_Or(m_And(m_Value(X), m_ConstantInt(C1)), |
19 | // m_And(m_Value(Y), m_ConstantInt(C2))))) { |
20 | // ... Pattern is matched and variables are bound ... |
21 | // } |
22 | // |
23 | // This is primarily useful to things like the instruction combiner, but can |
24 | // also be useful for static analysis tools or code generators. |
25 | // |
26 | //===----------------------------------------------------------------------===// |
27 | |
28 | #ifndef LLVM_IR_PATTERNMATCH_H |
29 | #define LLVM_IR_PATTERNMATCH_H |
30 | |
31 | #include "llvm/ADT/APFloat.h" |
32 | #include "llvm/ADT/APInt.h" |
33 | #include "llvm/IR/Constant.h" |
34 | #include "llvm/IR/Constants.h" |
35 | #include "llvm/IR/DataLayout.h" |
36 | #include "llvm/IR/InstrTypes.h" |
37 | #include "llvm/IR/Instruction.h" |
38 | #include "llvm/IR/Instructions.h" |
39 | #include "llvm/IR/IntrinsicInst.h" |
40 | #include "llvm/IR/Intrinsics.h" |
41 | #include "llvm/IR/Operator.h" |
42 | #include "llvm/IR/Value.h" |
43 | #include "llvm/Support/Casting.h" |
44 | #include <cstdint> |
45 | |
46 | namespace llvm { |
47 | namespace PatternMatch { |
48 | |
49 | template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) { |
50 | return const_cast<Pattern &>(P).match(V); |
51 | } |
52 | |
53 | template <typename Pattern> bool match(ArrayRef<int> Mask, const Pattern &P) { |
54 | return const_cast<Pattern &>(P).match(Mask); |
55 | } |
56 | |
57 | template <typename SubPattern_t> struct OneUse_match { |
58 | SubPattern_t SubPattern; |
59 | |
60 | OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {} |
61 | |
62 | template <typename OpTy> bool match(OpTy *V) { |
63 | return V->hasOneUse() && SubPattern.match(V); |
64 | } |
65 | }; |
66 | |
67 | template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) { |
68 | return SubPattern; |
69 | } |
70 | |
71 | template <typename Class> struct class_match { |
72 | template <typename ITy> bool match(ITy *V) { return isa<Class>(V); } |
73 | }; |
74 | |
75 | /// Match an arbitrary value and ignore it. |
76 | inline class_match<Value> m_Value() { return class_match<Value>(); } |
77 | |
78 | /// Match an arbitrary unary operation and ignore it. |
79 | inline class_match<UnaryOperator> m_UnOp() { |
80 | return class_match<UnaryOperator>(); |
81 | } |
82 | |
83 | /// Match an arbitrary binary operation and ignore it. |
84 | inline class_match<BinaryOperator> m_BinOp() { |
85 | return class_match<BinaryOperator>(); |
86 | } |
87 | |
88 | /// Matches any compare instruction and ignore it. |
89 | inline class_match<CmpInst> m_Cmp() { return class_match<CmpInst>(); } |
90 | |
91 | struct undef_match { |
92 | static bool check(const Value *V) { |
93 | if (isa<UndefValue>(V)) |
94 | return true; |
95 | |
96 | const auto *CA = dyn_cast<ConstantAggregate>(V); |
97 | if (!CA) |
98 | return false; |
99 | |
100 | SmallPtrSet<const ConstantAggregate *, 8> Seen; |
101 | SmallVector<const ConstantAggregate *, 8> Worklist; |
102 | |
103 | // Either UndefValue, PoisonValue, or an aggregate that only contains |
104 | // these is accepted by matcher. |
105 | // CheckValue returns false if CA cannot satisfy this constraint. |
106 | auto CheckValue = [&](const ConstantAggregate *CA) { |
107 | for (const Value *Op : CA->operand_values()) { |
108 | if (isa<UndefValue>(Op)) |
109 | continue; |
110 | |
111 | const auto *CA = dyn_cast<ConstantAggregate>(Op); |
112 | if (!CA) |
113 | return false; |
114 | if (Seen.insert(CA).second) |
115 | Worklist.emplace_back(CA); |
116 | } |
117 | |
118 | return true; |
119 | }; |
120 | |
121 | if (!CheckValue(CA)) |
122 | return false; |
123 | |
124 | while (!Worklist.empty()) { |
125 | if (!CheckValue(Worklist.pop_back_val())) |
126 | return false; |
127 | } |
128 | return true; |
129 | } |
130 | template <typename ITy> bool match(ITy *V) { return check(V); } |
131 | }; |
132 | |
133 | /// Match an arbitrary undef constant. This matches poison as well. |
134 | /// If this is an aggregate and contains a non-aggregate element that is |
135 | /// neither undef nor poison, the aggregate is not matched. |
136 | inline auto m_Undef() { return undef_match(); } |
137 | |
138 | /// Match an arbitrary poison constant. |
139 | inline class_match<PoisonValue> m_Poison() { return class_match<PoisonValue>(); } |
140 | |
141 | /// Match an arbitrary Constant and ignore it. |
142 | inline class_match<Constant> m_Constant() { return class_match<Constant>(); } |
143 | |
144 | /// Match an arbitrary ConstantInt and ignore it. |
145 | inline class_match<ConstantInt> m_ConstantInt() { |
146 | return class_match<ConstantInt>(); |
147 | } |
148 | |
149 | /// Match an arbitrary ConstantFP and ignore it. |
150 | inline class_match<ConstantFP> m_ConstantFP() { |
151 | return class_match<ConstantFP>(); |
152 | } |
153 | |
154 | /// Match an arbitrary ConstantExpr and ignore it. |
155 | inline class_match<ConstantExpr> m_ConstantExpr() { |
156 | return class_match<ConstantExpr>(); |
157 | } |
158 | |
159 | /// Match an arbitrary basic block value and ignore it. |
160 | inline class_match<BasicBlock> m_BasicBlock() { |
161 | return class_match<BasicBlock>(); |
162 | } |
163 | |
164 | /// Inverting matcher |
165 | template <typename Ty> struct match_unless { |
166 | Ty M; |
167 | |
168 | match_unless(const Ty &Matcher) : M(Matcher) {} |
169 | |
170 | template <typename ITy> bool match(ITy *V) { return !M.match(V); } |
171 | }; |
172 | |
173 | /// Match if the inner matcher does *NOT* match. |
174 | template <typename Ty> inline match_unless<Ty> m_Unless(const Ty &M) { |
175 | return match_unless<Ty>(M); |
176 | } |
177 | |
178 | /// Matching combinators |
179 | template <typename LTy, typename RTy> struct match_combine_or { |
180 | LTy L; |
181 | RTy R; |
182 | |
183 | match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} |
184 | |
185 | template <typename ITy> bool match(ITy *V) { |
186 | if (L.match(V)) |
187 | return true; |
188 | if (R.match(V)) |
189 | return true; |
190 | return false; |
191 | } |
192 | }; |
193 | |
194 | template <typename LTy, typename RTy> struct match_combine_and { |
195 | LTy L; |
196 | RTy R; |
197 | |
198 | match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} |
199 | |
200 | template <typename ITy> bool match(ITy *V) { |
201 | if (L.match(V)) |
202 | if (R.match(V)) |
203 | return true; |
204 | return false; |
205 | } |
206 | }; |
207 | |
208 | /// Combine two pattern matchers matching L || R |
209 | template <typename LTy, typename RTy> |
210 | inline match_combine_or<LTy, RTy> m_CombineOr(const LTy &L, const RTy &R) { |
211 | return match_combine_or<LTy, RTy>(L, R); |
212 | } |
213 | |
214 | /// Combine two pattern matchers matching L && R |
215 | template <typename LTy, typename RTy> |
216 | inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) { |
217 | return match_combine_and<LTy, RTy>(L, R); |
218 | } |
219 | |
220 | struct apint_match { |
221 | const APInt *&Res; |
222 | bool AllowUndef; |
223 | |
224 | apint_match(const APInt *&Res, bool AllowUndef) |
225 | : Res(Res), AllowUndef(AllowUndef) {} |
226 | |
227 | template <typename ITy> bool match(ITy *V) { |
228 | if (auto *CI = dyn_cast<ConstantInt>(V)) { |
229 | Res = &CI->getValue(); |
230 | return true; |
231 | } |
232 | if (V->getType()->isVectorTy()) |
233 | if (const auto *C = dyn_cast<Constant>(V)) |
234 | if (auto *CI = dyn_cast_or_null<ConstantInt>( |
235 | C->getSplatValue(AllowUndef))) { |
236 | Res = &CI->getValue(); |
237 | return true; |
238 | } |
239 | return false; |
240 | } |
241 | }; |
242 | // Either constexpr if or renaming ConstantFP::getValueAPF to |
243 | // ConstantFP::getValue is needed to do it via single template |
244 | // function for both apint/apfloat. |
245 | struct apfloat_match { |
246 | const APFloat *&Res; |
247 | bool AllowUndef; |
248 | |
249 | apfloat_match(const APFloat *&Res, bool AllowUndef) |
250 | : Res(Res), AllowUndef(AllowUndef) {} |
251 | |
252 | template <typename ITy> bool match(ITy *V) { |
253 | if (auto *CI = dyn_cast<ConstantFP>(V)) { |
254 | Res = &CI->getValueAPF(); |
255 | return true; |
256 | } |
257 | if (V->getType()->isVectorTy()) |
258 | if (const auto *C = dyn_cast<Constant>(V)) |
259 | if (auto *CI = dyn_cast_or_null<ConstantFP>( |
260 | C->getSplatValue(AllowUndef))) { |
261 | Res = &CI->getValueAPF(); |
262 | return true; |
263 | } |
264 | return false; |
265 | } |
266 | }; |
267 | |
268 | /// Match a ConstantInt or splatted ConstantVector, binding the |
269 | /// specified pointer to the contained APInt. |
270 | inline apint_match m_APInt(const APInt *&Res) { |
271 | // Forbid undefs by default to maintain previous behavior. |
272 | return apint_match(Res, /* AllowUndef */ false); |
273 | } |
274 | |
275 | /// Match APInt while allowing undefs in splat vector constants. |
276 | inline apint_match m_APIntAllowUndef(const APInt *&Res) { |
277 | return apint_match(Res, /* AllowUndef */ true); |
278 | } |
279 | |
280 | /// Match APInt while forbidding undefs in splat vector constants. |
281 | inline apint_match m_APIntForbidUndef(const APInt *&Res) { |
282 | return apint_match(Res, /* AllowUndef */ false); |
283 | } |
284 | |
285 | /// Match a ConstantFP or splatted ConstantVector, binding the |
286 | /// specified pointer to the contained APFloat. |
287 | inline apfloat_match m_APFloat(const APFloat *&Res) { |
288 | // Forbid undefs by default to maintain previous behavior. |
289 | return apfloat_match(Res, /* AllowUndef */ false); |
290 | } |
291 | |
292 | /// Match APFloat while allowing undefs in splat vector constants. |
293 | inline apfloat_match m_APFloatAllowUndef(const APFloat *&Res) { |
294 | return apfloat_match(Res, /* AllowUndef */ true); |
295 | } |
296 | |
297 | /// Match APFloat while forbidding undefs in splat vector constants. |
298 | inline apfloat_match m_APFloatForbidUndef(const APFloat *&Res) { |
299 | return apfloat_match(Res, /* AllowUndef */ false); |
300 | } |
301 | |
302 | template <int64_t Val> struct constantint_match { |
303 | template <typename ITy> bool match(ITy *V) { |
304 | if (const auto *CI = dyn_cast<ConstantInt>(V)) { |
305 | const APInt &CIV = CI->getValue(); |
306 | if (Val >= 0) |
307 | return CIV == static_cast<uint64_t>(Val); |
308 | // If Val is negative, and CI is shorter than it, truncate to the right |
309 | // number of bits. If it is larger, then we have to sign extend. Just |
310 | // compare their negated values. |
311 | return -CIV == -Val; |
312 | } |
313 | return false; |
314 | } |
315 | }; |
316 | |
317 | /// Match a ConstantInt with a specific value. |
318 | template <int64_t Val> inline constantint_match<Val> m_ConstantInt() { |
319 | return constantint_match<Val>(); |
320 | } |
321 | |
322 | /// This helper class is used to match constant scalars, vector splats, |
323 | /// and fixed width vectors that satisfy a specified predicate. |
324 | /// For fixed width vector constants, undefined elements are ignored. |
325 | template <typename Predicate, typename ConstantVal> |
326 | struct cstval_pred_ty : public Predicate { |
327 | template <typename ITy> bool match(ITy *V) { |
328 | if (const auto *CV = dyn_cast<ConstantVal>(V)) |
329 | return this->isValue(CV->getValue()); |
330 | if (const auto *VTy = dyn_cast<VectorType>(V->getType())) { |
331 | if (const auto *C = dyn_cast<Constant>(V)) { |
332 | if (const auto *CV = dyn_cast_or_null<ConstantVal>(C->getSplatValue())) |
333 | return this->isValue(CV->getValue()); |
334 | |
335 | // Number of elements of a scalable vector unknown at compile time |
336 | auto *FVTy = dyn_cast<FixedVectorType>(VTy); |
337 | if (!FVTy) |
338 | return false; |
339 | |
340 | // Non-splat vector constant: check each element for a match. |
341 | unsigned NumElts = FVTy->getNumElements(); |
342 | assert(NumElts != 0 && "Constant vector with no elements?")(static_cast <bool> (NumElts != 0 && "Constant vector with no elements?" ) ? void (0) : __assert_fail ("NumElts != 0 && \"Constant vector with no elements?\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/IR/PatternMatch.h" , 342, __extension__ __PRETTY_FUNCTION__)); |
343 | bool HasNonUndefElements = false; |
344 | for (unsigned i = 0; i != NumElts; ++i) { |
345 | Constant *Elt = C->getAggregateElement(i); |
346 | if (!Elt) |
347 | return false; |
348 | if (isa<UndefValue>(Elt)) |
349 | continue; |
350 | auto *CV = dyn_cast<ConstantVal>(Elt); |
351 | if (!CV || !this->isValue(CV->getValue())) |
352 | return false; |
353 | HasNonUndefElements = true; |
354 | } |
355 | return HasNonUndefElements; |
356 | } |
357 | } |
358 | return false; |
359 | } |
360 | }; |
361 | |
362 | /// specialization of cstval_pred_ty for ConstantInt |
363 | template <typename Predicate> |
364 | using cst_pred_ty = cstval_pred_ty<Predicate, ConstantInt>; |
365 | |
366 | /// specialization of cstval_pred_ty for ConstantFP |
367 | template <typename Predicate> |
368 | using cstfp_pred_ty = cstval_pred_ty<Predicate, ConstantFP>; |
369 | |
370 | /// This helper class is used to match scalar and vector constants that |
371 | /// satisfy a specified predicate, and bind them to an APInt. |
372 | template <typename Predicate> struct api_pred_ty : public Predicate { |
373 | const APInt *&Res; |
374 | |
375 | api_pred_ty(const APInt *&R) : Res(R) {} |
376 | |
377 | template <typename ITy> bool match(ITy *V) { |
378 | if (const auto *CI = dyn_cast<ConstantInt>(V)) |
379 | if (this->isValue(CI->getValue())) { |
380 | Res = &CI->getValue(); |
381 | return true; |
382 | } |
383 | if (V->getType()->isVectorTy()) |
384 | if (const auto *C = dyn_cast<Constant>(V)) |
385 | if (auto *CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue())) |
386 | if (this->isValue(CI->getValue())) { |
387 | Res = &CI->getValue(); |
388 | return true; |
389 | } |
390 | |
391 | return false; |
392 | } |
393 | }; |
394 | |
395 | /// This helper class is used to match scalar and vector constants that |
396 | /// satisfy a specified predicate, and bind them to an APFloat. |
397 | /// Undefs are allowed in splat vector constants. |
398 | template <typename Predicate> struct apf_pred_ty : public Predicate { |
399 | const APFloat *&Res; |
400 | |
401 | apf_pred_ty(const APFloat *&R) : Res(R) {} |
402 | |
403 | template <typename ITy> bool match(ITy *V) { |
404 | if (const auto *CI = dyn_cast<ConstantFP>(V)) |
405 | if (this->isValue(CI->getValue())) { |
406 | Res = &CI->getValue(); |
407 | return true; |
408 | } |
409 | if (V->getType()->isVectorTy()) |
410 | if (const auto *C = dyn_cast<Constant>(V)) |
411 | if (auto *CI = dyn_cast_or_null<ConstantFP>( |
412 | C->getSplatValue(/* AllowUndef */ true))) |
413 | if (this->isValue(CI->getValue())) { |
414 | Res = &CI->getValue(); |
415 | return true; |
416 | } |
417 | |
418 | return false; |
419 | } |
420 | }; |
421 | |
422 | /////////////////////////////////////////////////////////////////////////////// |
423 | // |
424 | // Encapsulate constant value queries for use in templated predicate matchers. |
425 | // This allows checking if constants match using compound predicates and works |
426 | // with vector constants, possibly with relaxed constraints. For example, ignore |
427 | // undef values. |
428 | // |
429 | /////////////////////////////////////////////////////////////////////////////// |
430 | |
431 | struct is_any_apint { |
432 | bool isValue(const APInt &C) { return true; } |
433 | }; |
434 | /// Match an integer or vector with any integral constant. |
435 | /// For vectors, this includes constants with undefined elements. |
436 | inline cst_pred_ty<is_any_apint> m_AnyIntegralConstant() { |
437 | return cst_pred_ty<is_any_apint>(); |
438 | } |
439 | |
440 | struct is_all_ones { |
441 | bool isValue(const APInt &C) { return C.isAllOnesValue(); } |
442 | }; |
443 | /// Match an integer or vector with all bits set. |
444 | /// For vectors, this includes constants with undefined elements. |
445 | inline cst_pred_ty<is_all_ones> m_AllOnes() { |
446 | return cst_pred_ty<is_all_ones>(); |
447 | } |
448 | |
449 | struct is_maxsignedvalue { |
450 | bool isValue(const APInt &C) { return C.isMaxSignedValue(); } |
451 | }; |
452 | /// Match an integer or vector with values having all bits except for the high |
453 | /// bit set (0x7f...). |
454 | /// For vectors, this includes constants with undefined elements. |
455 | inline cst_pred_ty<is_maxsignedvalue> m_MaxSignedValue() { |
456 | return cst_pred_ty<is_maxsignedvalue>(); |
457 | } |
458 | inline api_pred_ty<is_maxsignedvalue> m_MaxSignedValue(const APInt *&V) { |
459 | return V; |
460 | } |
461 | |
462 | struct is_negative { |
463 | bool isValue(const APInt &C) { return C.isNegative(); } |
464 | }; |
465 | /// Match an integer or vector of negative values. |
466 | /// For vectors, this includes constants with undefined elements. |
467 | inline cst_pred_ty<is_negative> m_Negative() { |
468 | return cst_pred_ty<is_negative>(); |
469 | } |
470 | inline api_pred_ty<is_negative> m_Negative(const APInt *&V) { |
471 | return V; |
472 | } |
473 | |
474 | struct is_nonnegative { |
475 | bool isValue(const APInt &C) { return C.isNonNegative(); } |
476 | }; |
477 | /// Match an integer or vector of non-negative values. |
478 | /// For vectors, this includes constants with undefined elements. |
479 | inline cst_pred_ty<is_nonnegative> m_NonNegative() { |
480 | return cst_pred_ty<is_nonnegative>(); |
481 | } |
482 | inline api_pred_ty<is_nonnegative> m_NonNegative(const APInt *&V) { |
483 | return V; |
484 | } |
485 | |
486 | struct is_strictlypositive { |
487 | bool isValue(const APInt &C) { return C.isStrictlyPositive(); } |
488 | }; |
489 | /// Match an integer or vector of strictly positive values. |
490 | /// For vectors, this includes constants with undefined elements. |
491 | inline cst_pred_ty<is_strictlypositive> m_StrictlyPositive() { |
492 | return cst_pred_ty<is_strictlypositive>(); |
493 | } |
494 | inline api_pred_ty<is_strictlypositive> m_StrictlyPositive(const APInt *&V) { |
495 | return V; |
496 | } |
497 | |
498 | struct is_nonpositive { |
499 | bool isValue(const APInt &C) { return C.isNonPositive(); } |
500 | }; |
501 | /// Match an integer or vector of non-positive values. |
502 | /// For vectors, this includes constants with undefined elements. |
503 | inline cst_pred_ty<is_nonpositive> m_NonPositive() { |
504 | return cst_pred_ty<is_nonpositive>(); |
505 | } |
506 | inline api_pred_ty<is_nonpositive> m_NonPositive(const APInt *&V) { return V; } |
507 | |
508 | struct is_one { |
509 | bool isValue(const APInt &C) { return C.isOneValue(); } |
510 | }; |
511 | /// Match an integer 1 or a vector with all elements equal to 1. |
512 | /// For vectors, this includes constants with undefined elements. |
513 | inline cst_pred_ty<is_one> m_One() { |
514 | return cst_pred_ty<is_one>(); |
515 | } |
516 | |
517 | struct is_zero_int { |
518 | bool isValue(const APInt &C) { return C.isNullValue(); } |
519 | }; |
520 | /// Match an integer 0 or a vector with all elements equal to 0. |
521 | /// For vectors, this includes constants with undefined elements. |
522 | inline cst_pred_ty<is_zero_int> m_ZeroInt() { |
523 | return cst_pred_ty<is_zero_int>(); |
524 | } |
525 | |
526 | struct is_zero { |
527 | template <typename ITy> bool match(ITy *V) { |
528 | auto *C = dyn_cast<Constant>(V); |
529 | // FIXME: this should be able to do something for scalable vectors |
530 | return C && (C->isNullValue() || cst_pred_ty<is_zero_int>().match(C)); |
531 | } |
532 | }; |
533 | /// Match any null constant or a vector with all elements equal to 0. |
534 | /// For vectors, this includes constants with undefined elements. |
535 | inline is_zero m_Zero() { |
536 | return is_zero(); |
537 | } |
538 | |
539 | struct is_power2 { |
540 | bool isValue(const APInt &C) { return C.isPowerOf2(); } |
541 | }; |
542 | /// Match an integer or vector power-of-2. |
543 | /// For vectors, this includes constants with undefined elements. |
544 | inline cst_pred_ty<is_power2> m_Power2() { |
545 | return cst_pred_ty<is_power2>(); |
546 | } |
547 | inline api_pred_ty<is_power2> m_Power2(const APInt *&V) { |
548 | return V; |
549 | } |
550 | |
551 | struct is_negated_power2 { |
552 | bool isValue(const APInt &C) { return (-C).isPowerOf2(); } |
553 | }; |
554 | /// Match a integer or vector negated power-of-2. |
555 | /// For vectors, this includes constants with undefined elements. |
556 | inline cst_pred_ty<is_negated_power2> m_NegatedPower2() { |
557 | return cst_pred_ty<is_negated_power2>(); |
558 | } |
559 | inline api_pred_ty<is_negated_power2> m_NegatedPower2(const APInt *&V) { |
560 | return V; |
561 | } |
562 | |
563 | struct is_power2_or_zero { |
564 | bool isValue(const APInt &C) { return !C || C.isPowerOf2(); } |
565 | }; |
566 | /// Match an integer or vector of 0 or power-of-2 values. |
567 | /// For vectors, this includes constants with undefined elements. |
568 | inline cst_pred_ty<is_power2_or_zero> m_Power2OrZero() { |
569 | return cst_pred_ty<is_power2_or_zero>(); |
570 | } |
571 | inline api_pred_ty<is_power2_or_zero> m_Power2OrZero(const APInt *&V) { |
572 | return V; |
573 | } |
574 | |
575 | struct is_sign_mask { |
576 | bool isValue(const APInt &C) { return C.isSignMask(); } |
577 | }; |
578 | /// Match an integer or vector with only the sign bit(s) set. |
579 | /// For vectors, this includes constants with undefined elements. |
580 | inline cst_pred_ty<is_sign_mask> m_SignMask() { |
581 | return cst_pred_ty<is_sign_mask>(); |
582 | } |
583 | |
584 | struct is_lowbit_mask { |
585 | bool isValue(const APInt &C) { return C.isMask(); } |
586 | }; |
587 | /// Match an integer or vector with only the low bit(s) set. |
588 | /// For vectors, this includes constants with undefined elements. |
589 | inline cst_pred_ty<is_lowbit_mask> m_LowBitMask() { |
590 | return cst_pred_ty<is_lowbit_mask>(); |
591 | } |
592 | |
593 | struct icmp_pred_with_threshold { |
594 | ICmpInst::Predicate Pred; |
595 | const APInt *Thr; |
596 | bool isValue(const APInt &C) { |
597 | switch (Pred) { |
598 | case ICmpInst::Predicate::ICMP_EQ: |
599 | return C.eq(*Thr); |
600 | case ICmpInst::Predicate::ICMP_NE: |
601 | return C.ne(*Thr); |
602 | case ICmpInst::Predicate::ICMP_UGT: |
603 | return C.ugt(*Thr); |
604 | case ICmpInst::Predicate::ICMP_UGE: |
605 | return C.uge(*Thr); |
606 | case ICmpInst::Predicate::ICMP_ULT: |
607 | return C.ult(*Thr); |
608 | case ICmpInst::Predicate::ICMP_ULE: |
609 | return C.ule(*Thr); |
610 | case ICmpInst::Predicate::ICMP_SGT: |
611 | return C.sgt(*Thr); |
612 | case ICmpInst::Predicate::ICMP_SGE: |
613 | return C.sge(*Thr); |
614 | case ICmpInst::Predicate::ICMP_SLT: |
615 | return C.slt(*Thr); |
616 | case ICmpInst::Predicate::ICMP_SLE: |
617 | return C.sle(*Thr); |
618 | default: |
619 | llvm_unreachable("Unhandled ICmp predicate")::llvm::llvm_unreachable_internal("Unhandled ICmp predicate", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/IR/PatternMatch.h" , 619); |
620 | } |
621 | } |
622 | }; |
623 | /// Match an integer or vector with every element comparing 'pred' (eg/ne/...) |
624 | /// to Threshold. For vectors, this includes constants with undefined elements. |
625 | inline cst_pred_ty<icmp_pred_with_threshold> |
626 | m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold) { |
627 | cst_pred_ty<icmp_pred_with_threshold> P; |
628 | P.Pred = Predicate; |
629 | P.Thr = &Threshold; |
630 | return P; |
631 | } |
632 | |
633 | struct is_nan { |
634 | bool isValue(const APFloat &C) { return C.isNaN(); } |
635 | }; |
636 | /// Match an arbitrary NaN constant. This includes quiet and signalling nans. |
637 | /// For vectors, this includes constants with undefined elements. |
638 | inline cstfp_pred_ty<is_nan> m_NaN() { |
639 | return cstfp_pred_ty<is_nan>(); |
640 | } |
641 | |
642 | struct is_nonnan { |
643 | bool isValue(const APFloat &C) { return !C.isNaN(); } |
644 | }; |
645 | /// Match a non-NaN FP constant. |
646 | /// For vectors, this includes constants with undefined elements. |
647 | inline cstfp_pred_ty<is_nonnan> m_NonNaN() { |
648 | return cstfp_pred_ty<is_nonnan>(); |
649 | } |
650 | |
651 | struct is_inf { |
652 | bool isValue(const APFloat &C) { return C.isInfinity(); } |
653 | }; |
654 | /// Match a positive or negative infinity FP constant. |
655 | /// For vectors, this includes constants with undefined elements. |
656 | inline cstfp_pred_ty<is_inf> m_Inf() { |
657 | return cstfp_pred_ty<is_inf>(); |
658 | } |
659 | |
660 | struct is_noninf { |
661 | bool isValue(const APFloat &C) { return !C.isInfinity(); } |
662 | }; |
663 | /// Match a non-infinity FP constant, i.e. finite or NaN. |
664 | /// For vectors, this includes constants with undefined elements. |
665 | inline cstfp_pred_ty<is_noninf> m_NonInf() { |
666 | return cstfp_pred_ty<is_noninf>(); |
667 | } |
668 | |
669 | struct is_finite { |
670 | bool isValue(const APFloat &C) { return C.isFinite(); } |
671 | }; |
672 | /// Match a finite FP constant, i.e. not infinity or NaN. |
673 | /// For vectors, this includes constants with undefined elements. |
674 | inline cstfp_pred_ty<is_finite> m_Finite() { |
675 | return cstfp_pred_ty<is_finite>(); |
676 | } |
677 | inline apf_pred_ty<is_finite> m_Finite(const APFloat *&V) { return V; } |
678 | |
679 | struct is_finitenonzero { |
680 | bool isValue(const APFloat &C) { return C.isFiniteNonZero(); } |
681 | }; |
682 | /// Match a finite non-zero FP constant. |
683 | /// For vectors, this includes constants with undefined elements. |
684 | inline cstfp_pred_ty<is_finitenonzero> m_FiniteNonZero() { |
685 | return cstfp_pred_ty<is_finitenonzero>(); |
686 | } |
687 | inline apf_pred_ty<is_finitenonzero> m_FiniteNonZero(const APFloat *&V) { |
688 | return V; |
689 | } |
690 | |
691 | struct is_any_zero_fp { |
692 | bool isValue(const APFloat &C) { return C.isZero(); } |
693 | }; |
694 | /// Match a floating-point negative zero or positive zero. |
695 | /// For vectors, this includes constants with undefined elements. |
696 | inline cstfp_pred_ty<is_any_zero_fp> m_AnyZeroFP() { |
697 | return cstfp_pred_ty<is_any_zero_fp>(); |
698 | } |
699 | |
700 | struct is_pos_zero_fp { |
701 | bool isValue(const APFloat &C) { return C.isPosZero(); } |
702 | }; |
703 | /// Match a floating-point positive zero. |
704 | /// For vectors, this includes constants with undefined elements. |
705 | inline cstfp_pred_ty<is_pos_zero_fp> m_PosZeroFP() { |
706 | return cstfp_pred_ty<is_pos_zero_fp>(); |
707 | } |
708 | |
709 | struct is_neg_zero_fp { |
710 | bool isValue(const APFloat &C) { return C.isNegZero(); } |
711 | }; |
712 | /// Match a floating-point negative zero. |
713 | /// For vectors, this includes constants with undefined elements. |
714 | inline cstfp_pred_ty<is_neg_zero_fp> m_NegZeroFP() { |
715 | return cstfp_pred_ty<is_neg_zero_fp>(); |
716 | } |
717 | |
718 | struct is_non_zero_fp { |
719 | bool isValue(const APFloat &C) { return C.isNonZero(); } |
720 | }; |
721 | /// Match a floating-point non-zero. |
722 | /// For vectors, this includes constants with undefined elements. |
723 | inline cstfp_pred_ty<is_non_zero_fp> m_NonZeroFP() { |
724 | return cstfp_pred_ty<is_non_zero_fp>(); |
725 | } |
726 | |
727 | /////////////////////////////////////////////////////////////////////////////// |
728 | |
729 | template <typename Class> struct bind_ty { |
730 | Class *&VR; |
731 | |
732 | bind_ty(Class *&V) : VR(V) {} |
733 | |
734 | template <typename ITy> bool match(ITy *V) { |
735 | if (auto *CV = dyn_cast<Class>(V)) { |
736 | VR = CV; |
737 | return true; |
738 | } |
739 | return false; |
740 | } |
741 | }; |
742 | |
743 | /// Match a value, capturing it if we match. |
744 | inline bind_ty<Value> m_Value(Value *&V) { return V; } |
745 | inline bind_ty<const Value> m_Value(const Value *&V) { return V; } |
746 | |
747 | /// Match an instruction, capturing it if we match. |
748 | inline bind_ty<Instruction> m_Instruction(Instruction *&I) { return I; } |
749 | /// Match a unary operator, capturing it if we match. |
750 | inline bind_ty<UnaryOperator> m_UnOp(UnaryOperator *&I) { return I; } |
751 | /// Match a binary operator, capturing it if we match. |
752 | inline bind_ty<BinaryOperator> m_BinOp(BinaryOperator *&I) { return I; } |
753 | /// Match a with overflow intrinsic, capturing it if we match. |
754 | inline bind_ty<WithOverflowInst> m_WithOverflowInst(WithOverflowInst *&I) { return I; } |
755 | inline bind_ty<const WithOverflowInst> |
756 | m_WithOverflowInst(const WithOverflowInst *&I) { |
757 | return I; |
758 | } |
759 | |
760 | /// Match a Constant, capturing the value if we match. |
761 | inline bind_ty<Constant> m_Constant(Constant *&C) { return C; } |
762 | |
763 | /// Match a ConstantInt, capturing the value if we match. |
764 | inline bind_ty<ConstantInt> m_ConstantInt(ConstantInt *&CI) { return CI; } |
765 | |
766 | /// Match a ConstantFP, capturing the value if we match. |
767 | inline bind_ty<ConstantFP> m_ConstantFP(ConstantFP *&C) { return C; } |
768 | |
769 | /// Match a ConstantExpr, capturing the value if we match. |
770 | inline bind_ty<ConstantExpr> m_ConstantExpr(ConstantExpr *&C) { return C; } |
771 | |
772 | /// Match a basic block value, capturing it if we match. |
773 | inline bind_ty<BasicBlock> m_BasicBlock(BasicBlock *&V) { return V; } |
774 | inline bind_ty<const BasicBlock> m_BasicBlock(const BasicBlock *&V) { |
775 | return V; |
776 | } |
777 | |
778 | /// Match an arbitrary immediate Constant and ignore it. |
779 | inline match_combine_and<class_match<Constant>, |
780 | match_unless<class_match<ConstantExpr>>> |
781 | m_ImmConstant() { |
782 | return m_CombineAnd(m_Constant(), m_Unless(m_ConstantExpr())); |
783 | } |
784 | |
785 | /// Match an immediate Constant, capturing the value if we match. |
786 | inline match_combine_and<bind_ty<Constant>, |
787 | match_unless<class_match<ConstantExpr>>> |
788 | m_ImmConstant(Constant *&C) { |
789 | return m_CombineAnd(m_Constant(C), m_Unless(m_ConstantExpr())); |
790 | } |
791 | |
792 | /// Match a specified Value*. |
793 | struct specificval_ty { |
794 | const Value *Val; |
795 | |
796 | specificval_ty(const Value *V) : Val(V) {} |
797 | |
798 | template <typename ITy> bool match(ITy *V) { return V == Val; } |
799 | }; |
800 | |
801 | /// Match if we have a specific specified value. |
802 | inline specificval_ty m_Specific(const Value *V) { return V; } |
803 | |
804 | /// Stores a reference to the Value *, not the Value * itself, |
805 | /// thus can be used in commutative matchers. |
806 | template <typename Class> struct deferredval_ty { |
807 | Class *const &Val; |
808 | |
809 | deferredval_ty(Class *const &V) : Val(V) {} |
810 | |
811 | template <typename ITy> bool match(ITy *const V) { return V == Val; } |
812 | }; |
813 | |
814 | /// Like m_Specific(), but works if the specific value to match is determined |
815 | /// as part of the same match() expression. For example: |
816 | /// m_Add(m_Value(X), m_Specific(X)) is incorrect, because m_Specific() will |
817 | /// bind X before the pattern match starts. |
818 | /// m_Add(m_Value(X), m_Deferred(X)) is correct, and will check against |
819 | /// whichever value m_Value(X) populated. |
820 | inline deferredval_ty<Value> m_Deferred(Value *const &V) { return V; } |
821 | inline deferredval_ty<const Value> m_Deferred(const Value *const &V) { |
822 | return V; |
823 | } |
824 | |
825 | /// Match a specified floating point value or vector of all elements of |
826 | /// that value. |
827 | struct specific_fpval { |
828 | double Val; |
829 | |
830 | specific_fpval(double V) : Val(V) {} |
831 | |
832 | template <typename ITy> bool match(ITy *V) { |
833 | if (const auto *CFP = dyn_cast<ConstantFP>(V)) |
834 | return CFP->isExactlyValue(Val); |
835 | if (V->getType()->isVectorTy()) |
836 | if (const auto *C = dyn_cast<Constant>(V)) |
837 | if (auto *CFP = dyn_cast_or_null<ConstantFP>(C->getSplatValue())) |
838 | return CFP->isExactlyValue(Val); |
839 | return false; |
840 | } |
841 | }; |
842 | |
843 | /// Match a specific floating point value or vector with all elements |
844 | /// equal to the value. |
845 | inline specific_fpval m_SpecificFP(double V) { return specific_fpval(V); } |
846 | |
847 | /// Match a float 1.0 or vector with all elements equal to 1.0. |
848 | inline specific_fpval m_FPOne() { return m_SpecificFP(1.0); } |
849 | |
850 | struct bind_const_intval_ty { |
851 | uint64_t &VR; |
852 | |
853 | bind_const_intval_ty(uint64_t &V) : VR(V) {} |
854 | |
855 | template <typename ITy> bool match(ITy *V) { |
856 | if (const auto *CV = dyn_cast<ConstantInt>(V)) |
857 | if (CV->getValue().ule(UINT64_MAX(18446744073709551615UL))) { |
858 | VR = CV->getZExtValue(); |
859 | return true; |
860 | } |
861 | return false; |
862 | } |
863 | }; |
864 | |
865 | /// Match a specified integer value or vector of all elements of that |
866 | /// value. |
867 | template <bool AllowUndefs> |
868 | struct specific_intval { |
869 | APInt Val; |
870 | |
871 | specific_intval(APInt V) : Val(std::move(V)) {} |
872 | |
873 | template <typename ITy> bool match(ITy *V) { |
874 | const auto *CI = dyn_cast<ConstantInt>(V); |
875 | if (!CI && V->getType()->isVectorTy()) |
876 | if (const auto *C = dyn_cast<Constant>(V)) |
877 | CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndefs)); |
878 | |
879 | return CI && APInt::isSameValue(CI->getValue(), Val); |
880 | } |
881 | }; |
882 | |
883 | /// Match a specific integer value or vector with all elements equal to |
884 | /// the value. |
885 | inline specific_intval<false> m_SpecificInt(APInt V) { |
886 | return specific_intval<false>(std::move(V)); |
887 | } |
888 | |
889 | inline specific_intval<false> m_SpecificInt(uint64_t V) { |
890 | return m_SpecificInt(APInt(64, V)); |
891 | } |
892 | |
893 | inline specific_intval<true> m_SpecificIntAllowUndef(APInt V) { |
894 | return specific_intval<true>(std::move(V)); |
895 | } |
896 | |
897 | inline specific_intval<true> m_SpecificIntAllowUndef(uint64_t V) { |
898 | return m_SpecificIntAllowUndef(APInt(64, V)); |
899 | } |
900 | |
901 | /// Match a ConstantInt and bind to its value. This does not match |
902 | /// ConstantInts wider than 64-bits. |
903 | inline bind_const_intval_ty m_ConstantInt(uint64_t &V) { return V; } |
904 | |
905 | /// Match a specified basic block value. |
906 | struct specific_bbval { |
907 | BasicBlock *Val; |
908 | |
909 | specific_bbval(BasicBlock *Val) : Val(Val) {} |
910 | |
911 | template <typename ITy> bool match(ITy *V) { |
912 | const auto *BB = dyn_cast<BasicBlock>(V); |
913 | return BB && BB == Val; |
914 | } |
915 | }; |
916 | |
917 | /// Match a specific basic block value. |
918 | inline specific_bbval m_SpecificBB(BasicBlock *BB) { |
919 | return specific_bbval(BB); |
920 | } |
921 | |
922 | /// A commutative-friendly version of m_Specific(). |
923 | inline deferredval_ty<BasicBlock> m_Deferred(BasicBlock *const &BB) { |
924 | return BB; |
925 | } |
926 | inline deferredval_ty<const BasicBlock> |
927 | m_Deferred(const BasicBlock *const &BB) { |
928 | return BB; |
929 | } |
930 | |
931 | //===----------------------------------------------------------------------===// |
932 | // Matcher for any binary operator. |
933 | // |
934 | template <typename LHS_t, typename RHS_t, bool Commutable = false> |
935 | struct AnyBinaryOp_match { |
936 | LHS_t L; |
937 | RHS_t R; |
938 | |
939 | // The evaluation order is always stable, regardless of Commutability. |
940 | // The LHS is always matched first. |
941 | AnyBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} |
942 | |
943 | template <typename OpTy> bool match(OpTy *V) { |
944 | if (auto *I = dyn_cast<BinaryOperator>(V)) |
945 | return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) || |
946 | (Commutable && L.match(I->getOperand(1)) && |
947 | R.match(I->getOperand(0))); |
948 | return false; |
949 | } |
950 | }; |
951 | |
952 | template <typename LHS, typename RHS> |
953 | inline AnyBinaryOp_match<LHS, RHS> m_BinOp(const LHS &L, const RHS &R) { |
954 | return AnyBinaryOp_match<LHS, RHS>(L, R); |
955 | } |
956 | |
957 | //===----------------------------------------------------------------------===// |
958 | // Matcher for any unary operator. |
959 | // TODO fuse unary, binary matcher into n-ary matcher |
960 | // |
961 | template <typename OP_t> struct AnyUnaryOp_match { |
962 | OP_t X; |
963 | |
964 | AnyUnaryOp_match(const OP_t &X) : X(X) {} |
965 | |
966 | template <typename OpTy> bool match(OpTy *V) { |
967 | if (auto *I = dyn_cast<UnaryOperator>(V)) |
968 | return X.match(I->getOperand(0)); |
969 | return false; |
970 | } |
971 | }; |
972 | |
973 | template <typename OP_t> inline AnyUnaryOp_match<OP_t> m_UnOp(const OP_t &X) { |
974 | return AnyUnaryOp_match<OP_t>(X); |
975 | } |
976 | |
977 | //===----------------------------------------------------------------------===// |
978 | // Matchers for specific binary operators. |
979 | // |
980 | |
981 | template <typename LHS_t, typename RHS_t, unsigned Opcode, |
982 | bool Commutable = false> |
983 | struct BinaryOp_match { |
984 | LHS_t L; |
985 | RHS_t R; |
986 | |
987 | // The evaluation order is always stable, regardless of Commutability. |
988 | // The LHS is always matched first. |
989 | BinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} |
990 | |
991 | template <typename OpTy> bool match(OpTy *V) { |
992 | if (V->getValueID() == Value::InstructionVal + Opcode) { |
993 | auto *I = cast<BinaryOperator>(V); |
994 | return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) || |
995 | (Commutable && L.match(I->getOperand(1)) && |
996 | R.match(I->getOperand(0))); |
997 | } |
998 | if (auto *CE = dyn_cast<ConstantExpr>(V)) |
999 | return CE->getOpcode() == Opcode && |
1000 | ((L.match(CE->getOperand(0)) && R.match(CE->getOperand(1))) || |
1001 | (Commutable && L.match(CE->getOperand(1)) && |
1002 | R.match(CE->getOperand(0)))); |
1003 | return false; |
1004 | } |
1005 | }; |
1006 | |
1007 | template <typename LHS, typename RHS> |
1008 | inline BinaryOp_match<LHS, RHS, Instruction::Add> m_Add(const LHS &L, |
1009 | const RHS &R) { |
1010 | return BinaryOp_match<LHS, RHS, Instruction::Add>(L, R); |
1011 | } |
1012 | |
1013 | template <typename LHS, typename RHS> |
1014 | inline BinaryOp_match<LHS, RHS, Instruction::FAdd> m_FAdd(const LHS &L, |
1015 | const RHS &R) { |
1016 | return BinaryOp_match<LHS, RHS, Instruction::FAdd>(L, R); |
1017 | } |
1018 | |
1019 | template <typename LHS, typename RHS> |
1020 | inline BinaryOp_match<LHS, RHS, Instruction::Sub> m_Sub(const LHS &L, |
1021 | const RHS &R) { |
1022 | return BinaryOp_match<LHS, RHS, Instruction::Sub>(L, R); |
1023 | } |
1024 | |
1025 | template <typename LHS, typename RHS> |
1026 | inline BinaryOp_match<LHS, RHS, Instruction::FSub> m_FSub(const LHS &L, |
1027 | const RHS &R) { |
1028 | return BinaryOp_match<LHS, RHS, Instruction::FSub>(L, R); |
1029 | } |
1030 | |
1031 | template <typename Op_t> struct FNeg_match { |
1032 | Op_t X; |
1033 | |
1034 | FNeg_match(const Op_t &Op) : X(Op) {} |
1035 | template <typename OpTy> bool match(OpTy *V) { |
1036 | auto *FPMO = dyn_cast<FPMathOperator>(V); |
1037 | if (!FPMO) return false; |
1038 | |
1039 | if (FPMO->getOpcode() == Instruction::FNeg) |
1040 | return X.match(FPMO->getOperand(0)); |
1041 | |
1042 | if (FPMO->getOpcode() == Instruction::FSub) { |
1043 | if (FPMO->hasNoSignedZeros()) { |
1044 | // With 'nsz', any zero goes. |
1045 | if (!cstfp_pred_ty<is_any_zero_fp>().match(FPMO->getOperand(0))) |
1046 | return false; |
1047 | } else { |
1048 | // Without 'nsz', we need fsub -0.0, X exactly. |
1049 | if (!cstfp_pred_ty<is_neg_zero_fp>().match(FPMO->getOperand(0))) |
1050 | return false; |
1051 | } |
1052 | |
1053 | return X.match(FPMO->getOperand(1)); |
1054 | } |
1055 | |
1056 | return false; |
1057 | } |
1058 | }; |
1059 | |
1060 | /// Match 'fneg X' as 'fsub -0.0, X'. |
1061 | template <typename OpTy> |
1062 | inline FNeg_match<OpTy> |
1063 | m_FNeg(const OpTy &X) { |
1064 | return FNeg_match<OpTy>(X); |
1065 | } |
1066 | |
1067 | /// Match 'fneg X' as 'fsub +-0.0, X'. |
1068 | template <typename RHS> |
1069 | inline BinaryOp_match<cstfp_pred_ty<is_any_zero_fp>, RHS, Instruction::FSub> |
1070 | m_FNegNSZ(const RHS &X) { |
1071 | return m_FSub(m_AnyZeroFP(), X); |
1072 | } |
1073 | |
1074 | template <typename LHS, typename RHS> |
1075 | inline BinaryOp_match<LHS, RHS, Instruction::Mul> m_Mul(const LHS &L, |
1076 | const RHS &R) { |
1077 | return BinaryOp_match<LHS, RHS, Instruction::Mul>(L, R); |
1078 | } |
1079 | |
1080 | template <typename LHS, typename RHS> |
1081 | inline BinaryOp_match<LHS, RHS, Instruction::FMul> m_FMul(const LHS &L, |
1082 | const RHS &R) { |
1083 | return BinaryOp_match<LHS, RHS, Instruction::FMul>(L, R); |
1084 | } |
1085 | |
1086 | template <typename LHS, typename RHS> |
1087 | inline BinaryOp_match<LHS, RHS, Instruction::UDiv> m_UDiv(const LHS &L, |
1088 | const RHS &R) { |
1089 | return BinaryOp_match<LHS, RHS, Instruction::UDiv>(L, R); |
1090 | } |
1091 | |
1092 | template <typename LHS, typename RHS> |
1093 | inline BinaryOp_match<LHS, RHS, Instruction::SDiv> m_SDiv(const LHS &L, |
1094 | const RHS &R) { |
1095 | return BinaryOp_match<LHS, RHS, Instruction::SDiv>(L, R); |
1096 | } |
1097 | |
1098 | template <typename LHS, typename RHS> |
1099 | inline BinaryOp_match<LHS, RHS, Instruction::FDiv> m_FDiv(const LHS &L, |
1100 | const RHS &R) { |
1101 | return BinaryOp_match<LHS, RHS, Instruction::FDiv>(L, R); |
1102 | } |
1103 | |
1104 | template <typename LHS, typename RHS> |
1105 | inline BinaryOp_match<LHS, RHS, Instruction::URem> m_URem(const LHS &L, |
1106 | const RHS &R) { |
1107 | return BinaryOp_match<LHS, RHS, Instruction::URem>(L, R); |
1108 | } |
1109 | |
1110 | template <typename LHS, typename RHS> |
1111 | inline BinaryOp_match<LHS, RHS, Instruction::SRem> m_SRem(const LHS &L, |
1112 | const RHS &R) { |
1113 | return BinaryOp_match<LHS, RHS, Instruction::SRem>(L, R); |
1114 | } |
1115 | |
1116 | template <typename LHS, typename RHS> |
1117 | inline BinaryOp_match<LHS, RHS, Instruction::FRem> m_FRem(const LHS &L, |
1118 | const RHS &R) { |
1119 | return BinaryOp_match<LHS, RHS, Instruction::FRem>(L, R); |
1120 | } |
1121 | |
1122 | template <typename LHS, typename RHS> |
1123 | inline BinaryOp_match<LHS, RHS, Instruction::And> m_And(const LHS &L, |
1124 | const RHS &R) { |
1125 | return BinaryOp_match<LHS, RHS, Instruction::And>(L, R); |
1126 | } |
1127 | |
1128 | template <typename LHS, typename RHS> |
1129 | inline BinaryOp_match<LHS, RHS, Instruction::Or> m_Or(const LHS &L, |
1130 | const RHS &R) { |
1131 | return BinaryOp_match<LHS, RHS, Instruction::Or>(L, R); |
1132 | } |
1133 | |
1134 | template <typename LHS, typename RHS> |
1135 | inline BinaryOp_match<LHS, RHS, Instruction::Xor> m_Xor(const LHS &L, |
1136 | const RHS &R) { |
1137 | return BinaryOp_match<LHS, RHS, Instruction::Xor>(L, R); |
1138 | } |
1139 | |
1140 | template <typename LHS, typename RHS> |
1141 | inline BinaryOp_match<LHS, RHS, Instruction::Shl> m_Shl(const LHS &L, |
1142 | const RHS &R) { |
1143 | return BinaryOp_match<LHS, RHS, Instruction::Shl>(L, R); |
1144 | } |
1145 | |
1146 | template <typename LHS, typename RHS> |
1147 | inline BinaryOp_match<LHS, RHS, Instruction::LShr> m_LShr(const LHS &L, |
1148 | const RHS &R) { |
1149 | return BinaryOp_match<LHS, RHS, Instruction::LShr>(L, R); |
1150 | } |
1151 | |
1152 | template <typename LHS, typename RHS> |
1153 | inline BinaryOp_match<LHS, RHS, Instruction::AShr> m_AShr(const LHS &L, |
1154 | const RHS &R) { |
1155 | return BinaryOp_match<LHS, RHS, Instruction::AShr>(L, R); |
1156 | } |
1157 | |
1158 | template <typename LHS_t, typename RHS_t, unsigned Opcode, |
1159 | unsigned WrapFlags = 0> |
1160 | struct OverflowingBinaryOp_match { |
1161 | LHS_t L; |
1162 | RHS_t R; |
1163 | |
1164 | OverflowingBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) |
1165 | : L(LHS), R(RHS) {} |
1166 | |
1167 | template <typename OpTy> bool match(OpTy *V) { |
1168 | if (auto *Op = dyn_cast<OverflowingBinaryOperator>(V)) { |
1169 | if (Op->getOpcode() != Opcode) |
1170 | return false; |
1171 | if ((WrapFlags & OverflowingBinaryOperator::NoUnsignedWrap) && |
1172 | !Op->hasNoUnsignedWrap()) |
1173 | return false; |
1174 | if ((WrapFlags & OverflowingBinaryOperator::NoSignedWrap) && |
1175 | !Op->hasNoSignedWrap()) |
1176 | return false; |
1177 | return L.match(Op->getOperand(0)) && R.match(Op->getOperand(1)); |
1178 | } |
1179 | return false; |
1180 | } |
1181 | }; |
1182 | |
1183 | template <typename LHS, typename RHS> |
1184 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, |
1185 | OverflowingBinaryOperator::NoSignedWrap> |
1186 | m_NSWAdd(const LHS &L, const RHS &R) { |
1187 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, |
1188 | OverflowingBinaryOperator::NoSignedWrap>( |
1189 | L, R); |
1190 | } |
1191 | template <typename LHS, typename RHS> |
1192 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, |
1193 | OverflowingBinaryOperator::NoSignedWrap> |
1194 | m_NSWSub(const LHS &L, const RHS &R) { |
1195 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, |
1196 | OverflowingBinaryOperator::NoSignedWrap>( |
1197 | L, R); |
1198 | } |
1199 | template <typename LHS, typename RHS> |
1200 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, |
1201 | OverflowingBinaryOperator::NoSignedWrap> |
1202 | m_NSWMul(const LHS &L, const RHS &R) { |
1203 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, |
1204 | OverflowingBinaryOperator::NoSignedWrap>( |
1205 | L, R); |
1206 | } |
1207 | template <typename LHS, typename RHS> |
1208 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, |
1209 | OverflowingBinaryOperator::NoSignedWrap> |
1210 | m_NSWShl(const LHS &L, const RHS &R) { |
1211 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, |
1212 | OverflowingBinaryOperator::NoSignedWrap>( |
1213 | L, R); |
1214 | } |
1215 | |
1216 | template <typename LHS, typename RHS> |
1217 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, |
1218 | OverflowingBinaryOperator::NoUnsignedWrap> |
1219 | m_NUWAdd(const LHS &L, const RHS &R) { |
1220 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, |
1221 | OverflowingBinaryOperator::NoUnsignedWrap>( |
1222 | L, R); |
1223 | } |
1224 | template <typename LHS, typename RHS> |
1225 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, |
1226 | OverflowingBinaryOperator::NoUnsignedWrap> |
1227 | m_NUWSub(const LHS &L, const RHS &R) { |
1228 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, |
1229 | OverflowingBinaryOperator::NoUnsignedWrap>( |
1230 | L, R); |
1231 | } |
1232 | template <typename LHS, typename RHS> |
1233 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, |
1234 | OverflowingBinaryOperator::NoUnsignedWrap> |
1235 | m_NUWMul(const LHS &L, const RHS &R) { |
1236 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, |
1237 | OverflowingBinaryOperator::NoUnsignedWrap>( |
1238 | L, R); |
1239 | } |
1240 | template <typename LHS, typename RHS> |
1241 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, |
1242 | OverflowingBinaryOperator::NoUnsignedWrap> |
1243 | m_NUWShl(const LHS &L, const RHS &R) { |
1244 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, |
1245 | OverflowingBinaryOperator::NoUnsignedWrap>( |
1246 | L, R); |
1247 | } |
1248 | |
1249 | //===----------------------------------------------------------------------===// |
1250 | // Class that matches a group of binary opcodes. |
1251 | // |
1252 | template <typename LHS_t, typename RHS_t, typename Predicate> |
1253 | struct BinOpPred_match : Predicate { |
1254 | LHS_t L; |
1255 | RHS_t R; |
1256 | |
1257 | BinOpPred_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} |
1258 | |
1259 | template <typename OpTy> bool match(OpTy *V) { |
1260 | if (auto *I = dyn_cast<Instruction>(V)) |
1261 | return this->isOpType(I->getOpcode()) && L.match(I->getOperand(0)) && |
1262 | R.match(I->getOperand(1)); |
1263 | if (auto *CE = dyn_cast<ConstantExpr>(V)) |
1264 | return this->isOpType(CE->getOpcode()) && L.match(CE->getOperand(0)) && |
1265 | R.match(CE->getOperand(1)); |
1266 | return false; |
1267 | } |
1268 | }; |
1269 | |
1270 | struct is_shift_op { |
1271 | bool isOpType(unsigned Opcode) { return Instruction::isShift(Opcode); } |
1272 | }; |
1273 | |
1274 | struct is_right_shift_op { |
1275 | bool isOpType(unsigned Opcode) { |
1276 | return Opcode == Instruction::LShr || Opcode == Instruction::AShr; |
1277 | } |
1278 | }; |
1279 | |
1280 | struct is_logical_shift_op { |
1281 | bool isOpType(unsigned Opcode) { |
1282 | return Opcode == Instruction::LShr || Opcode == Instruction::Shl; |
1283 | } |
1284 | }; |
1285 | |
1286 | struct is_bitwiselogic_op { |
1287 | bool isOpType(unsigned Opcode) { |
1288 | return Instruction::isBitwiseLogicOp(Opcode); |
1289 | } |
1290 | }; |
1291 | |
1292 | struct is_idiv_op { |
1293 | bool isOpType(unsigned Opcode) { |
1294 | return Opcode == Instruction::SDiv || Opcode == Instruction::UDiv; |
1295 | } |
1296 | }; |
1297 | |
1298 | struct is_irem_op { |
1299 | bool isOpType(unsigned Opcode) { |
1300 | return Opcode == Instruction::SRem || Opcode == Instruction::URem; |
1301 | } |
1302 | }; |
1303 | |
1304 | /// Matches shift operations. |
1305 | template <typename LHS, typename RHS> |
1306 | inline BinOpPred_match<LHS, RHS, is_shift_op> m_Shift(const LHS &L, |
1307 | const RHS &R) { |
1308 | return BinOpPred_match<LHS, RHS, is_shift_op>(L, R); |
1309 | } |
1310 | |
1311 | /// Matches logical shift operations. |
1312 | template <typename LHS, typename RHS> |
1313 | inline BinOpPred_match<LHS, RHS, is_right_shift_op> m_Shr(const LHS &L, |
1314 | const RHS &R) { |
1315 | return BinOpPred_match<LHS, RHS, is_right_shift_op>(L, R); |
1316 | } |
1317 | |
1318 | /// Matches logical shift operations. |
1319 | template <typename LHS, typename RHS> |
1320 | inline BinOpPred_match<LHS, RHS, is_logical_shift_op> |
1321 | m_LogicalShift(const LHS &L, const RHS &R) { |
1322 | return BinOpPred_match<LHS, RHS, is_logical_shift_op>(L, R); |
1323 | } |
1324 | |
1325 | /// Matches bitwise logic operations. |
1326 | template <typename LHS, typename RHS> |
1327 | inline BinOpPred_match<LHS, RHS, is_bitwiselogic_op> |
1328 | m_BitwiseLogic(const LHS &L, const RHS &R) { |
1329 | return BinOpPred_match<LHS, RHS, is_bitwiselogic_op>(L, R); |
1330 | } |
1331 | |
1332 | /// Matches integer division operations. |
1333 | template <typename LHS, typename RHS> |
1334 | inline BinOpPred_match<LHS, RHS, is_idiv_op> m_IDiv(const LHS &L, |
1335 | const RHS &R) { |
1336 | return BinOpPred_match<LHS, RHS, is_idiv_op>(L, R); |
1337 | } |
1338 | |
1339 | /// Matches integer remainder operations. |
1340 | template <typename LHS, typename RHS> |
1341 | inline BinOpPred_match<LHS, RHS, is_irem_op> m_IRem(const LHS &L, |
1342 | const RHS &R) { |
1343 | return BinOpPred_match<LHS, RHS, is_irem_op>(L, R); |
1344 | } |
1345 | |
1346 | //===----------------------------------------------------------------------===// |
1347 | // Class that matches exact binary ops. |
1348 | // |
1349 | template <typename SubPattern_t> struct Exact_match { |
1350 | SubPattern_t SubPattern; |
1351 | |
1352 | Exact_match(const SubPattern_t &SP) : SubPattern(SP) {} |
1353 | |
1354 | template <typename OpTy> bool match(OpTy *V) { |
1355 | if (auto *PEO = dyn_cast<PossiblyExactOperator>(V)) |
1356 | return PEO->isExact() && SubPattern.match(V); |
1357 | return false; |
1358 | } |
1359 | }; |
1360 | |
1361 | template <typename T> inline Exact_match<T> m_Exact(const T &SubPattern) { |
1362 | return SubPattern; |
1363 | } |
1364 | |
1365 | //===----------------------------------------------------------------------===// |
1366 | // Matchers for CmpInst classes |
1367 | // |
1368 | |
1369 | template <typename LHS_t, typename RHS_t, typename Class, typename PredicateTy, |
1370 | bool Commutable = false> |
1371 | struct CmpClass_match { |
1372 | PredicateTy &Predicate; |
1373 | LHS_t L; |
1374 | RHS_t R; |
1375 | |
1376 | // The evaluation order is always stable, regardless of Commutability. |
1377 | // The LHS is always matched first. |
1378 | CmpClass_match(PredicateTy &Pred, const LHS_t &LHS, const RHS_t &RHS) |
1379 | : Predicate(Pred), L(LHS), R(RHS) {} |
1380 | |
1381 | template <typename OpTy> bool match(OpTy *V) { |
1382 | if (auto *I = dyn_cast<Class>(V)) { |
1383 | if (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) { |
1384 | Predicate = I->getPredicate(); |
1385 | return true; |
1386 | } else if (Commutable && L.match(I->getOperand(1)) && |
1387 | R.match(I->getOperand(0))) { |
1388 | Predicate = I->getSwappedPredicate(); |
1389 | return true; |
1390 | } |
1391 | } |
1392 | return false; |
1393 | } |
1394 | }; |
1395 | |
1396 | template <typename LHS, typename RHS> |
1397 | inline CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate> |
1398 | m_Cmp(CmpInst::Predicate &Pred, const LHS &L, const RHS &R) { |
1399 | return CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate>(Pred, L, R); |
1400 | } |
1401 | |
1402 | template <typename LHS, typename RHS> |
1403 | inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate> |
1404 | m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { |
1405 | return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate>(Pred, L, R); |
1406 | } |
1407 | |
1408 | template <typename LHS, typename RHS> |
1409 | inline CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate> |
1410 | m_FCmp(FCmpInst::Predicate &Pred, const LHS &L, const RHS &R) { |
1411 | return CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate>(Pred, L, R); |
1412 | } |
1413 | |
1414 | //===----------------------------------------------------------------------===// |
1415 | // Matchers for instructions with a given opcode and number of operands. |
1416 | // |
1417 | |
1418 | /// Matches instructions with Opcode and three operands. |
1419 | template <typename T0, unsigned Opcode> struct OneOps_match { |
1420 | T0 Op1; |
1421 | |
1422 | OneOps_match(const T0 &Op1) : Op1(Op1) {} |
1423 | |
1424 | template <typename OpTy> bool match(OpTy *V) { |
1425 | if (V->getValueID() == Value::InstructionVal + Opcode) { |
1426 | auto *I = cast<Instruction>(V); |
1427 | return Op1.match(I->getOperand(0)); |
1428 | } |
1429 | return false; |
1430 | } |
1431 | }; |
1432 | |
1433 | /// Matches instructions with Opcode and three operands. |
1434 | template <typename T0, typename T1, unsigned Opcode> struct TwoOps_match { |
1435 | T0 Op1; |
1436 | T1 Op2; |
1437 | |
1438 | TwoOps_match(const T0 &Op1, const T1 &Op2) : Op1(Op1), Op2(Op2) {} |
1439 | |
1440 | template <typename OpTy> bool match(OpTy *V) { |
1441 | if (V->getValueID() == Value::InstructionVal + Opcode) { |
1442 | auto *I = cast<Instruction>(V); |
1443 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)); |
1444 | } |
1445 | return false; |
1446 | } |
1447 | }; |
1448 | |
1449 | /// Matches instructions with Opcode and three operands. |
1450 | template <typename T0, typename T1, typename T2, unsigned Opcode> |
1451 | struct ThreeOps_match { |
1452 | T0 Op1; |
1453 | T1 Op2; |
1454 | T2 Op3; |
1455 | |
1456 | ThreeOps_match(const T0 &Op1, const T1 &Op2, const T2 &Op3) |
1457 | : Op1(Op1), Op2(Op2), Op3(Op3) {} |
1458 | |
1459 | template <typename OpTy> bool match(OpTy *V) { |
1460 | if (V->getValueID() == Value::InstructionVal + Opcode) { |
1461 | auto *I = cast<Instruction>(V); |
1462 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) && |
1463 | Op3.match(I->getOperand(2)); |
1464 | } |
1465 | return false; |
1466 | } |
1467 | }; |
1468 | |
1469 | /// Matches SelectInst. |
1470 | template <typename Cond, typename LHS, typename RHS> |
1471 | inline ThreeOps_match<Cond, LHS, RHS, Instruction::Select> |
1472 | m_Select(const Cond &C, const LHS &L, const RHS &R) { |
1473 | return ThreeOps_match<Cond, LHS, RHS, Instruction::Select>(C, L, R); |
1474 | } |
1475 | |
1476 | /// This matches a select of two constants, e.g.: |
1477 | /// m_SelectCst<-1, 0>(m_Value(V)) |
1478 | template <int64_t L, int64_t R, typename Cond> |
1479 | inline ThreeOps_match<Cond, constantint_match<L>, constantint_match<R>, |
1480 | Instruction::Select> |
1481 | m_SelectCst(const Cond &C) { |
1482 | return m_Select(C, m_ConstantInt<L>(), m_ConstantInt<R>()); |
1483 | } |
1484 | |
1485 | /// Matches FreezeInst. |
1486 | template <typename OpTy> |
1487 | inline OneOps_match<OpTy, Instruction::Freeze> m_Freeze(const OpTy &Op) { |
1488 | return OneOps_match<OpTy, Instruction::Freeze>(Op); |
1489 | } |
1490 | |
1491 | /// Matches InsertElementInst. |
1492 | template <typename Val_t, typename Elt_t, typename Idx_t> |
1493 | inline ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement> |
1494 | m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx) { |
1495 | return ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement>( |
1496 | Val, Elt, Idx); |
1497 | } |
1498 | |
1499 | /// Matches ExtractElementInst. |
1500 | template <typename Val_t, typename Idx_t> |
1501 | inline TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement> |
1502 | m_ExtractElt(const Val_t &Val, const Idx_t &Idx) { |
1503 | return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx); |
1504 | } |
1505 | |
1506 | /// Matches shuffle. |
1507 | template <typename T0, typename T1, typename T2> struct Shuffle_match { |
1508 | T0 Op1; |
1509 | T1 Op2; |
1510 | T2 Mask; |
1511 | |
1512 | Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask) |
1513 | : Op1(Op1), Op2(Op2), Mask(Mask) {} |
1514 | |
1515 | template <typename OpTy> bool match(OpTy *V) { |
1516 | if (auto *I = dyn_cast<ShuffleVectorInst>(V)) { |
1517 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) && |
1518 | Mask.match(I->getShuffleMask()); |
1519 | } |
1520 | return false; |
1521 | } |
1522 | }; |
1523 | |
1524 | struct m_Mask { |
1525 | ArrayRef<int> &MaskRef; |
1526 | m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {} |
1527 | bool match(ArrayRef<int> Mask) { |
1528 | MaskRef = Mask; |
1529 | return true; |
1530 | } |
1531 | }; |
1532 | |
1533 | struct m_ZeroMask { |
1534 | bool match(ArrayRef<int> Mask) { |
1535 | return all_of(Mask, [](int Elem) { return Elem == 0 || Elem == -1; }); |
1536 | } |
1537 | }; |
1538 | |
1539 | struct m_SpecificMask { |
1540 | ArrayRef<int> &MaskRef; |
1541 | m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {} |
1542 | bool match(ArrayRef<int> Mask) { return MaskRef == Mask; } |
1543 | }; |
1544 | |
1545 | struct m_SplatOrUndefMask { |
1546 | int &SplatIndex; |
1547 | m_SplatOrUndefMask(int &SplatIndex) : SplatIndex(SplatIndex) {} |
1548 | bool match(ArrayRef<int> Mask) { |
1549 | auto First = find_if(Mask, [](int Elem) { return Elem != -1; }); |
1550 | if (First == Mask.end()) |
1551 | return false; |
1552 | SplatIndex = *First; |
1553 | return all_of(Mask, |
1554 | [First](int Elem) { return Elem == *First || Elem == -1; }); |
1555 | } |
1556 | }; |
1557 | |
1558 | /// Matches ShuffleVectorInst independently of mask value. |
1559 | template <typename V1_t, typename V2_t> |
1560 | inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector> |
1561 | m_Shuffle(const V1_t &v1, const V2_t &v2) { |
1562 | return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2); |
1563 | } |
1564 | |
1565 | template <typename V1_t, typename V2_t, typename Mask_t> |
1566 | inline Shuffle_match<V1_t, V2_t, Mask_t> |
1567 | m_Shuffle(const V1_t &v1, const V2_t &v2, const Mask_t &mask) { |
1568 | return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask); |
1569 | } |
1570 | |
1571 | /// Matches LoadInst. |
1572 | template <typename OpTy> |
1573 | inline OneOps_match<OpTy, Instruction::Load> m_Load(const OpTy &Op) { |
1574 | return OneOps_match<OpTy, Instruction::Load>(Op); |
1575 | } |
1576 | |
1577 | /// Matches StoreInst. |
1578 | template <typename ValueOpTy, typename PointerOpTy> |
1579 | inline TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store> |
1580 | m_Store(const ValueOpTy &ValueOp, const PointerOpTy &PointerOp) { |
1581 | return TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store>(ValueOp, |
1582 | PointerOp); |
1583 | } |
1584 | |
1585 | //===----------------------------------------------------------------------===// |
1586 | // Matchers for CastInst classes |
1587 | // |
1588 | |
1589 | template <typename Op_t, unsigned Opcode> struct CastClass_match { |
1590 | Op_t Op; |
1591 | |
1592 | CastClass_match(const Op_t &OpMatch) : Op(OpMatch) {} |
1593 | |
1594 | template <typename OpTy> bool match(OpTy *V) { |
1595 | if (auto *O = dyn_cast<Operator>(V)) |
1596 | return O->getOpcode() == Opcode && Op.match(O->getOperand(0)); |
1597 | return false; |
1598 | } |
1599 | }; |
1600 | |
1601 | /// Matches BitCast. |
1602 | template <typename OpTy> |
1603 | inline CastClass_match<OpTy, Instruction::BitCast> m_BitCast(const OpTy &Op) { |
1604 | return CastClass_match<OpTy, Instruction::BitCast>(Op); |
1605 | } |
1606 | |
1607 | /// Matches PtrToInt. |
1608 | template <typename OpTy> |
1609 | inline CastClass_match<OpTy, Instruction::PtrToInt> m_PtrToInt(const OpTy &Op) { |
1610 | return CastClass_match<OpTy, Instruction::PtrToInt>(Op); |
1611 | } |
1612 | |
1613 | /// Matches IntToPtr. |
1614 | template <typename OpTy> |
1615 | inline CastClass_match<OpTy, Instruction::IntToPtr> m_IntToPtr(const OpTy &Op) { |
1616 | return CastClass_match<OpTy, Instruction::IntToPtr>(Op); |
1617 | } |
1618 | |
1619 | /// Matches Trunc. |
1620 | template <typename OpTy> |
1621 | inline CastClass_match<OpTy, Instruction::Trunc> m_Trunc(const OpTy &Op) { |
1622 | return CastClass_match<OpTy, Instruction::Trunc>(Op); |
1623 | } |
1624 | |
1625 | template <typename OpTy> |
1626 | inline match_combine_or<CastClass_match<OpTy, Instruction::Trunc>, OpTy> |
1627 | m_TruncOrSelf(const OpTy &Op) { |
1628 | return m_CombineOr(m_Trunc(Op), Op); |
1629 | } |
1630 | |
1631 | /// Matches SExt. |
1632 | template <typename OpTy> |
1633 | inline CastClass_match<OpTy, Instruction::SExt> m_SExt(const OpTy &Op) { |
1634 | return CastClass_match<OpTy, Instruction::SExt>(Op); |
1635 | } |
1636 | |
1637 | /// Matches ZExt. |
1638 | template <typename OpTy> |
1639 | inline CastClass_match<OpTy, Instruction::ZExt> m_ZExt(const OpTy &Op) { |
1640 | return CastClass_match<OpTy, Instruction::ZExt>(Op); |
1641 | } |
1642 | |
1643 | template <typename OpTy> |
1644 | inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, OpTy> |
1645 | m_ZExtOrSelf(const OpTy &Op) { |
1646 | return m_CombineOr(m_ZExt(Op), Op); |
1647 | } |
1648 | |
1649 | template <typename OpTy> |
1650 | inline match_combine_or<CastClass_match<OpTy, Instruction::SExt>, OpTy> |
1651 | m_SExtOrSelf(const OpTy &Op) { |
1652 | return m_CombineOr(m_SExt(Op), Op); |
1653 | } |
1654 | |
1655 | template <typename OpTy> |
1656 | inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, |
1657 | CastClass_match<OpTy, Instruction::SExt>> |
1658 | m_ZExtOrSExt(const OpTy &Op) { |
1659 | return m_CombineOr(m_ZExt(Op), m_SExt(Op)); |
1660 | } |
1661 | |
1662 | template <typename OpTy> |
1663 | inline match_combine_or< |
1664 | match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, |
1665 | CastClass_match<OpTy, Instruction::SExt>>, |
1666 | OpTy> |
1667 | m_ZExtOrSExtOrSelf(const OpTy &Op) { |
1668 | return m_CombineOr(m_ZExtOrSExt(Op), Op); |
1669 | } |
1670 | |
1671 | template <typename OpTy> |
1672 | inline CastClass_match<OpTy, Instruction::UIToFP> m_UIToFP(const OpTy &Op) { |
1673 | return CastClass_match<OpTy, Instruction::UIToFP>(Op); |
1674 | } |
1675 | |
1676 | template <typename OpTy> |
1677 | inline CastClass_match<OpTy, Instruction::SIToFP> m_SIToFP(const OpTy &Op) { |
1678 | return CastClass_match<OpTy, Instruction::SIToFP>(Op); |
1679 | } |
1680 | |
1681 | template <typename OpTy> |
1682 | inline CastClass_match<OpTy, Instruction::FPToUI> m_FPToUI(const OpTy &Op) { |
1683 | return CastClass_match<OpTy, Instruction::FPToUI>(Op); |
1684 | } |
1685 | |
1686 | template <typename OpTy> |
1687 | inline CastClass_match<OpTy, Instruction::FPToSI> m_FPToSI(const OpTy &Op) { |
1688 | return CastClass_match<OpTy, Instruction::FPToSI>(Op); |
1689 | } |
1690 | |
1691 | template <typename OpTy> |
1692 | inline CastClass_match<OpTy, Instruction::FPTrunc> m_FPTrunc(const OpTy &Op) { |
1693 | return CastClass_match<OpTy, Instruction::FPTrunc>(Op); |
1694 | } |
1695 | |
1696 | template <typename OpTy> |
1697 | inline CastClass_match<OpTy, Instruction::FPExt> m_FPExt(const OpTy &Op) { |
1698 | return CastClass_match<OpTy, Instruction::FPExt>(Op); |
1699 | } |
1700 | |
1701 | //===----------------------------------------------------------------------===// |
1702 | // Matchers for control flow. |
1703 | // |
1704 | |
1705 | struct br_match { |
1706 | BasicBlock *&Succ; |
1707 | |
1708 | br_match(BasicBlock *&Succ) : Succ(Succ) {} |
1709 | |
1710 | template <typename OpTy> bool match(OpTy *V) { |
1711 | if (auto *BI = dyn_cast<BranchInst>(V)) |
1712 | if (BI->isUnconditional()) { |
1713 | Succ = BI->getSuccessor(0); |
1714 | return true; |
1715 | } |
1716 | return false; |
1717 | } |
1718 | }; |
1719 | |
1720 | inline br_match m_UnconditionalBr(BasicBlock *&Succ) { return br_match(Succ); } |
1721 | |
1722 | template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> |
1723 | struct brc_match { |
1724 | Cond_t Cond; |
1725 | TrueBlock_t T; |
1726 | FalseBlock_t F; |
1727 | |
1728 | brc_match(const Cond_t &C, const TrueBlock_t &t, const FalseBlock_t &f) |
1729 | : Cond(C), T(t), F(f) {} |
1730 | |
1731 | template <typename OpTy> bool match(OpTy *V) { |
1732 | if (auto *BI = dyn_cast<BranchInst>(V)) |
1733 | if (BI->isConditional() && Cond.match(BI->getCondition())) |
1734 | return T.match(BI->getSuccessor(0)) && F.match(BI->getSuccessor(1)); |
1735 | return false; |
1736 | } |
1737 | }; |
1738 | |
1739 | template <typename Cond_t> |
1740 | inline brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>> |
1741 | m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) { |
1742 | return brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>( |
1743 | C, m_BasicBlock(T), m_BasicBlock(F)); |
1744 | } |
1745 | |
1746 | template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> |
1747 | inline brc_match<Cond_t, TrueBlock_t, FalseBlock_t> |
1748 | m_Br(const Cond_t &C, const TrueBlock_t &T, const FalseBlock_t &F) { |
1749 | return brc_match<Cond_t, TrueBlock_t, FalseBlock_t>(C, T, F); |
1750 | } |
1751 | |
1752 | //===----------------------------------------------------------------------===// |
1753 | // Matchers for max/min idioms, eg: "select (sgt x, y), x, y" -> smax(x,y). |
1754 | // |
1755 | |
1756 | template <typename CmpInst_t, typename LHS_t, typename RHS_t, typename Pred_t, |
1757 | bool Commutable = false> |
1758 | struct MaxMin_match { |
1759 | using PredType = Pred_t; |
1760 | LHS_t L; |
1761 | RHS_t R; |
1762 | |
1763 | // The evaluation order is always stable, regardless of Commutability. |
1764 | // The LHS is always matched first. |
1765 | MaxMin_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} |
1766 | |
1767 | template <typename OpTy> bool match(OpTy *V) { |
1768 | if (auto *II = dyn_cast<IntrinsicInst>(V)) { |
1769 | Intrinsic::ID IID = II->getIntrinsicID(); |
1770 | if ((IID == Intrinsic::smax && Pred_t::match(ICmpInst::ICMP_SGT)) || |
1771 | (IID == Intrinsic::smin && Pred_t::match(ICmpInst::ICMP_SLT)) || |
1772 | (IID == Intrinsic::umax && Pred_t::match(ICmpInst::ICMP_UGT)) || |
1773 | (IID == Intrinsic::umin && Pred_t::match(ICmpInst::ICMP_ULT))) { |
1774 | Value *LHS = II->getOperand(0), *RHS = II->getOperand(1); |
1775 | return (L.match(LHS) && R.match(RHS)) || |
1776 | (Commutable && L.match(RHS) && R.match(LHS)); |
1777 | } |
1778 | } |
1779 | // Look for "(x pred y) ? x : y" or "(x pred y) ? y : x". |
1780 | auto *SI = dyn_cast<SelectInst>(V); |
1781 | if (!SI) |
1782 | return false; |
1783 | auto *Cmp = dyn_cast<CmpInst_t>(SI->getCondition()); |
1784 | if (!Cmp) |
1785 | return false; |
1786 | // At this point we have a select conditioned on a comparison. Check that |
1787 | // it is the values returned by the select that are being compared. |
1788 | auto *TrueVal = SI->getTrueValue(); |
1789 | auto *FalseVal = SI->getFalseValue(); |
1790 | auto *LHS = Cmp->getOperand(0); |
1791 | auto *RHS = Cmp->getOperand(1); |
1792 | if ((TrueVal != LHS || FalseVal != RHS) && |
1793 | (TrueVal != RHS || FalseVal != LHS)) |
1794 | return false; |
1795 | typename CmpInst_t::Predicate Pred = |
1796 | LHS == TrueVal ? Cmp->getPredicate() : Cmp->getInversePredicate(); |
1797 | // Does "(x pred y) ? x : y" represent the desired max/min operation? |
1798 | if (!Pred_t::match(Pred)) |
1799 | return false; |
1800 | // It does! Bind the operands. |
1801 | return (L.match(LHS) && R.match(RHS)) || |
1802 | (Commutable && L.match(RHS) && R.match(LHS)); |
1803 | } |
1804 | }; |
1805 | |
1806 | /// Helper class for identifying signed max predicates. |
1807 | struct smax_pred_ty { |
1808 | static bool match(ICmpInst::Predicate Pred) { |
1809 | return Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SGE; |
1810 | } |
1811 | }; |
1812 | |
1813 | /// Helper class for identifying signed min predicates. |
1814 | struct smin_pred_ty { |
1815 | static bool match(ICmpInst::Predicate Pred) { |
1816 | return Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SLE; |
1817 | } |
1818 | }; |
1819 | |
1820 | /// Helper class for identifying unsigned max predicates. |
1821 | struct umax_pred_ty { |
1822 | static bool match(ICmpInst::Predicate Pred) { |
1823 | return Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_UGE; |
1824 | } |
1825 | }; |
1826 | |
1827 | /// Helper class for identifying unsigned min predicates. |
1828 | struct umin_pred_ty { |
1829 | static bool match(ICmpInst::Predicate Pred) { |
1830 | return Pred == CmpInst::ICMP_ULT || Pred == CmpInst::ICMP_ULE; |
1831 | } |
1832 | }; |
1833 | |
1834 | /// Helper class for identifying ordered max predicates. |
1835 | struct ofmax_pred_ty { |
1836 | static bool match(FCmpInst::Predicate Pred) { |
1837 | return Pred == CmpInst::FCMP_OGT || Pred == CmpInst::FCMP_OGE; |
1838 | } |
1839 | }; |
1840 | |
1841 | /// Helper class for identifying ordered min predicates. |
1842 | struct ofmin_pred_ty { |
1843 | static bool match(FCmpInst::Predicate Pred) { |
1844 | return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE; |
1845 | } |
1846 | }; |
1847 | |
1848 | /// Helper class for identifying unordered max predicates. |
1849 | struct ufmax_pred_ty { |
1850 | static bool match(FCmpInst::Predicate Pred) { |
1851 | return Pred == CmpInst::FCMP_UGT || Pred == CmpInst::FCMP_UGE; |
1852 | } |
1853 | }; |
1854 | |
1855 | /// Helper class for identifying unordered min predicates. |
1856 | struct ufmin_pred_ty { |
1857 | static bool match(FCmpInst::Predicate Pred) { |
1858 | return Pred == CmpInst::FCMP_ULT || Pred == CmpInst::FCMP_ULE; |
1859 | } |
1860 | }; |
1861 | |
1862 | template <typename LHS, typename RHS> |
1863 | inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty> m_SMax(const LHS &L, |
1864 | const RHS &R) { |
1865 | return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>(L, R); |
1866 | } |
1867 | |
1868 | template <typename LHS, typename RHS> |
1869 | inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty> m_SMin(const LHS &L, |
1870 | const RHS &R) { |
1871 | return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>(L, R); |
1872 | } |
1873 | |
1874 | template <typename LHS, typename RHS> |
1875 | inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty> m_UMax(const LHS &L, |
1876 | const RHS &R) { |
1877 | return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>(L, R); |
1878 | } |
1879 | |
1880 | template <typename LHS, typename RHS> |
1881 | inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty> m_UMin(const LHS &L, |
1882 | const RHS &R) { |
1883 | return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>(L, R); |
1884 | } |
1885 | |
1886 | template <typename LHS, typename RHS> |
1887 | inline match_combine_or< |
1888 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>, |
1889 | MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>>, |
1890 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>, |
1891 | MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>>> |
1892 | m_MaxOrMin(const LHS &L, const RHS &R) { |
1893 | return m_CombineOr(m_CombineOr(m_SMax(L, R), m_SMin(L, R)), |
1894 | m_CombineOr(m_UMax(L, R), m_UMin(L, R))); |
1895 | } |
1896 | |
1897 | /// Match an 'ordered' floating point maximum function. |
1898 | /// Floating point has one special value 'NaN'. Therefore, there is no total |
1899 | /// order. However, if we can ignore the 'NaN' value (for example, because of a |
1900 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum' |
1901 | /// semantics. In the presence of 'NaN' we have to preserve the original |
1902 | /// select(fcmp(ogt/ge, L, R), L, R) semantics matched by this predicate. |
1903 | /// |
1904 | /// max(L, R) iff L and R are not NaN |
1905 | /// m_OrdFMax(L, R) = R iff L or R are NaN |
1906 | template <typename LHS, typename RHS> |
1907 | inline MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty> m_OrdFMax(const LHS &L, |
1908 | const RHS &R) { |
1909 | return MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty>(L, R); |
1910 | } |
1911 | |
1912 | /// Match an 'ordered' floating point minimum function. |
1913 | /// Floating point has one special value 'NaN'. Therefore, there is no total |
1914 | /// order. However, if we can ignore the 'NaN' value (for example, because of a |
1915 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum' |
1916 | /// semantics. In the presence of 'NaN' we have to preserve the original |
1917 | /// select(fcmp(olt/le, L, R), L, R) semantics matched by this predicate. |
1918 | /// |
1919 | /// min(L, R) iff L and R are not NaN |
1920 | /// m_OrdFMin(L, R) = R iff L or R are NaN |
1921 | template <typename LHS, typename RHS> |
1922 | inline MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty> m_OrdFMin(const LHS &L, |
1923 | const RHS &R) { |
1924 | return MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty>(L, R); |
1925 | } |
1926 | |
1927 | /// Match an 'unordered' floating point maximum function. |
1928 | /// Floating point has one special value 'NaN'. Therefore, there is no total |
1929 | /// order. However, if we can ignore the 'NaN' value (for example, because of a |
1930 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum' |
1931 | /// semantics. In the presence of 'NaN' we have to preserve the original |
1932 | /// select(fcmp(ugt/ge, L, R), L, R) semantics matched by this predicate. |
1933 | /// |
1934 | /// max(L, R) iff L and R are not NaN |
1935 | /// m_UnordFMax(L, R) = L iff L or R are NaN |
1936 | template <typename LHS, typename RHS> |
1937 | inline MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty> |
1938 | m_UnordFMax(const LHS &L, const RHS &R) { |
1939 | return MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>(L, R); |
1940 | } |
1941 | |
1942 | /// Match an 'unordered' floating point minimum function. |
1943 | /// Floating point has one special value 'NaN'. Therefore, there is no total |
1944 | /// order. However, if we can ignore the 'NaN' value (for example, because of a |
1945 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum' |
1946 | /// semantics. In the presence of 'NaN' we have to preserve the original |
1947 | /// select(fcmp(ult/le, L, R), L, R) semantics matched by this predicate. |
1948 | /// |
1949 | /// min(L, R) iff L and R are not NaN |
1950 | /// m_UnordFMin(L, R) = L iff L or R are NaN |
1951 | template <typename LHS, typename RHS> |
1952 | inline MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty> |
1953 | m_UnordFMin(const LHS &L, const RHS &R) { |
1954 | return MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>(L, R); |
1955 | } |
1956 | |
1957 | //===----------------------------------------------------------------------===// |
1958 | // Matchers for overflow check patterns: e.g. (a + b) u< a, (a ^ -1) <u b |
1959 | // Note that S might be matched to other instructions than AddInst. |
1960 | // |
1961 | |
1962 | template <typename LHS_t, typename RHS_t, typename Sum_t> |
1963 | struct UAddWithOverflow_match { |
1964 | LHS_t L; |
1965 | RHS_t R; |
1966 | Sum_t S; |
1967 | |
1968 | UAddWithOverflow_match(const LHS_t &L, const RHS_t &R, const Sum_t &S) |
1969 | : L(L), R(R), S(S) {} |
1970 | |
1971 | template <typename OpTy> bool match(OpTy *V) { |
1972 | Value *ICmpLHS, *ICmpRHS; |
1973 | ICmpInst::Predicate Pred; |
1974 | if (!m_ICmp(Pred, m_Value(ICmpLHS), m_Value(ICmpRHS)).match(V)) |
1975 | return false; |
1976 | |
1977 | Value *AddLHS, *AddRHS; |
1978 | auto AddExpr = m_Add(m_Value(AddLHS), m_Value(AddRHS)); |
1979 | |
1980 | // (a + b) u< a, (a + b) u< b |
1981 | if (Pred == ICmpInst::ICMP_ULT) |
1982 | if (AddExpr.match(ICmpLHS) && (ICmpRHS == AddLHS || ICmpRHS == AddRHS)) |
1983 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS); |
1984 | |
1985 | // a >u (a + b), b >u (a + b) |
1986 | if (Pred == ICmpInst::ICMP_UGT) |
1987 | if (AddExpr.match(ICmpRHS) && (ICmpLHS == AddLHS || ICmpLHS == AddRHS)) |
1988 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); |
1989 | |
1990 | Value *Op1; |
1991 | auto XorExpr = m_OneUse(m_Xor(m_Value(Op1), m_AllOnes())); |
1992 | // (a ^ -1) <u b |
1993 | if (Pred == ICmpInst::ICMP_ULT) { |
1994 | if (XorExpr.match(ICmpLHS)) |
1995 | return L.match(Op1) && R.match(ICmpRHS) && S.match(ICmpLHS); |
1996 | } |
1997 | // b > u (a ^ -1) |
1998 | if (Pred == ICmpInst::ICMP_UGT) { |
1999 | if (XorExpr.match(ICmpRHS)) |
2000 | return L.match(Op1) && R.match(ICmpLHS) && S.match(ICmpRHS); |
2001 | } |
2002 | |
2003 | // Match special-case for increment-by-1. |
2004 | if (Pred == ICmpInst::ICMP_EQ) { |
2005 | // (a + 1) == 0 |
2006 | // (1 + a) == 0 |
2007 | if (AddExpr.match(ICmpLHS) && m_ZeroInt().match(ICmpRHS) && |
2008 | (m_One().match(AddLHS) || m_One().match(AddRHS))) |
2009 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS); |
2010 | // 0 == (a + 1) |
2011 | // 0 == (1 + a) |
2012 | if (m_ZeroInt().match(ICmpLHS) && AddExpr.match(ICmpRHS) && |
2013 | (m_One().match(AddLHS) || m_One().match(AddRHS))) |
2014 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); |
2015 | } |
2016 | |
2017 | return false; |
2018 | } |
2019 | }; |
2020 | |
2021 | /// Match an icmp instruction checking for unsigned overflow on addition. |
2022 | /// |
2023 | /// S is matched to the addition whose result is being checked for overflow, and |
2024 | /// L and R are matched to the LHS and RHS of S. |
2025 | template <typename LHS_t, typename RHS_t, typename Sum_t> |
2026 | UAddWithOverflow_match<LHS_t, RHS_t, Sum_t> |
2027 | m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S) { |
2028 | return UAddWithOverflow_match<LHS_t, RHS_t, Sum_t>(L, R, S); |
2029 | } |
2030 | |
2031 | template <typename Opnd_t> struct Argument_match { |
2032 | unsigned OpI; |
2033 | Opnd_t Val; |
2034 | |
2035 | Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {} |
2036 | |
2037 | template <typename OpTy> bool match(OpTy *V) { |
2038 | // FIXME: Should likely be switched to use `CallBase`. |
2039 | if (const auto *CI = dyn_cast<CallInst>(V)) |
2040 | return Val.match(CI->getArgOperand(OpI)); |
2041 | return false; |
2042 | } |
2043 | }; |
2044 | |
2045 | /// Match an argument. |
2046 | template <unsigned OpI, typename Opnd_t> |
2047 | inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) { |
2048 | return Argument_match<Opnd_t>(OpI, Op); |
2049 | } |
2050 | |
2051 | /// Intrinsic matchers. |
2052 | struct IntrinsicID_match { |
2053 | unsigned ID; |
2054 | |
2055 | IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {} |
2056 | |
2057 | template <typename OpTy> bool match(OpTy *V) { |
2058 | if (const auto *CI = dyn_cast<CallInst>(V)) |
2059 | if (const auto *F = CI->getCalledFunction()) |
2060 | return F->getIntrinsicID() == ID; |
2061 | return false; |
2062 | } |
2063 | }; |
2064 | |
2065 | /// Intrinsic matches are combinations of ID matchers, and argument |
2066 | /// matchers. Higher arity matcher are defined recursively in terms of and-ing |
2067 | /// them with lower arity matchers. Here's some convenient typedefs for up to |
2068 | /// several arguments, and more can be added as needed |
2069 | template <typename T0 = void, typename T1 = void, typename T2 = void, |
2070 | typename T3 = void, typename T4 = void, typename T5 = void, |
2071 | typename T6 = void, typename T7 = void, typename T8 = void, |
2072 | typename T9 = void, typename T10 = void> |
2073 | struct m_Intrinsic_Ty; |
2074 | template <typename T0> struct m_Intrinsic_Ty<T0> { |
2075 | using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>; |
2076 | }; |
2077 | template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> { |
2078 | using Ty = |
2079 | match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>; |
2080 | }; |
2081 | template <typename T0, typename T1, typename T2> |
2082 | struct m_Intrinsic_Ty<T0, T1, T2> { |
2083 | using Ty = |
2084 | match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty, |
2085 | Argument_match<T2>>; |
2086 | }; |
2087 | template <typename T0, typename T1, typename T2, typename T3> |
2088 | struct m_Intrinsic_Ty<T0, T1, T2, T3> { |
2089 | using Ty = |
2090 | match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty, |
2091 | Argument_match<T3>>; |
2092 | }; |
2093 | |
2094 | template <typename T0, typename T1, typename T2, typename T3, typename T4> |
2095 | struct m_Intrinsic_Ty<T0, T1, T2, T3, T4> { |
2096 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty, |
2097 | Argument_match<T4>>; |
2098 | }; |
2099 | |
2100 | template <typename T0, typename T1, typename T2, typename T3, typename T4, typename T5> |
2101 | struct m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5> { |
2102 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty, |
2103 | Argument_match<T5>>; |
2104 | }; |
2105 | |
2106 | /// Match intrinsic calls like this: |
2107 | /// m_Intrinsic<Intrinsic::fabs>(m_Value(X)) |
2108 | template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() { |
2109 | return IntrinsicID_match(IntrID); |
2110 | } |
2111 | |
2112 | /// Matches MaskedLoad Intrinsic. |
2113 | template <typename Opnd0, typename Opnd1, typename Opnd2, typename Opnd3> |
2114 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2, Opnd3>::Ty |
2115 | m_MaskedLoad(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2, |
2116 | const Opnd3 &Op3) { |
2117 | return m_Intrinsic<Intrinsic::masked_load>(Op0, Op1, Op2, Op3); |
2118 | } |
2119 | |
2120 | template <Intrinsic::ID IntrID, typename T0> |
2121 | inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) { |
2122 | return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0)); |
2123 | } |
2124 | |
2125 | template <Intrinsic::ID IntrID, typename T0, typename T1> |
2126 | inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0, |
2127 | const T1 &Op1) { |
2128 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1)); |
2129 | } |
2130 | |
2131 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2> |
2132 | inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty |
2133 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) { |
2134 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2)); |
2135 | } |
2136 | |
2137 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, |
2138 | typename T3> |
2139 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty |
2140 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) { |
2141 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3)); |
2142 | } |
2143 | |
2144 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, |
2145 | typename T3, typename T4> |
2146 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty |
2147 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3, |
2148 | const T4 &Op4) { |
2149 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3), |
2150 | m_Argument<4>(Op4)); |
2151 | } |
2152 | |
2153 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, |
2154 | typename T3, typename T4, typename T5> |
2155 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5>::Ty |
2156 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3, |
2157 | const T4 &Op4, const T5 &Op5) { |
2158 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3, Op4), |
2159 | m_Argument<5>(Op5)); |
2160 | } |
2161 | |
2162 | // Helper intrinsic matching specializations. |
2163 | template <typename Opnd0> |
2164 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BitReverse(const Opnd0 &Op0) { |
2165 | return m_Intrinsic<Intrinsic::bitreverse>(Op0); |
2166 | } |
2167 | |
2168 | template <typename Opnd0> |
2169 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BSwap(const Opnd0 &Op0) { |
2170 | return m_Intrinsic<Intrinsic::bswap>(Op0); |
2171 | } |
2172 | |
2173 | template <typename Opnd0> |
2174 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FAbs(const Opnd0 &Op0) { |
2175 | return m_Intrinsic<Intrinsic::fabs>(Op0); |
2176 | } |
2177 | |
2178 | template <typename Opnd0> |
2179 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FCanonicalize(const Opnd0 &Op0) { |
2180 | return m_Intrinsic<Intrinsic::canonicalize>(Op0); |
2181 | } |
2182 | |
2183 | template <typename Opnd0, typename Opnd1> |
2184 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMin(const Opnd0 &Op0, |
2185 | const Opnd1 &Op1) { |
2186 | return m_Intrinsic<Intrinsic::minnum>(Op0, Op1); |
2187 | } |
2188 | |
2189 | template <typename Opnd0, typename Opnd1> |
2190 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMax(const Opnd0 &Op0, |
2191 | const Opnd1 &Op1) { |
2192 | return m_Intrinsic<Intrinsic::maxnum>(Op0, Op1); |
2193 | } |
2194 | |
2195 | template <typename Opnd0, typename Opnd1, typename Opnd2> |
2196 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty |
2197 | m_FShl(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) { |
2198 | return m_Intrinsic<Intrinsic::fshl>(Op0, Op1, Op2); |
2199 | } |
2200 | |
2201 | template <typename Opnd0, typename Opnd1, typename Opnd2> |
2202 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty |
2203 | m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) { |
2204 | return m_Intrinsic<Intrinsic::fshr>(Op0, Op1, Op2); |
2205 | } |
2206 | |
2207 | //===----------------------------------------------------------------------===// |
2208 | // Matchers for two-operands operators with the operators in either order |
2209 | // |
2210 | |
2211 | /// Matches a BinaryOperator with LHS and RHS in either order. |
2212 | template <typename LHS, typename RHS> |
2213 | inline AnyBinaryOp_match<LHS, RHS, true> m_c_BinOp(const LHS &L, const RHS &R) { |
2214 | return AnyBinaryOp_match<LHS, RHS, true>(L, R); |
2215 | } |
2216 | |
2217 | /// Matches an ICmp with a predicate over LHS and RHS in either order. |
2218 | /// Swaps the predicate if operands are commuted. |
2219 | template <typename LHS, typename RHS> |
2220 | inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true> |
2221 | m_c_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { |
2222 | return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true>(Pred, L, |
2223 | R); |
2224 | } |
2225 | |
2226 | /// Matches a Add with LHS and RHS in either order. |
2227 | template <typename LHS, typename RHS> |
2228 | inline BinaryOp_match<LHS, RHS, Instruction::Add, true> m_c_Add(const LHS &L, |
2229 | const RHS &R) { |
2230 | return BinaryOp_match<LHS, RHS, Instruction::Add, true>(L, R); |
2231 | } |
2232 | |
2233 | /// Matches a Mul with LHS and RHS in either order. |
2234 | template <typename LHS, typename RHS> |
2235 | inline BinaryOp_match<LHS, RHS, Instruction::Mul, true> m_c_Mul(const LHS &L, |
2236 | const RHS &R) { |
2237 | return BinaryOp_match<LHS, RHS, Instruction::Mul, true>(L, R); |
2238 | } |
2239 | |
2240 | /// Matches an And with LHS and RHS in either order. |
2241 | template <typename LHS, typename RHS> |
2242 | inline BinaryOp_match<LHS, RHS, Instruction::And, true> m_c_And(const LHS &L, |
2243 | const RHS &R) { |
2244 | return BinaryOp_match<LHS, RHS, Instruction::And, true>(L, R); |
2245 | } |
2246 | |
2247 | /// Matches an Or with LHS and RHS in either order. |
2248 | template <typename LHS, typename RHS> |
2249 | inline BinaryOp_match<LHS, RHS, Instruction::Or, true> m_c_Or(const LHS &L, |
2250 | const RHS &R) { |
2251 | return BinaryOp_match<LHS, RHS, Instruction::Or, true>(L, R); |
2252 | } |
2253 | |
2254 | /// Matches an Xor with LHS and RHS in either order. |
2255 | template <typename LHS, typename RHS> |
2256 | inline BinaryOp_match<LHS, RHS, Instruction::Xor, true> m_c_Xor(const LHS &L, |
2257 | const RHS &R) { |
2258 | return BinaryOp_match<LHS, RHS, Instruction::Xor, true>(L, R); |
2259 | } |
2260 | |
2261 | /// Matches a 'Neg' as 'sub 0, V'. |
2262 | template <typename ValTy> |
2263 | inline BinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, Instruction::Sub> |
2264 | m_Neg(const ValTy &V) { |
2265 | return m_Sub(m_ZeroInt(), V); |
2266 | } |
2267 | |
2268 | /// Matches a 'Neg' as 'sub nsw 0, V'. |
2269 | template <typename ValTy> |
2270 | inline OverflowingBinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, |
2271 | Instruction::Sub, |
2272 | OverflowingBinaryOperator::NoSignedWrap> |
2273 | m_NSWNeg(const ValTy &V) { |
2274 | return m_NSWSub(m_ZeroInt(), V); |
2275 | } |
2276 | |
2277 | /// Matches a 'Not' as 'xor V, -1' or 'xor -1, V'. |
2278 | template <typename ValTy> |
2279 | inline BinaryOp_match<ValTy, cst_pred_ty<is_all_ones>, Instruction::Xor, true> |
2280 | m_Not(const ValTy &V) { |
2281 | return m_c_Xor(V, m_AllOnes()); |
2282 | } |
2283 | |
2284 | /// Matches an SMin with LHS and RHS in either order. |
2285 | template <typename LHS, typename RHS> |
2286 | inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true> |
2287 | m_c_SMin(const LHS &L, const RHS &R) { |
2288 | return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>(L, R); |
2289 | } |
2290 | /// Matches an SMax with LHS and RHS in either order. |
2291 | template <typename LHS, typename RHS> |
2292 | inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true> |
2293 | m_c_SMax(const LHS &L, const RHS &R) { |
2294 | return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>(L, R); |
2295 | } |
2296 | /// Matches a UMin with LHS and RHS in either order. |
2297 | template <typename LHS, typename RHS> |
2298 | inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true> |
2299 | m_c_UMin(const LHS &L, const RHS &R) { |
2300 | return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>(L, R); |
2301 | } |
2302 | /// Matches a UMax with LHS and RHS in either order. |
2303 | template <typename LHS, typename RHS> |
2304 | inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true> |
2305 | m_c_UMax(const LHS &L, const RHS &R) { |
2306 | return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>(L, R); |
2307 | } |
2308 | |
2309 | template <typename LHS, typename RHS> |
2310 | inline match_combine_or< |
2311 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>, |
2312 | MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>>, |
2313 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>, |
2314 | MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>>> |
2315 | m_c_MaxOrMin(const LHS &L, const RHS &R) { |
2316 | return m_CombineOr(m_CombineOr(m_c_SMax(L, R), m_c_SMin(L, R)), |
2317 | m_CombineOr(m_c_UMax(L, R), m_c_UMin(L, R))); |
2318 | } |
2319 | |
2320 | /// Matches FAdd with LHS and RHS in either order. |
2321 | template <typename LHS, typename RHS> |
2322 | inline BinaryOp_match<LHS, RHS, Instruction::FAdd, true> |
2323 | m_c_FAdd(const LHS &L, const RHS &R) { |
2324 | return BinaryOp_match<LHS, RHS, Instruction::FAdd, true>(L, R); |
2325 | } |
2326 | |
2327 | /// Matches FMul with LHS and RHS in either order. |
2328 | template <typename LHS, typename RHS> |
2329 | inline BinaryOp_match<LHS, RHS, Instruction::FMul, true> |
2330 | m_c_FMul(const LHS &L, const RHS &R) { |
2331 | return BinaryOp_match<LHS, RHS, Instruction::FMul, true>(L, R); |
2332 | } |
2333 | |
2334 | template <typename Opnd_t> struct Signum_match { |
2335 | Opnd_t Val; |
2336 | Signum_match(const Opnd_t &V) : Val(V) {} |
2337 | |
2338 | template <typename OpTy> bool match(OpTy *V) { |
2339 | unsigned TypeSize = V->getType()->getScalarSizeInBits(); |
2340 | if (TypeSize == 0) |
2341 | return false; |
2342 | |
2343 | unsigned ShiftWidth = TypeSize - 1; |
2344 | Value *OpL = nullptr, *OpR = nullptr; |
2345 | |
2346 | // This is the representation of signum we match: |
2347 | // |
2348 | // signum(x) == (x >> 63) | (-x >>u 63) |
2349 | // |
2350 | // An i1 value is its own signum, so it's correct to match |
2351 | // |
2352 | // signum(x) == (x >> 0) | (-x >>u 0) |
2353 | // |
2354 | // for i1 values. |
2355 | |
2356 | auto LHS = m_AShr(m_Value(OpL), m_SpecificInt(ShiftWidth)); |
2357 | auto RHS = m_LShr(m_Neg(m_Value(OpR)), m_SpecificInt(ShiftWidth)); |
2358 | auto Signum = m_Or(LHS, RHS); |
2359 | |
2360 | return Signum.match(V) && OpL == OpR && Val.match(OpL); |
2361 | } |
2362 | }; |
2363 | |
2364 | /// Matches a signum pattern. |
2365 | /// |
2366 | /// signum(x) = |
2367 | /// x > 0 -> 1 |
2368 | /// x == 0 -> 0 |
2369 | /// x < 0 -> -1 |
2370 | template <typename Val_t> inline Signum_match<Val_t> m_Signum(const Val_t &V) { |
2371 | return Signum_match<Val_t>(V); |
2372 | } |
2373 | |
2374 | template <int Ind, typename Opnd_t> struct ExtractValue_match { |
2375 | Opnd_t Val; |
2376 | ExtractValue_match(const Opnd_t &V) : Val(V) {} |
2377 | |
2378 | template <typename OpTy> bool match(OpTy *V) { |
2379 | if (auto *I = dyn_cast<ExtractValueInst>(V)) { |
2380 | // If Ind is -1, don't inspect indices |
2381 | if (Ind != -1 && |
2382 | !(I->getNumIndices() == 1 && I->getIndices()[0] == (unsigned)Ind)) |
2383 | return false; |
2384 | return Val.match(I->getAggregateOperand()); |
2385 | } |
2386 | return false; |
2387 | } |
2388 | }; |
2389 | |
2390 | /// Match a single index ExtractValue instruction. |
2391 | /// For example m_ExtractValue<1>(...) |
2392 | template <int Ind, typename Val_t> |
2393 | inline ExtractValue_match<Ind, Val_t> m_ExtractValue(const Val_t &V) { |
2394 | return ExtractValue_match<Ind, Val_t>(V); |
2395 | } |
2396 | |
2397 | /// Match an ExtractValue instruction with any index. |
2398 | /// For example m_ExtractValue(...) |
2399 | template <typename Val_t> |
2400 | inline ExtractValue_match<-1, Val_t> m_ExtractValue(const Val_t &V) { |
2401 | return ExtractValue_match<-1, Val_t>(V); |
2402 | } |
2403 | |
2404 | /// Matcher for a single index InsertValue instruction. |
2405 | template <int Ind, typename T0, typename T1> struct InsertValue_match { |
2406 | T0 Op0; |
2407 | T1 Op1; |
2408 | |
2409 | InsertValue_match(const T0 &Op0, const T1 &Op1) : Op0(Op0), Op1(Op1) {} |
2410 | |
2411 | template <typename OpTy> bool match(OpTy *V) { |
2412 | if (auto *I = dyn_cast<InsertValueInst>(V)) { |
2413 | return Op0.match(I->getOperand(0)) && Op1.match(I->getOperand(1)) && |
2414 | I->getNumIndices() == 1 && Ind == I->getIndices()[0]; |
2415 | } |
2416 | return false; |
2417 | } |
2418 | }; |
2419 | |
2420 | /// Matches a single index InsertValue instruction. |
2421 | template <int Ind, typename Val_t, typename Elt_t> |
2422 | inline InsertValue_match<Ind, Val_t, Elt_t> m_InsertValue(const Val_t &Val, |
2423 | const Elt_t &Elt) { |
2424 | return InsertValue_match<Ind, Val_t, Elt_t>(Val, Elt); |
2425 | } |
2426 | |
2427 | /// Matches patterns for `vscale`. This can either be a call to `llvm.vscale` or |
2428 | /// the constant expression |
2429 | /// `ptrtoint(gep <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1>` |
2430 | /// under the right conditions determined by DataLayout. |
2431 | struct VScaleVal_match { |
2432 | const DataLayout &DL; |
2433 | VScaleVal_match(const DataLayout &DL) : DL(DL) {} |
2434 | |
2435 | template <typename ITy> bool match(ITy *V) { |
2436 | if (m_Intrinsic<Intrinsic::vscale>().match(V)) |
2437 | return true; |
2438 | |
2439 | Value *Ptr; |
2440 | if (m_PtrToInt(m_Value(Ptr)).match(V)) { |
2441 | if (auto *GEP = dyn_cast<GEPOperator>(Ptr)) { |
2442 | auto *DerefTy = GEP->getSourceElementType(); |
2443 | if (GEP->getNumIndices() == 1 && isa<ScalableVectorType>(DerefTy) && |
2444 | m_Zero().match(GEP->getPointerOperand()) && |
2445 | m_SpecificInt(1).match(GEP->idx_begin()->get()) && |
2446 | DL.getTypeAllocSizeInBits(DerefTy).getKnownMinSize() == 8) |
2447 | return true; |
2448 | } |
2449 | } |
2450 | |
2451 | return false; |
2452 | } |
2453 | }; |
2454 | |
2455 | inline VScaleVal_match m_VScale(const DataLayout &DL) { |
2456 | return VScaleVal_match(DL); |
2457 | } |
2458 | |
2459 | template <typename LHS, typename RHS, unsigned Opcode> |
2460 | struct LogicalOp_match { |
2461 | LHS L; |
2462 | RHS R; |
2463 | |
2464 | LogicalOp_match(const LHS &L, const RHS &R) : L(L), R(R) {} |
2465 | |
2466 | template <typename T> bool match(T *V) { |
2467 | if (auto *I = dyn_cast<Instruction>(V)) { |
2468 | if (!I->getType()->isIntOrIntVectorTy(1)) |
2469 | return false; |
2470 | |
2471 | if (I->getOpcode() == Opcode && L.match(I->getOperand(0)) && |
2472 | R.match(I->getOperand(1))) |
2473 | return true; |
2474 | |
2475 | if (auto *SI = dyn_cast<SelectInst>(I)) { |
2476 | if (Opcode == Instruction::And) { |
2477 | if (const auto *C = dyn_cast<Constant>(SI->getFalseValue())) |
2478 | if (C->isNullValue() && L.match(SI->getCondition()) && |
2479 | R.match(SI->getTrueValue())) |
2480 | return true; |
2481 | } else { |
2482 | assert(Opcode == Instruction::Or)(static_cast <bool> (Opcode == Instruction::Or) ? void ( 0) : __assert_fail ("Opcode == Instruction::Or", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/IR/PatternMatch.h" , 2482, __extension__ __PRETTY_FUNCTION__)); |
2483 | if (const auto *C = dyn_cast<Constant>(SI->getTrueValue())) |
2484 | if (C->isOneValue() && L.match(SI->getCondition()) && |
2485 | R.match(SI->getFalseValue())) |
2486 | return true; |
2487 | } |
2488 | } |
2489 | } |
2490 | |
2491 | return false; |
2492 | } |
2493 | }; |
2494 | |
2495 | /// Matches L && R either in the form of L & R or L ? R : false. |
2496 | /// Note that the latter form is poison-blocking. |
2497 | template <typename LHS, typename RHS> |
2498 | inline LogicalOp_match<LHS, RHS, Instruction::And> |
2499 | m_LogicalAnd(const LHS &L, const RHS &R) { |
2500 | return LogicalOp_match<LHS, RHS, Instruction::And>(L, R); |
2501 | } |
2502 | |
2503 | /// Matches L && R where L and R are arbitrary values. |
2504 | inline auto m_LogicalAnd() { return m_LogicalAnd(m_Value(), m_Value()); } |
2505 | |
2506 | /// Matches L || R either in the form of L | R or L ? true : R. |
2507 | /// Note that the latter form is poison-blocking. |
2508 | template <typename LHS, typename RHS> |
2509 | inline LogicalOp_match<LHS, RHS, Instruction::Or> |
2510 | m_LogicalOr(const LHS &L, const RHS &R) { |
2511 | return LogicalOp_match<LHS, RHS, Instruction::Or>(L, R); |
2512 | } |
2513 | |
2514 | /// Matches L || R where L and R are arbitrary values. |
2515 | inline auto m_LogicalOr() { |
2516 | return m_LogicalOr(m_Value(), m_Value()); |
2517 | } |
2518 | |
2519 | } // end namespace PatternMatch |
2520 | } // end namespace llvm |
2521 | |
2522 | #endif // LLVM_IR_PATTERNMATCH_H |