File: | llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp |
Warning: | line 556, column 14 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- InstCombineCompares.cpp --------------------------------------------===// | ||||
2 | // | ||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||
6 | // | ||||
7 | //===----------------------------------------------------------------------===// | ||||
8 | // | ||||
9 | // This file implements the visitICmp and visitFCmp functions. | ||||
10 | // | ||||
11 | //===----------------------------------------------------------------------===// | ||||
12 | |||||
13 | #include "InstCombineInternal.h" | ||||
14 | #include "llvm/ADT/APSInt.h" | ||||
15 | #include "llvm/ADT/SetVector.h" | ||||
16 | #include "llvm/ADT/Statistic.h" | ||||
17 | #include "llvm/Analysis/ConstantFolding.h" | ||||
18 | #include "llvm/Analysis/InstructionSimplify.h" | ||||
19 | #include "llvm/Analysis/TargetLibraryInfo.h" | ||||
20 | #include "llvm/IR/ConstantRange.h" | ||||
21 | #include "llvm/IR/DataLayout.h" | ||||
22 | #include "llvm/IR/GetElementPtrTypeIterator.h" | ||||
23 | #include "llvm/IR/IntrinsicInst.h" | ||||
24 | #include "llvm/IR/PatternMatch.h" | ||||
25 | #include "llvm/Support/Debug.h" | ||||
26 | #include "llvm/Support/KnownBits.h" | ||||
27 | #include "llvm/Transforms/InstCombine/InstCombiner.h" | ||||
28 | |||||
29 | using namespace llvm; | ||||
30 | using namespace PatternMatch; | ||||
31 | |||||
32 | #define DEBUG_TYPE"instcombine" "instcombine" | ||||
33 | |||||
34 | // How many times is a select replaced by one of its operands? | ||||
35 | STATISTIC(NumSel, "Number of select opts")static llvm::Statistic NumSel = {"instcombine", "NumSel", "Number of select opts" }; | ||||
36 | |||||
37 | |||||
38 | /// Compute Result = In1+In2, returning true if the result overflowed for this | ||||
39 | /// type. | ||||
40 | static bool addWithOverflow(APInt &Result, const APInt &In1, | ||||
41 | const APInt &In2, bool IsSigned = false) { | ||||
42 | bool Overflow; | ||||
43 | if (IsSigned) | ||||
44 | Result = In1.sadd_ov(In2, Overflow); | ||||
45 | else | ||||
46 | Result = In1.uadd_ov(In2, Overflow); | ||||
47 | |||||
48 | return Overflow; | ||||
49 | } | ||||
50 | |||||
51 | /// Compute Result = In1-In2, returning true if the result overflowed for this | ||||
52 | /// type. | ||||
53 | static bool subWithOverflow(APInt &Result, const APInt &In1, | ||||
54 | const APInt &In2, bool IsSigned = false) { | ||||
55 | bool Overflow; | ||||
56 | if (IsSigned) | ||||
57 | Result = In1.ssub_ov(In2, Overflow); | ||||
58 | else | ||||
59 | Result = In1.usub_ov(In2, Overflow); | ||||
60 | |||||
61 | return Overflow; | ||||
62 | } | ||||
63 | |||||
64 | /// Given an icmp instruction, return true if any use of this comparison is a | ||||
65 | /// branch on sign bit comparison. | ||||
66 | static bool hasBranchUse(ICmpInst &I) { | ||||
67 | for (auto *U : I.users()) | ||||
68 | if (isa<BranchInst>(U)) | ||||
69 | return true; | ||||
70 | return false; | ||||
71 | } | ||||
72 | |||||
73 | /// Returns true if the exploded icmp can be expressed as a signed comparison | ||||
74 | /// to zero and updates the predicate accordingly. | ||||
75 | /// The signedness of the comparison is preserved. | ||||
76 | /// TODO: Refactor with decomposeBitTestICmp()? | ||||
77 | static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) { | ||||
78 | if (!ICmpInst::isSigned(Pred)) | ||||
79 | return false; | ||||
80 | |||||
81 | if (C.isNullValue()) | ||||
82 | return ICmpInst::isRelational(Pred); | ||||
83 | |||||
84 | if (C.isOneValue()) { | ||||
85 | if (Pred == ICmpInst::ICMP_SLT) { | ||||
86 | Pred = ICmpInst::ICMP_SLE; | ||||
87 | return true; | ||||
88 | } | ||||
89 | } else if (C.isAllOnesValue()) { | ||||
90 | if (Pred == ICmpInst::ICMP_SGT) { | ||||
91 | Pred = ICmpInst::ICMP_SGE; | ||||
92 | return true; | ||||
93 | } | ||||
94 | } | ||||
95 | |||||
96 | return false; | ||||
97 | } | ||||
98 | |||||
99 | /// Given a signed integer type and a set of known zero and one bits, compute | ||||
100 | /// the maximum and minimum values that could have the specified known zero and | ||||
101 | /// known one bits, returning them in Min/Max. | ||||
102 | /// TODO: Move to method on KnownBits struct? | ||||
103 | static void computeSignedMinMaxValuesFromKnownBits(const KnownBits &Known, | ||||
104 | APInt &Min, APInt &Max) { | ||||
105 | assert(Known.getBitWidth() == Min.getBitWidth() &&((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 107, __PRETTY_FUNCTION__)) | ||||
106 | Known.getBitWidth() == Max.getBitWidth() &&((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 107, __PRETTY_FUNCTION__)) | ||||
107 | "KnownZero, KnownOne and Min, Max must have equal bitwidth.")((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 107, __PRETTY_FUNCTION__)); | ||||
108 | APInt UnknownBits = ~(Known.Zero|Known.One); | ||||
109 | |||||
110 | // The minimum value is when all unknown bits are zeros, EXCEPT for the sign | ||||
111 | // bit if it is unknown. | ||||
112 | Min = Known.One; | ||||
113 | Max = Known.One|UnknownBits; | ||||
114 | |||||
115 | if (UnknownBits.isNegative()) { // Sign bit is unknown | ||||
116 | Min.setSignBit(); | ||||
117 | Max.clearSignBit(); | ||||
118 | } | ||||
119 | } | ||||
120 | |||||
121 | /// Given an unsigned integer type and a set of known zero and one bits, compute | ||||
122 | /// the maximum and minimum values that could have the specified known zero and | ||||
123 | /// known one bits, returning them in Min/Max. | ||||
124 | /// TODO: Move to method on KnownBits struct? | ||||
125 | static void computeUnsignedMinMaxValuesFromKnownBits(const KnownBits &Known, | ||||
126 | APInt &Min, APInt &Max) { | ||||
127 | assert(Known.getBitWidth() == Min.getBitWidth() &&((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 129, __PRETTY_FUNCTION__)) | ||||
128 | Known.getBitWidth() == Max.getBitWidth() &&((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 129, __PRETTY_FUNCTION__)) | ||||
129 | "Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth.")((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 129, __PRETTY_FUNCTION__)); | ||||
130 | APInt UnknownBits = ~(Known.Zero|Known.One); | ||||
131 | |||||
132 | // The minimum value is when the unknown bits are all zeros. | ||||
133 | Min = Known.One; | ||||
134 | // The maximum value is when the unknown bits are all ones. | ||||
135 | Max = Known.One|UnknownBits; | ||||
136 | } | ||||
137 | |||||
138 | /// This is called when we see this pattern: | ||||
139 | /// cmp pred (load (gep GV, ...)), cmpcst | ||||
140 | /// where GV is a global variable with a constant initializer. Try to simplify | ||||
141 | /// this into some simple computation that does not need the load. For example | ||||
142 | /// we can optimize "icmp eq (load (gep "foo", 0, i)), 0" into "icmp eq i, 3". | ||||
143 | /// | ||||
144 | /// If AndCst is non-null, then the loaded value is masked with that constant | ||||
145 | /// before doing the comparison. This handles cases like "A[i]&4 == 0". | ||||
146 | Instruction * | ||||
147 | InstCombinerImpl::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, | ||||
148 | GlobalVariable *GV, CmpInst &ICI, | ||||
149 | ConstantInt *AndCst) { | ||||
150 | Constant *Init = GV->getInitializer(); | ||||
151 | if (!isa<ConstantArray>(Init) && !isa<ConstantDataArray>(Init)) | ||||
152 | return nullptr; | ||||
153 | |||||
154 | uint64_t ArrayElementCount = Init->getType()->getArrayNumElements(); | ||||
155 | // Don't blow up on huge arrays. | ||||
156 | if (ArrayElementCount > MaxArraySizeForCombine) | ||||
157 | return nullptr; | ||||
158 | |||||
159 | // There are many forms of this optimization we can handle, for now, just do | ||||
160 | // the simple index into a single-dimensional array. | ||||
161 | // | ||||
162 | // Require: GEP GV, 0, i {{, constant indices}} | ||||
163 | if (GEP->getNumOperands() < 3 || | ||||
164 | !isa<ConstantInt>(GEP->getOperand(1)) || | ||||
165 | !cast<ConstantInt>(GEP->getOperand(1))->isZero() || | ||||
166 | isa<Constant>(GEP->getOperand(2))) | ||||
167 | return nullptr; | ||||
168 | |||||
169 | // Check that indices after the variable are constants and in-range for the | ||||
170 | // type they index. Collect the indices. This is typically for arrays of | ||||
171 | // structs. | ||||
172 | SmallVector<unsigned, 4> LaterIndices; | ||||
173 | |||||
174 | Type *EltTy = Init->getType()->getArrayElementType(); | ||||
175 | for (unsigned i = 3, e = GEP->getNumOperands(); i != e; ++i) { | ||||
176 | ConstantInt *Idx = dyn_cast<ConstantInt>(GEP->getOperand(i)); | ||||
177 | if (!Idx) return nullptr; // Variable index. | ||||
178 | |||||
179 | uint64_t IdxVal = Idx->getZExtValue(); | ||||
180 | if ((unsigned)IdxVal != IdxVal) return nullptr; // Too large array index. | ||||
181 | |||||
182 | if (StructType *STy = dyn_cast<StructType>(EltTy)) | ||||
183 | EltTy = STy->getElementType(IdxVal); | ||||
184 | else if (ArrayType *ATy = dyn_cast<ArrayType>(EltTy)) { | ||||
185 | if (IdxVal >= ATy->getNumElements()) return nullptr; | ||||
186 | EltTy = ATy->getElementType(); | ||||
187 | } else { | ||||
188 | return nullptr; // Unknown type. | ||||
189 | } | ||||
190 | |||||
191 | LaterIndices.push_back(IdxVal); | ||||
192 | } | ||||
193 | |||||
194 | enum { Overdefined = -3, Undefined = -2 }; | ||||
195 | |||||
196 | // Variables for our state machines. | ||||
197 | |||||
198 | // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form | ||||
199 | // "i == 47 | i == 87", where 47 is the first index the condition is true for, | ||||
200 | // and 87 is the second (and last) index. FirstTrueElement is -2 when | ||||
201 | // undefined, otherwise set to the first true element. SecondTrueElement is | ||||
202 | // -2 when undefined, -3 when overdefined and >= 0 when that index is true. | ||||
203 | int FirstTrueElement = Undefined, SecondTrueElement = Undefined; | ||||
204 | |||||
205 | // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the | ||||
206 | // form "i != 47 & i != 87". Same state transitions as for true elements. | ||||
207 | int FirstFalseElement = Undefined, SecondFalseElement = Undefined; | ||||
208 | |||||
209 | /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these | ||||
210 | /// define a state machine that triggers for ranges of values that the index | ||||
211 | /// is true or false for. This triggers on things like "abbbbc"[i] == 'b'. | ||||
212 | /// This is -2 when undefined, -3 when overdefined, and otherwise the last | ||||
213 | /// index in the range (inclusive). We use -2 for undefined here because we | ||||
214 | /// use relative comparisons and don't want 0-1 to match -1. | ||||
215 | int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined; | ||||
216 | |||||
217 | // MagicBitvector - This is a magic bitvector where we set a bit if the | ||||
218 | // comparison is true for element 'i'. If there are 64 elements or less in | ||||
219 | // the array, this will fully represent all the comparison results. | ||||
220 | uint64_t MagicBitvector = 0; | ||||
221 | |||||
222 | // Scan the array and see if one of our patterns matches. | ||||
223 | Constant *CompareRHS = cast<Constant>(ICI.getOperand(1)); | ||||
224 | for (unsigned i = 0, e = ArrayElementCount; i != e; ++i) { | ||||
225 | Constant *Elt = Init->getAggregateElement(i); | ||||
226 | if (!Elt) return nullptr; | ||||
227 | |||||
228 | // If this is indexing an array of structures, get the structure element. | ||||
229 | if (!LaterIndices.empty()) | ||||
230 | Elt = ConstantExpr::getExtractValue(Elt, LaterIndices); | ||||
231 | |||||
232 | // If the element is masked, handle it. | ||||
233 | if (AndCst) Elt = ConstantExpr::getAnd(Elt, AndCst); | ||||
234 | |||||
235 | // Find out if the comparison would be true or false for the i'th element. | ||||
236 | Constant *C = ConstantFoldCompareInstOperands(ICI.getPredicate(), Elt, | ||||
237 | CompareRHS, DL, &TLI); | ||||
238 | // If the result is undef for this element, ignore it. | ||||
239 | if (isa<UndefValue>(C)) { | ||||
240 | // Extend range state machines to cover this element in case there is an | ||||
241 | // undef in the middle of the range. | ||||
242 | if (TrueRangeEnd == (int)i-1) | ||||
243 | TrueRangeEnd = i; | ||||
244 | if (FalseRangeEnd == (int)i-1) | ||||
245 | FalseRangeEnd = i; | ||||
246 | continue; | ||||
247 | } | ||||
248 | |||||
249 | // If we can't compute the result for any of the elements, we have to give | ||||
250 | // up evaluating the entire conditional. | ||||
251 | if (!isa<ConstantInt>(C)) return nullptr; | ||||
252 | |||||
253 | // Otherwise, we know if the comparison is true or false for this element, | ||||
254 | // update our state machines. | ||||
255 | bool IsTrueForElt = !cast<ConstantInt>(C)->isZero(); | ||||
256 | |||||
257 | // State machine for single/double/range index comparison. | ||||
258 | if (IsTrueForElt) { | ||||
259 | // Update the TrueElement state machine. | ||||
260 | if (FirstTrueElement == Undefined) | ||||
261 | FirstTrueElement = TrueRangeEnd = i; // First true element. | ||||
262 | else { | ||||
263 | // Update double-compare state machine. | ||||
264 | if (SecondTrueElement == Undefined) | ||||
265 | SecondTrueElement = i; | ||||
266 | else | ||||
267 | SecondTrueElement = Overdefined; | ||||
268 | |||||
269 | // Update range state machine. | ||||
270 | if (TrueRangeEnd == (int)i-1) | ||||
271 | TrueRangeEnd = i; | ||||
272 | else | ||||
273 | TrueRangeEnd = Overdefined; | ||||
274 | } | ||||
275 | } else { | ||||
276 | // Update the FalseElement state machine. | ||||
277 | if (FirstFalseElement == Undefined) | ||||
278 | FirstFalseElement = FalseRangeEnd = i; // First false element. | ||||
279 | else { | ||||
280 | // Update double-compare state machine. | ||||
281 | if (SecondFalseElement == Undefined) | ||||
282 | SecondFalseElement = i; | ||||
283 | else | ||||
284 | SecondFalseElement = Overdefined; | ||||
285 | |||||
286 | // Update range state machine. | ||||
287 | if (FalseRangeEnd == (int)i-1) | ||||
288 | FalseRangeEnd = i; | ||||
289 | else | ||||
290 | FalseRangeEnd = Overdefined; | ||||
291 | } | ||||
292 | } | ||||
293 | |||||
294 | // If this element is in range, update our magic bitvector. | ||||
295 | if (i < 64 && IsTrueForElt) | ||||
296 | MagicBitvector |= 1ULL << i; | ||||
297 | |||||
298 | // If all of our states become overdefined, bail out early. Since the | ||||
299 | // predicate is expensive, only check it every 8 elements. This is only | ||||
300 | // really useful for really huge arrays. | ||||
301 | if ((i & 8) == 0 && i >= 64 && SecondTrueElement == Overdefined && | ||||
302 | SecondFalseElement == Overdefined && TrueRangeEnd == Overdefined && | ||||
303 | FalseRangeEnd == Overdefined) | ||||
304 | return nullptr; | ||||
305 | } | ||||
306 | |||||
307 | // Now that we've scanned the entire array, emit our new comparison(s). We | ||||
308 | // order the state machines in complexity of the generated code. | ||||
309 | Value *Idx = GEP->getOperand(2); | ||||
310 | |||||
311 | // If the index is larger than the pointer size of the target, truncate the | ||||
312 | // index down like the GEP would do implicitly. We don't have to do this for | ||||
313 | // an inbounds GEP because the index can't be out of range. | ||||
314 | if (!GEP->isInBounds()) { | ||||
315 | Type *IntPtrTy = DL.getIntPtrType(GEP->getType()); | ||||
316 | unsigned PtrSize = IntPtrTy->getIntegerBitWidth(); | ||||
317 | if (Idx->getType()->getPrimitiveSizeInBits() > PtrSize) | ||||
318 | Idx = Builder.CreateTrunc(Idx, IntPtrTy); | ||||
319 | } | ||||
320 | |||||
321 | // If the comparison is only true for one or two elements, emit direct | ||||
322 | // comparisons. | ||||
323 | if (SecondTrueElement != Overdefined) { | ||||
324 | // None true -> false. | ||||
325 | if (FirstTrueElement == Undefined) | ||||
326 | return replaceInstUsesWith(ICI, Builder.getFalse()); | ||||
327 | |||||
328 | Value *FirstTrueIdx = ConstantInt::get(Idx->getType(), FirstTrueElement); | ||||
329 | |||||
330 | // True for one element -> 'i == 47'. | ||||
331 | if (SecondTrueElement == Undefined) | ||||
332 | return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx); | ||||
333 | |||||
334 | // True for two elements -> 'i == 47 | i == 72'. | ||||
335 | Value *C1 = Builder.CreateICmpEQ(Idx, FirstTrueIdx); | ||||
336 | Value *SecondTrueIdx = ConstantInt::get(Idx->getType(), SecondTrueElement); | ||||
337 | Value *C2 = Builder.CreateICmpEQ(Idx, SecondTrueIdx); | ||||
338 | return BinaryOperator::CreateOr(C1, C2); | ||||
339 | } | ||||
340 | |||||
341 | // If the comparison is only false for one or two elements, emit direct | ||||
342 | // comparisons. | ||||
343 | if (SecondFalseElement != Overdefined) { | ||||
344 | // None false -> true. | ||||
345 | if (FirstFalseElement == Undefined) | ||||
346 | return replaceInstUsesWith(ICI, Builder.getTrue()); | ||||
347 | |||||
348 | Value *FirstFalseIdx = ConstantInt::get(Idx->getType(), FirstFalseElement); | ||||
349 | |||||
350 | // False for one element -> 'i != 47'. | ||||
351 | if (SecondFalseElement == Undefined) | ||||
352 | return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx); | ||||
353 | |||||
354 | // False for two elements -> 'i != 47 & i != 72'. | ||||
355 | Value *C1 = Builder.CreateICmpNE(Idx, FirstFalseIdx); | ||||
356 | Value *SecondFalseIdx = ConstantInt::get(Idx->getType(),SecondFalseElement); | ||||
357 | Value *C2 = Builder.CreateICmpNE(Idx, SecondFalseIdx); | ||||
358 | return BinaryOperator::CreateAnd(C1, C2); | ||||
359 | } | ||||
360 | |||||
361 | // If the comparison can be replaced with a range comparison for the elements | ||||
362 | // where it is true, emit the range check. | ||||
363 | if (TrueRangeEnd != Overdefined) { | ||||
364 | assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare")((TrueRangeEnd != FirstTrueElement && "Should emit single compare" ) ? static_cast<void> (0) : __assert_fail ("TrueRangeEnd != FirstTrueElement && \"Should emit single compare\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 364, __PRETTY_FUNCTION__)); | ||||
365 | |||||
366 | // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1). | ||||
367 | if (FirstTrueElement) { | ||||
368 | Value *Offs = ConstantInt::get(Idx->getType(), -FirstTrueElement); | ||||
369 | Idx = Builder.CreateAdd(Idx, Offs); | ||||
370 | } | ||||
371 | |||||
372 | Value *End = ConstantInt::get(Idx->getType(), | ||||
373 | TrueRangeEnd-FirstTrueElement+1); | ||||
374 | return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End); | ||||
375 | } | ||||
376 | |||||
377 | // False range check. | ||||
378 | if (FalseRangeEnd != Overdefined) { | ||||
379 | assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare")((FalseRangeEnd != FirstFalseElement && "Should emit single compare" ) ? static_cast<void> (0) : __assert_fail ("FalseRangeEnd != FirstFalseElement && \"Should emit single compare\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 379, __PRETTY_FUNCTION__)); | ||||
380 | // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse). | ||||
381 | if (FirstFalseElement) { | ||||
382 | Value *Offs = ConstantInt::get(Idx->getType(), -FirstFalseElement); | ||||
383 | Idx = Builder.CreateAdd(Idx, Offs); | ||||
384 | } | ||||
385 | |||||
386 | Value *End = ConstantInt::get(Idx->getType(), | ||||
387 | FalseRangeEnd-FirstFalseElement); | ||||
388 | return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End); | ||||
389 | } | ||||
390 | |||||
391 | // If a magic bitvector captures the entire comparison state | ||||
392 | // of this load, replace it with computation that does: | ||||
393 | // ((magic_cst >> i) & 1) != 0 | ||||
394 | { | ||||
395 | Type *Ty = nullptr; | ||||
396 | |||||
397 | // Look for an appropriate type: | ||||
398 | // - The type of Idx if the magic fits | ||||
399 | // - The smallest fitting legal type | ||||
400 | if (ArrayElementCount <= Idx->getType()->getIntegerBitWidth()) | ||||
401 | Ty = Idx->getType(); | ||||
402 | else | ||||
403 | Ty = DL.getSmallestLegalIntType(Init->getContext(), ArrayElementCount); | ||||
404 | |||||
405 | if (Ty) { | ||||
406 | Value *V = Builder.CreateIntCast(Idx, Ty, false); | ||||
407 | V = Builder.CreateLShr(ConstantInt::get(Ty, MagicBitvector), V); | ||||
408 | V = Builder.CreateAnd(ConstantInt::get(Ty, 1), V); | ||||
409 | return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, 0)); | ||||
410 | } | ||||
411 | } | ||||
412 | |||||
413 | return nullptr; | ||||
414 | } | ||||
415 | |||||
416 | /// Return a value that can be used to compare the *offset* implied by a GEP to | ||||
417 | /// zero. For example, if we have &A[i], we want to return 'i' for | ||||
418 | /// "icmp ne i, 0". Note that, in general, indices can be complex, and scales | ||||
419 | /// are involved. The above expression would also be legal to codegen as | ||||
420 | /// "icmp ne (i*4), 0" (assuming A is a pointer to i32). | ||||
421 | /// This latter form is less amenable to optimization though, and we are allowed | ||||
422 | /// to generate the first by knowing that pointer arithmetic doesn't overflow. | ||||
423 | /// | ||||
424 | /// If we can't emit an optimized form for this expression, this returns null. | ||||
425 | /// | ||||
426 | static Value *evaluateGEPOffsetExpression(User *GEP, InstCombinerImpl &IC, | ||||
427 | const DataLayout &DL) { | ||||
428 | gep_type_iterator GTI = gep_type_begin(GEP); | ||||
429 | |||||
430 | // Check to see if this gep only has a single variable index. If so, and if | ||||
431 | // any constant indices are a multiple of its scale, then we can compute this | ||||
432 | // in terms of the scale of the variable index. For example, if the GEP | ||||
433 | // implies an offset of "12 + i*4", then we can codegen this as "3 + i", | ||||
434 | // because the expression will cross zero at the same point. | ||||
435 | unsigned i, e = GEP->getNumOperands(); | ||||
436 | int64_t Offset = 0; | ||||
437 | for (i = 1; i != e; ++i, ++GTI) { | ||||
438 | if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) { | ||||
439 | // Compute the aggregate offset of constant indices. | ||||
440 | if (CI->isZero()) continue; | ||||
441 | |||||
442 | // Handle a struct index, which adds its field offset to the pointer. | ||||
443 | if (StructType *STy = GTI.getStructTypeOrNull()) { | ||||
444 | Offset += DL.getStructLayout(STy)->getElementOffset(CI->getZExtValue()); | ||||
445 | } else { | ||||
446 | uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType()); | ||||
447 | Offset += Size*CI->getSExtValue(); | ||||
448 | } | ||||
449 | } else { | ||||
450 | // Found our variable index. | ||||
451 | break; | ||||
452 | } | ||||
453 | } | ||||
454 | |||||
455 | // If there are no variable indices, we must have a constant offset, just | ||||
456 | // evaluate it the general way. | ||||
457 | if (i == e) return nullptr; | ||||
458 | |||||
459 | Value *VariableIdx = GEP->getOperand(i); | ||||
460 | // Determine the scale factor of the variable element. For example, this is | ||||
461 | // 4 if the variable index is into an array of i32. | ||||
462 | uint64_t VariableScale = DL.getTypeAllocSize(GTI.getIndexedType()); | ||||
463 | |||||
464 | // Verify that there are no other variable indices. If so, emit the hard way. | ||||
465 | for (++i, ++GTI; i != e; ++i, ++GTI) { | ||||
466 | ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i)); | ||||
467 | if (!CI) return nullptr; | ||||
468 | |||||
469 | // Compute the aggregate offset of constant indices. | ||||
470 | if (CI->isZero()) continue; | ||||
471 | |||||
472 | // Handle a struct index, which adds its field offset to the pointer. | ||||
473 | if (StructType *STy = GTI.getStructTypeOrNull()) { | ||||
474 | Offset += DL.getStructLayout(STy)->getElementOffset(CI->getZExtValue()); | ||||
475 | } else { | ||||
476 | uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType()); | ||||
477 | Offset += Size*CI->getSExtValue(); | ||||
478 | } | ||||
479 | } | ||||
480 | |||||
481 | // Okay, we know we have a single variable index, which must be a | ||||
482 | // pointer/array/vector index. If there is no offset, life is simple, return | ||||
483 | // the index. | ||||
484 | Type *IntPtrTy = DL.getIntPtrType(GEP->getOperand(0)->getType()); | ||||
485 | unsigned IntPtrWidth = IntPtrTy->getIntegerBitWidth(); | ||||
486 | if (Offset == 0) { | ||||
487 | // Cast to intptrty in case a truncation occurs. If an extension is needed, | ||||
488 | // we don't need to bother extending: the extension won't affect where the | ||||
489 | // computation crosses zero. | ||||
490 | if (VariableIdx->getType()->getPrimitiveSizeInBits() > IntPtrWidth) { | ||||
491 | VariableIdx = IC.Builder.CreateTrunc(VariableIdx, IntPtrTy); | ||||
492 | } | ||||
493 | return VariableIdx; | ||||
494 | } | ||||
495 | |||||
496 | // Otherwise, there is an index. The computation we will do will be modulo | ||||
497 | // the pointer size. | ||||
498 | Offset = SignExtend64(Offset, IntPtrWidth); | ||||
499 | VariableScale = SignExtend64(VariableScale, IntPtrWidth); | ||||
500 | |||||
501 | // To do this transformation, any constant index must be a multiple of the | ||||
502 | // variable scale factor. For example, we can evaluate "12 + 4*i" as "3 + i", | ||||
503 | // but we can't evaluate "10 + 3*i" in terms of i. Check that the offset is a | ||||
504 | // multiple of the variable scale. | ||||
505 | int64_t NewOffs = Offset / (int64_t)VariableScale; | ||||
506 | if (Offset != NewOffs*(int64_t)VariableScale) | ||||
507 | return nullptr; | ||||
508 | |||||
509 | // Okay, we can do this evaluation. Start by converting the index to intptr. | ||||
510 | if (VariableIdx->getType() != IntPtrTy) | ||||
511 | VariableIdx = IC.Builder.CreateIntCast(VariableIdx, IntPtrTy, | ||||
512 | true /*Signed*/); | ||||
513 | Constant *OffsetVal = ConstantInt::get(IntPtrTy, NewOffs); | ||||
514 | return IC.Builder.CreateAdd(VariableIdx, OffsetVal, "offset"); | ||||
515 | } | ||||
516 | |||||
517 | /// Returns true if we can rewrite Start as a GEP with pointer Base | ||||
518 | /// and some integer offset. The nodes that need to be re-written | ||||
519 | /// for this transformation will be added to Explored. | ||||
520 | static bool canRewriteGEPAsOffset(Value *Start, Value *Base, | ||||
521 | const DataLayout &DL, | ||||
522 | SetVector<Value *> &Explored) { | ||||
523 | SmallVector<Value *, 16> WorkList(1, Start); | ||||
524 | Explored.insert(Base); | ||||
525 | |||||
526 | // The following traversal gives us an order which can be used | ||||
527 | // when doing the final transformation. Since in the final | ||||
528 | // transformation we create the PHI replacement instructions first, | ||||
529 | // we don't have to get them in any particular order. | ||||
530 | // | ||||
531 | // However, for other instructions we will have to traverse the | ||||
532 | // operands of an instruction first, which means that we have to | ||||
533 | // do a post-order traversal. | ||||
534 | while (!WorkList.empty()) { | ||||
535 | SetVector<PHINode *> PHIs; | ||||
536 | |||||
537 | while (!WorkList.empty()) { | ||||
538 | if (Explored.size() >= 100) | ||||
539 | return false; | ||||
540 | |||||
541 | Value *V = WorkList.back(); | ||||
542 | |||||
543 | if (Explored.count(V) != 0) { | ||||
544 | WorkList.pop_back(); | ||||
545 | continue; | ||||
546 | } | ||||
547 | |||||
548 | if (!isa<IntToPtrInst>(V) && !isa<PtrToIntInst>(V) && | ||||
549 | !isa<GetElementPtrInst>(V) && !isa<PHINode>(V)) | ||||
550 | // We've found some value that we can't explore which is different from | ||||
551 | // the base. Therefore we can't do this transformation. | ||||
552 | return false; | ||||
553 | |||||
554 | if (isa<IntToPtrInst>(V) || isa<PtrToIntInst>(V)) { | ||||
555 | auto *CI = dyn_cast<CastInst>(V); | ||||
556 | if (!CI->isNoopCast(DL)) | ||||
| |||||
557 | return false; | ||||
558 | |||||
559 | if (Explored.count(CI->getOperand(0)) == 0) | ||||
560 | WorkList.push_back(CI->getOperand(0)); | ||||
561 | } | ||||
562 | |||||
563 | if (auto *GEP = dyn_cast<GEPOperator>(V)) { | ||||
564 | // We're limiting the GEP to having one index. This will preserve | ||||
565 | // the original pointer type. We could handle more cases in the | ||||
566 | // future. | ||||
567 | if (GEP->getNumIndices() != 1 || !GEP->isInBounds() || | ||||
568 | GEP->getType() != Start->getType()) | ||||
569 | return false; | ||||
570 | |||||
571 | if (Explored.count(GEP->getOperand(0)) == 0) | ||||
572 | WorkList.push_back(GEP->getOperand(0)); | ||||
573 | } | ||||
574 | |||||
575 | if (WorkList.back() == V) { | ||||
576 | WorkList.pop_back(); | ||||
577 | // We've finished visiting this node, mark it as such. | ||||
578 | Explored.insert(V); | ||||
579 | } | ||||
580 | |||||
581 | if (auto *PN = dyn_cast<PHINode>(V)) { | ||||
582 | // We cannot transform PHIs on unsplittable basic blocks. | ||||
583 | if (isa<CatchSwitchInst>(PN->getParent()->getTerminator())) | ||||
584 | return false; | ||||
585 | Explored.insert(PN); | ||||
586 | PHIs.insert(PN); | ||||
587 | } | ||||
588 | } | ||||
589 | |||||
590 | // Explore the PHI nodes further. | ||||
591 | for (auto *PN : PHIs) | ||||
592 | for (Value *Op : PN->incoming_values()) | ||||
593 | if (Explored.count(Op) == 0) | ||||
594 | WorkList.push_back(Op); | ||||
595 | } | ||||
596 | |||||
597 | // Make sure that we can do this. Since we can't insert GEPs in a basic | ||||
598 | // block before a PHI node, we can't easily do this transformation if | ||||
599 | // we have PHI node users of transformed instructions. | ||||
600 | for (Value *Val : Explored) { | ||||
601 | for (Value *Use : Val->uses()) { | ||||
602 | |||||
603 | auto *PHI = dyn_cast<PHINode>(Use); | ||||
604 | auto *Inst = dyn_cast<Instruction>(Val); | ||||
605 | |||||
606 | if (Inst == Base || Inst == PHI || !Inst || !PHI || | ||||
607 | Explored.count(PHI) == 0) | ||||
608 | continue; | ||||
609 | |||||
610 | if (PHI->getParent() == Inst->getParent()) | ||||
611 | return false; | ||||
612 | } | ||||
613 | } | ||||
614 | return true; | ||||
615 | } | ||||
616 | |||||
617 | // Sets the appropriate insert point on Builder where we can add | ||||
618 | // a replacement Instruction for V (if that is possible). | ||||
619 | static void setInsertionPoint(IRBuilder<> &Builder, Value *V, | ||||
620 | bool Before = true) { | ||||
621 | if (auto *PHI = dyn_cast<PHINode>(V)) { | ||||
622 | Builder.SetInsertPoint(&*PHI->getParent()->getFirstInsertionPt()); | ||||
623 | return; | ||||
624 | } | ||||
625 | if (auto *I = dyn_cast<Instruction>(V)) { | ||||
626 | if (!Before) | ||||
627 | I = &*std::next(I->getIterator()); | ||||
628 | Builder.SetInsertPoint(I); | ||||
629 | return; | ||||
630 | } | ||||
631 | if (auto *A = dyn_cast<Argument>(V)) { | ||||
632 | // Set the insertion point in the entry block. | ||||
633 | BasicBlock &Entry = A->getParent()->getEntryBlock(); | ||||
634 | Builder.SetInsertPoint(&*Entry.getFirstInsertionPt()); | ||||
635 | return; | ||||
636 | } | ||||
637 | // Otherwise, this is a constant and we don't need to set a new | ||||
638 | // insertion point. | ||||
639 | assert(isa<Constant>(V) && "Setting insertion point for unknown value!")((isa<Constant>(V) && "Setting insertion point for unknown value!" ) ? static_cast<void> (0) : __assert_fail ("isa<Constant>(V) && \"Setting insertion point for unknown value!\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 639, __PRETTY_FUNCTION__)); | ||||
640 | } | ||||
641 | |||||
642 | /// Returns a re-written value of Start as an indexed GEP using Base as a | ||||
643 | /// pointer. | ||||
644 | static Value *rewriteGEPAsOffset(Value *Start, Value *Base, | ||||
645 | const DataLayout &DL, | ||||
646 | SetVector<Value *> &Explored) { | ||||
647 | // Perform all the substitutions. This is a bit tricky because we can | ||||
648 | // have cycles in our use-def chains. | ||||
649 | // 1. Create the PHI nodes without any incoming values. | ||||
650 | // 2. Create all the other values. | ||||
651 | // 3. Add the edges for the PHI nodes. | ||||
652 | // 4. Emit GEPs to get the original pointers. | ||||
653 | // 5. Remove the original instructions. | ||||
654 | Type *IndexType = IntegerType::get( | ||||
655 | Base->getContext(), DL.getIndexTypeSizeInBits(Start->getType())); | ||||
656 | |||||
657 | DenseMap<Value *, Value *> NewInsts; | ||||
658 | NewInsts[Base] = ConstantInt::getNullValue(IndexType); | ||||
659 | |||||
660 | // Create the new PHI nodes, without adding any incoming values. | ||||
661 | for (Value *Val : Explored) { | ||||
662 | if (Val == Base) | ||||
663 | continue; | ||||
664 | // Create empty phi nodes. This avoids cyclic dependencies when creating | ||||
665 | // the remaining instructions. | ||||
666 | if (auto *PHI = dyn_cast<PHINode>(Val)) | ||||
667 | NewInsts[PHI] = PHINode::Create(IndexType, PHI->getNumIncomingValues(), | ||||
668 | PHI->getName() + ".idx", PHI); | ||||
669 | } | ||||
670 | IRBuilder<> Builder(Base->getContext()); | ||||
671 | |||||
672 | // Create all the other instructions. | ||||
673 | for (Value *Val : Explored) { | ||||
674 | |||||
675 | if (NewInsts.find(Val) != NewInsts.end()) | ||||
676 | continue; | ||||
677 | |||||
678 | if (auto *CI = dyn_cast<CastInst>(Val)) { | ||||
679 | // Don't get rid of the intermediate variable here; the store can grow | ||||
680 | // the map which will invalidate the reference to the input value. | ||||
681 | Value *V = NewInsts[CI->getOperand(0)]; | ||||
682 | NewInsts[CI] = V; | ||||
683 | continue; | ||||
684 | } | ||||
685 | if (auto *GEP = dyn_cast<GEPOperator>(Val)) { | ||||
686 | Value *Index = NewInsts[GEP->getOperand(1)] ? NewInsts[GEP->getOperand(1)] | ||||
687 | : GEP->getOperand(1); | ||||
688 | setInsertionPoint(Builder, GEP); | ||||
689 | // Indices might need to be sign extended. GEPs will magically do | ||||
690 | // this, but we need to do it ourselves here. | ||||
691 | if (Index->getType()->getScalarSizeInBits() != | ||||
692 | NewInsts[GEP->getOperand(0)]->getType()->getScalarSizeInBits()) { | ||||
693 | Index = Builder.CreateSExtOrTrunc( | ||||
694 | Index, NewInsts[GEP->getOperand(0)]->getType(), | ||||
695 | GEP->getOperand(0)->getName() + ".sext"); | ||||
696 | } | ||||
697 | |||||
698 | auto *Op = NewInsts[GEP->getOperand(0)]; | ||||
699 | if (isa<ConstantInt>(Op) && cast<ConstantInt>(Op)->isZero()) | ||||
700 | NewInsts[GEP] = Index; | ||||
701 | else | ||||
702 | NewInsts[GEP] = Builder.CreateNSWAdd( | ||||
703 | Op, Index, GEP->getOperand(0)->getName() + ".add"); | ||||
704 | continue; | ||||
705 | } | ||||
706 | if (isa<PHINode>(Val)) | ||||
707 | continue; | ||||
708 | |||||
709 | llvm_unreachable("Unexpected instruction type")::llvm::llvm_unreachable_internal("Unexpected instruction type" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 709); | ||||
710 | } | ||||
711 | |||||
712 | // Add the incoming values to the PHI nodes. | ||||
713 | for (Value *Val : Explored) { | ||||
714 | if (Val == Base) | ||||
715 | continue; | ||||
716 | // All the instructions have been created, we can now add edges to the | ||||
717 | // phi nodes. | ||||
718 | if (auto *PHI = dyn_cast<PHINode>(Val)) { | ||||
719 | PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]); | ||||
720 | for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) { | ||||
721 | Value *NewIncoming = PHI->getIncomingValue(I); | ||||
722 | |||||
723 | if (NewInsts.find(NewIncoming) != NewInsts.end()) | ||||
724 | NewIncoming = NewInsts[NewIncoming]; | ||||
725 | |||||
726 | NewPhi->addIncoming(NewIncoming, PHI->getIncomingBlock(I)); | ||||
727 | } | ||||
728 | } | ||||
729 | } | ||||
730 | |||||
731 | for (Value *Val : Explored) { | ||||
732 | if (Val == Base) | ||||
733 | continue; | ||||
734 | |||||
735 | // Depending on the type, for external users we have to emit | ||||
736 | // a GEP or a GEP + ptrtoint. | ||||
737 | setInsertionPoint(Builder, Val, false); | ||||
738 | |||||
739 | // If required, create an inttoptr instruction for Base. | ||||
740 | Value *NewBase = Base; | ||||
741 | if (!Base->getType()->isPointerTy()) | ||||
742 | NewBase = Builder.CreateBitOrPointerCast(Base, Start->getType(), | ||||
743 | Start->getName() + "to.ptr"); | ||||
744 | |||||
745 | Value *GEP = Builder.CreateInBoundsGEP( | ||||
746 | Start->getType()->getPointerElementType(), NewBase, | ||||
747 | makeArrayRef(NewInsts[Val]), Val->getName() + ".ptr"); | ||||
748 | |||||
749 | if (!Val->getType()->isPointerTy()) { | ||||
750 | Value *Cast = Builder.CreatePointerCast(GEP, Val->getType(), | ||||
751 | Val->getName() + ".conv"); | ||||
752 | GEP = Cast; | ||||
753 | } | ||||
754 | Val->replaceAllUsesWith(GEP); | ||||
755 | } | ||||
756 | |||||
757 | return NewInsts[Start]; | ||||
758 | } | ||||
759 | |||||
760 | /// Looks through GEPs, IntToPtrInsts and PtrToIntInsts in order to express | ||||
761 | /// the input Value as a constant indexed GEP. Returns a pair containing | ||||
762 | /// the GEPs Pointer and Index. | ||||
763 | static std::pair<Value *, Value *> | ||||
764 | getAsConstantIndexedAddress(Value *V, const DataLayout &DL) { | ||||
765 | Type *IndexType = IntegerType::get(V->getContext(), | ||||
766 | DL.getIndexTypeSizeInBits(V->getType())); | ||||
767 | |||||
768 | Constant *Index = ConstantInt::getNullValue(IndexType); | ||||
769 | while (true) { | ||||
770 | if (GEPOperator *GEP = dyn_cast<GEPOperator>(V)) { | ||||
771 | // We accept only inbouds GEPs here to exclude the possibility of | ||||
772 | // overflow. | ||||
773 | if (!GEP->isInBounds()) | ||||
774 | break; | ||||
775 | if (GEP->hasAllConstantIndices() && GEP->getNumIndices() == 1 && | ||||
776 | GEP->getType() == V->getType()) { | ||||
777 | V = GEP->getOperand(0); | ||||
778 | Constant *GEPIndex = static_cast<Constant *>(GEP->getOperand(1)); | ||||
779 | Index = ConstantExpr::getAdd( | ||||
780 | Index, ConstantExpr::getSExtOrBitCast(GEPIndex, IndexType)); | ||||
781 | continue; | ||||
782 | } | ||||
783 | break; | ||||
784 | } | ||||
785 | if (auto *CI = dyn_cast<IntToPtrInst>(V)) { | ||||
786 | if (!CI->isNoopCast(DL)) | ||||
787 | break; | ||||
788 | V = CI->getOperand(0); | ||||
789 | continue; | ||||
790 | } | ||||
791 | if (auto *CI = dyn_cast<PtrToIntInst>(V)) { | ||||
792 | if (!CI->isNoopCast(DL)) | ||||
793 | break; | ||||
794 | V = CI->getOperand(0); | ||||
795 | continue; | ||||
796 | } | ||||
797 | break; | ||||
798 | } | ||||
799 | return {V, Index}; | ||||
800 | } | ||||
801 | |||||
802 | /// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant. | ||||
803 | /// We can look through PHIs, GEPs and casts in order to determine a common base | ||||
804 | /// between GEPLHS and RHS. | ||||
805 | static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS, | ||||
806 | ICmpInst::Predicate Cond, | ||||
807 | const DataLayout &DL) { | ||||
808 | // FIXME: Support vector of pointers. | ||||
809 | if (GEPLHS->getType()->isVectorTy()) | ||||
810 | return nullptr; | ||||
811 | |||||
812 | if (!GEPLHS->hasAllConstantIndices()) | ||||
813 | return nullptr; | ||||
814 | |||||
815 | // Make sure the pointers have the same type. | ||||
816 | if (GEPLHS->getType() != RHS->getType()) | ||||
817 | return nullptr; | ||||
818 | |||||
819 | Value *PtrBase, *Index; | ||||
820 | std::tie(PtrBase, Index) = getAsConstantIndexedAddress(GEPLHS, DL); | ||||
821 | |||||
822 | // The set of nodes that will take part in this transformation. | ||||
823 | SetVector<Value *> Nodes; | ||||
824 | |||||
825 | if (!canRewriteGEPAsOffset(RHS, PtrBase, DL, Nodes)) | ||||
826 | return nullptr; | ||||
827 | |||||
828 | // We know we can re-write this as | ||||
829 | // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) | ||||
830 | // Since we've only looked through inbouds GEPs we know that we | ||||
831 | // can't have overflow on either side. We can therefore re-write | ||||
832 | // this as: | ||||
833 | // OFFSET1 cmp OFFSET2 | ||||
834 | Value *NewRHS = rewriteGEPAsOffset(RHS, PtrBase, DL, Nodes); | ||||
835 | |||||
836 | // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written | ||||
837 | // GEP having PtrBase as the pointer base, and has returned in NewRHS the | ||||
838 | // offset. Since Index is the offset of LHS to the base pointer, we will now | ||||
839 | // compare the offsets instead of comparing the pointers. | ||||
840 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Index, NewRHS); | ||||
841 | } | ||||
842 | |||||
843 | /// Fold comparisons between a GEP instruction and something else. At this point | ||||
844 | /// we know that the GEP is on the LHS of the comparison. | ||||
845 | Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, | ||||
846 | ICmpInst::Predicate Cond, | ||||
847 | Instruction &I) { | ||||
848 | // Don't transform signed compares of GEPs into index compares. Even if the | ||||
849 | // GEP is inbounds, the final add of the base pointer can have signed overflow | ||||
850 | // and would change the result of the icmp. | ||||
851 | // e.g. "&foo[0] <s &foo[1]" can't be folded to "true" because "foo" could be | ||||
852 | // the maximum signed value for the pointer type. | ||||
853 | if (ICmpInst::isSigned(Cond)) | ||||
854 | return nullptr; | ||||
855 | |||||
856 | // Look through bitcasts and addrspacecasts. We do not however want to remove | ||||
857 | // 0 GEPs. | ||||
858 | if (!isa<GetElementPtrInst>(RHS)) | ||||
859 | RHS = RHS->stripPointerCasts(); | ||||
860 | |||||
861 | Value *PtrBase = GEPLHS->getOperand(0); | ||||
862 | // FIXME: Support vector pointer GEPs. | ||||
863 | if (PtrBase == RHS && GEPLHS->isInBounds() && | ||||
864 | !GEPLHS->getType()->isVectorTy()) { | ||||
865 | // ((gep Ptr, OFFSET) cmp Ptr) ---> (OFFSET cmp 0). | ||||
866 | // This transformation (ignoring the base and scales) is valid because we | ||||
867 | // know pointers can't overflow since the gep is inbounds. See if we can | ||||
868 | // output an optimized form. | ||||
869 | Value *Offset = evaluateGEPOffsetExpression(GEPLHS, *this, DL); | ||||
870 | |||||
871 | // If not, synthesize the offset the hard way. | ||||
872 | if (!Offset) | ||||
873 | Offset = EmitGEPOffset(GEPLHS); | ||||
874 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Offset, | ||||
875 | Constant::getNullValue(Offset->getType())); | ||||
876 | } | ||||
877 | |||||
878 | if (GEPLHS->isInBounds() && ICmpInst::isEquality(Cond) && | ||||
879 | isa<Constant>(RHS) && cast<Constant>(RHS)->isNullValue() && | ||||
880 | !NullPointerIsDefined(I.getFunction(), | ||||
881 | RHS->getType()->getPointerAddressSpace())) { | ||||
882 | // For most address spaces, an allocation can't be placed at null, but null | ||||
883 | // itself is treated as a 0 size allocation in the in bounds rules. Thus, | ||||
884 | // the only valid inbounds address derived from null, is null itself. | ||||
885 | // Thus, we have four cases to consider: | ||||
886 | // 1) Base == nullptr, Offset == 0 -> inbounds, null | ||||
887 | // 2) Base == nullptr, Offset != 0 -> poison as the result is out of bounds | ||||
888 | // 3) Base != nullptr, Offset == (-base) -> poison (crossing allocations) | ||||
889 | // 4) Base != nullptr, Offset != (-base) -> nonnull (and possibly poison) | ||||
890 | // | ||||
891 | // (Note if we're indexing a type of size 0, that simply collapses into one | ||||
892 | // of the buckets above.) | ||||
893 | // | ||||
894 | // In general, we're allowed to make values less poison (i.e. remove | ||||
895 | // sources of full UB), so in this case, we just select between the two | ||||
896 | // non-poison cases (1 and 4 above). | ||||
897 | // | ||||
898 | // For vectors, we apply the same reasoning on a per-lane basis. | ||||
899 | auto *Base = GEPLHS->getPointerOperand(); | ||||
900 | if (GEPLHS->getType()->isVectorTy() && Base->getType()->isPointerTy()) { | ||||
901 | int NumElts = cast<FixedVectorType>(GEPLHS->getType())->getNumElements(); | ||||
902 | Base = Builder.CreateVectorSplat(NumElts, Base); | ||||
903 | } | ||||
904 | return new ICmpInst(Cond, Base, | ||||
905 | ConstantExpr::getPointerBitCastOrAddrSpaceCast( | ||||
906 | cast<Constant>(RHS), Base->getType())); | ||||
907 | } else if (GEPOperator *GEPRHS
| ||||
908 | // If the base pointers are different, but the indices are the same, just | ||||
909 | // compare the base pointer. | ||||
910 | if (PtrBase != GEPRHS->getOperand(0)) { | ||||
911 | bool IndicesTheSame = GEPLHS->getNumOperands()==GEPRHS->getNumOperands(); | ||||
912 | IndicesTheSame &= GEPLHS->getOperand(0)->getType() == | ||||
913 | GEPRHS->getOperand(0)->getType(); | ||||
914 | if (IndicesTheSame) | ||||
915 | for (unsigned i = 1, e = GEPLHS->getNumOperands(); i != e; ++i) | ||||
916 | if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) { | ||||
917 | IndicesTheSame = false; | ||||
918 | break; | ||||
919 | } | ||||
920 | |||||
921 | // If all indices are the same, just compare the base pointers. | ||||
922 | Type *BaseType = GEPLHS->getOperand(0)->getType(); | ||||
923 | if (IndicesTheSame && CmpInst::makeCmpResultType(BaseType) == I.getType()) | ||||
924 | return new ICmpInst(Cond, GEPLHS->getOperand(0), GEPRHS->getOperand(0)); | ||||
925 | |||||
926 | // If we're comparing GEPs with two base pointers that only differ in type | ||||
927 | // and both GEPs have only constant indices or just one use, then fold | ||||
928 | // the compare with the adjusted indices. | ||||
929 | // FIXME: Support vector of pointers. | ||||
930 | if (GEPLHS->isInBounds() && GEPRHS->isInBounds() && | ||||
931 | (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) && | ||||
932 | (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse()) && | ||||
933 | PtrBase->stripPointerCasts() == | ||||
934 | GEPRHS->getOperand(0)->stripPointerCasts() && | ||||
935 | !GEPLHS->getType()->isVectorTy()) { | ||||
936 | Value *LOffset = EmitGEPOffset(GEPLHS); | ||||
937 | Value *ROffset = EmitGEPOffset(GEPRHS); | ||||
938 | |||||
939 | // If we looked through an addrspacecast between different sized address | ||||
940 | // spaces, the LHS and RHS pointers are different sized | ||||
941 | // integers. Truncate to the smaller one. | ||||
942 | Type *LHSIndexTy = LOffset->getType(); | ||||
943 | Type *RHSIndexTy = ROffset->getType(); | ||||
944 | if (LHSIndexTy != RHSIndexTy) { | ||||
945 | if (LHSIndexTy->getPrimitiveSizeInBits() < | ||||
946 | RHSIndexTy->getPrimitiveSizeInBits()) { | ||||
947 | ROffset = Builder.CreateTrunc(ROffset, LHSIndexTy); | ||||
948 | } else | ||||
949 | LOffset = Builder.CreateTrunc(LOffset, RHSIndexTy); | ||||
950 | } | ||||
951 | |||||
952 | Value *Cmp = Builder.CreateICmp(ICmpInst::getSignedPredicate(Cond), | ||||
953 | LOffset, ROffset); | ||||
954 | return replaceInstUsesWith(I, Cmp); | ||||
955 | } | ||||
956 | |||||
957 | // Otherwise, the base pointers are different and the indices are | ||||
958 | // different. Try convert this to an indexed compare by looking through | ||||
959 | // PHIs/casts. | ||||
960 | return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); | ||||
961 | } | ||||
962 | |||||
963 | // If one of the GEPs has all zero indices, recurse. | ||||
964 | // FIXME: Handle vector of pointers. | ||||
965 | if (!GEPLHS->getType()->isVectorTy() && GEPLHS->hasAllZeroIndices()) | ||||
966 | return foldGEPICmp(GEPRHS, GEPLHS->getOperand(0), | ||||
967 | ICmpInst::getSwappedPredicate(Cond), I); | ||||
968 | |||||
969 | // If the other GEP has all zero indices, recurse. | ||||
970 | // FIXME: Handle vector of pointers. | ||||
971 | if (!GEPRHS->getType()->isVectorTy() && GEPRHS->hasAllZeroIndices()) | ||||
972 | return foldGEPICmp(GEPLHS, GEPRHS->getOperand(0), Cond, I); | ||||
973 | |||||
974 | bool GEPsInBounds = GEPLHS->isInBounds() && GEPRHS->isInBounds(); | ||||
975 | if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands()) { | ||||
976 | // If the GEPs only differ by one index, compare it. | ||||
977 | unsigned NumDifferences = 0; // Keep track of # differences. | ||||
978 | unsigned DiffOperand = 0; // The operand that differs. | ||||
979 | for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i) | ||||
980 | if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) { | ||||
981 | Type *LHSType = GEPLHS->getOperand(i)->getType(); | ||||
982 | Type *RHSType = GEPRHS->getOperand(i)->getType(); | ||||
983 | // FIXME: Better support for vector of pointers. | ||||
984 | if (LHSType->getPrimitiveSizeInBits() != | ||||
985 | RHSType->getPrimitiveSizeInBits() || | ||||
986 | (GEPLHS->getType()->isVectorTy() && | ||||
987 | (!LHSType->isVectorTy() || !RHSType->isVectorTy()))) { | ||||
988 | // Irreconcilable differences. | ||||
989 | NumDifferences = 2; | ||||
990 | break; | ||||
991 | } | ||||
992 | |||||
993 | if (NumDifferences++) break; | ||||
994 | DiffOperand = i; | ||||
995 | } | ||||
996 | |||||
997 | if (NumDifferences == 0) // SAME GEP? | ||||
998 | return replaceInstUsesWith(I, // No comparison is needed here. | ||||
999 | ConstantInt::get(I.getType(), ICmpInst::isTrueWhenEqual(Cond))); | ||||
1000 | |||||
1001 | else if (NumDifferences == 1 && GEPsInBounds) { | ||||
1002 | Value *LHSV = GEPLHS->getOperand(DiffOperand); | ||||
1003 | Value *RHSV = GEPRHS->getOperand(DiffOperand); | ||||
1004 | // Make sure we do a signed comparison here. | ||||
1005 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), LHSV, RHSV); | ||||
1006 | } | ||||
1007 | } | ||||
1008 | |||||
1009 | // Only lower this if the icmp is the only user of the GEP or if we expect | ||||
1010 | // the result to fold to a constant! | ||||
1011 | if (GEPsInBounds && (isa<ConstantExpr>(GEPLHS) || GEPLHS->hasOneUse()) && | ||||
1012 | (isa<ConstantExpr>(GEPRHS) || GEPRHS->hasOneUse())) { | ||||
1013 | // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2) | ||||
1014 | Value *L = EmitGEPOffset(GEPLHS); | ||||
1015 | Value *R = EmitGEPOffset(GEPRHS); | ||||
1016 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), L, R); | ||||
1017 | } | ||||
1018 | } | ||||
1019 | |||||
1020 | // Try convert this to an indexed compare by looking through PHIs/casts as a | ||||
1021 | // last resort. | ||||
1022 | return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); | ||||
1023 | } | ||||
1024 | |||||
1025 | Instruction *InstCombinerImpl::foldAllocaCmp(ICmpInst &ICI, | ||||
1026 | const AllocaInst *Alloca, | ||||
1027 | const Value *Other) { | ||||
1028 | assert(ICI.isEquality() && "Cannot fold non-equality comparison.")((ICI.isEquality() && "Cannot fold non-equality comparison." ) ? static_cast<void> (0) : __assert_fail ("ICI.isEquality() && \"Cannot fold non-equality comparison.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1028, __PRETTY_FUNCTION__)); | ||||
1029 | |||||
1030 | // It would be tempting to fold away comparisons between allocas and any | ||||
1031 | // pointer not based on that alloca (e.g. an argument). However, even | ||||
1032 | // though such pointers cannot alias, they can still compare equal. | ||||
1033 | // | ||||
1034 | // But LLVM doesn't specify where allocas get their memory, so if the alloca | ||||
1035 | // doesn't escape we can argue that it's impossible to guess its value, and we | ||||
1036 | // can therefore act as if any such guesses are wrong. | ||||
1037 | // | ||||
1038 | // The code below checks that the alloca doesn't escape, and that it's only | ||||
1039 | // used in a comparison once (the current instruction). The | ||||
1040 | // single-comparison-use condition ensures that we're trivially folding all | ||||
1041 | // comparisons against the alloca consistently, and avoids the risk of | ||||
1042 | // erroneously folding a comparison of the pointer with itself. | ||||
1043 | |||||
1044 | unsigned MaxIter = 32; // Break cycles and bound to constant-time. | ||||
1045 | |||||
1046 | SmallVector<const Use *, 32> Worklist; | ||||
1047 | for (const Use &U : Alloca->uses()) { | ||||
1048 | if (Worklist.size() >= MaxIter) | ||||
1049 | return nullptr; | ||||
1050 | Worklist.push_back(&U); | ||||
1051 | } | ||||
1052 | |||||
1053 | unsigned NumCmps = 0; | ||||
1054 | while (!Worklist.empty()) { | ||||
1055 | assert(Worklist.size() <= MaxIter)((Worklist.size() <= MaxIter) ? static_cast<void> (0 ) : __assert_fail ("Worklist.size() <= MaxIter", "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1055, __PRETTY_FUNCTION__)); | ||||
1056 | const Use *U = Worklist.pop_back_val(); | ||||
1057 | const Value *V = U->getUser(); | ||||
1058 | --MaxIter; | ||||
1059 | |||||
1060 | if (isa<BitCastInst>(V) || isa<GetElementPtrInst>(V) || isa<PHINode>(V) || | ||||
1061 | isa<SelectInst>(V)) { | ||||
1062 | // Track the uses. | ||||
1063 | } else if (isa<LoadInst>(V)) { | ||||
1064 | // Loading from the pointer doesn't escape it. | ||||
1065 | continue; | ||||
1066 | } else if (const auto *SI = dyn_cast<StoreInst>(V)) { | ||||
1067 | // Storing *to* the pointer is fine, but storing the pointer escapes it. | ||||
1068 | if (SI->getValueOperand() == U->get()) | ||||
1069 | return nullptr; | ||||
1070 | continue; | ||||
1071 | } else if (isa<ICmpInst>(V)) { | ||||
1072 | if (NumCmps++) | ||||
1073 | return nullptr; // Found more than one cmp. | ||||
1074 | continue; | ||||
1075 | } else if (const auto *Intrin = dyn_cast<IntrinsicInst>(V)) { | ||||
1076 | switch (Intrin->getIntrinsicID()) { | ||||
1077 | // These intrinsics don't escape or compare the pointer. Memset is safe | ||||
1078 | // because we don't allow ptrtoint. Memcpy and memmove are safe because | ||||
1079 | // we don't allow stores, so src cannot point to V. | ||||
1080 | case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: | ||||
1081 | case Intrinsic::memcpy: case Intrinsic::memmove: case Intrinsic::memset: | ||||
1082 | continue; | ||||
1083 | default: | ||||
1084 | return nullptr; | ||||
1085 | } | ||||
1086 | } else { | ||||
1087 | return nullptr; | ||||
1088 | } | ||||
1089 | for (const Use &U : V->uses()) { | ||||
1090 | if (Worklist.size() >= MaxIter) | ||||
1091 | return nullptr; | ||||
1092 | Worklist.push_back(&U); | ||||
1093 | } | ||||
1094 | } | ||||
1095 | |||||
1096 | Type *CmpTy = CmpInst::makeCmpResultType(Other->getType()); | ||||
1097 | return replaceInstUsesWith( | ||||
1098 | ICI, | ||||
1099 | ConstantInt::get(CmpTy, !CmpInst::isTrueWhenEqual(ICI.getPredicate()))); | ||||
1100 | } | ||||
1101 | |||||
1102 | /// Fold "icmp pred (X+C), X". | ||||
1103 | Instruction *InstCombinerImpl::foldICmpAddOpConst(Value *X, const APInt &C, | ||||
1104 | ICmpInst::Predicate Pred) { | ||||
1105 | // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0, | ||||
1106 | // so the values can never be equal. Similarly for all other "or equals" | ||||
1107 | // operators. | ||||
1108 | assert(!!C && "C should not be zero!")((!!C && "C should not be zero!") ? static_cast<void > (0) : __assert_fail ("!!C && \"C should not be zero!\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1108, __PRETTY_FUNCTION__)); | ||||
1109 | |||||
1110 | // (X+1) <u X --> X >u (MAXUINT-1) --> X == 255 | ||||
1111 | // (X+2) <u X --> X >u (MAXUINT-2) --> X > 253 | ||||
1112 | // (X+MAXUINT) <u X --> X >u (MAXUINT-MAXUINT) --> X != 0 | ||||
1113 | if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) { | ||||
1114 | Constant *R = ConstantInt::get(X->getType(), | ||||
1115 | APInt::getMaxValue(C.getBitWidth()) - C); | ||||
1116 | return new ICmpInst(ICmpInst::ICMP_UGT, X, R); | ||||
1117 | } | ||||
1118 | |||||
1119 | // (X+1) >u X --> X <u (0-1) --> X != 255 | ||||
1120 | // (X+2) >u X --> X <u (0-2) --> X <u 254 | ||||
1121 | // (X+MAXUINT) >u X --> X <u (0-MAXUINT) --> X <u 1 --> X == 0 | ||||
1122 | if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) | ||||
1123 | return new ICmpInst(ICmpInst::ICMP_ULT, X, | ||||
1124 | ConstantInt::get(X->getType(), -C)); | ||||
1125 | |||||
1126 | APInt SMax = APInt::getSignedMaxValue(C.getBitWidth()); | ||||
1127 | |||||
1128 | // (X+ 1) <s X --> X >s (MAXSINT-1) --> X == 127 | ||||
1129 | // (X+ 2) <s X --> X >s (MAXSINT-2) --> X >s 125 | ||||
1130 | // (X+MAXSINT) <s X --> X >s (MAXSINT-MAXSINT) --> X >s 0 | ||||
1131 | // (X+MINSINT) <s X --> X >s (MAXSINT-MINSINT) --> X >s -1 | ||||
1132 | // (X+ -2) <s X --> X >s (MAXSINT- -2) --> X >s 126 | ||||
1133 | // (X+ -1) <s X --> X >s (MAXSINT- -1) --> X != 127 | ||||
1134 | if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) | ||||
1135 | return new ICmpInst(ICmpInst::ICMP_SGT, X, | ||||
1136 | ConstantInt::get(X->getType(), SMax - C)); | ||||
1137 | |||||
1138 | // (X+ 1) >s X --> X <s (MAXSINT-(1-1)) --> X != 127 | ||||
1139 | // (X+ 2) >s X --> X <s (MAXSINT-(2-1)) --> X <s 126 | ||||
1140 | // (X+MAXSINT) >s X --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1 | ||||
1141 | // (X+MINSINT) >s X --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2 | ||||
1142 | // (X+ -2) >s X --> X <s (MAXSINT-(-2-1)) --> X <s -126 | ||||
1143 | // (X+ -1) >s X --> X <s (MAXSINT-(-1-1)) --> X == -128 | ||||
1144 | |||||
1145 | assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE)((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) ? static_cast<void> (0) : __assert_fail ("Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1145, __PRETTY_FUNCTION__)); | ||||
1146 | return new ICmpInst(ICmpInst::ICMP_SLT, X, | ||||
1147 | ConstantInt::get(X->getType(), SMax - (C - 1))); | ||||
1148 | } | ||||
1149 | |||||
1150 | /// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" -> | ||||
1151 | /// (icmp eq/ne A, Log2(AP2/AP1)) -> | ||||
1152 | /// (icmp eq/ne A, Log2(AP2) - Log2(AP1)). | ||||
1153 | Instruction *InstCombinerImpl::foldICmpShrConstConst(ICmpInst &I, Value *A, | ||||
1154 | const APInt &AP1, | ||||
1155 | const APInt &AP2) { | ||||
1156 | assert(I.isEquality() && "Cannot fold icmp gt/lt")((I.isEquality() && "Cannot fold icmp gt/lt") ? static_cast <void> (0) : __assert_fail ("I.isEquality() && \"Cannot fold icmp gt/lt\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1156, __PRETTY_FUNCTION__)); | ||||
1157 | |||||
1158 | auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { | ||||
1159 | if (I.getPredicate() == I.ICMP_NE) | ||||
1160 | Pred = CmpInst::getInversePredicate(Pred); | ||||
1161 | return new ICmpInst(Pred, LHS, RHS); | ||||
1162 | }; | ||||
1163 | |||||
1164 | // Don't bother doing any work for cases which InstSimplify handles. | ||||
1165 | if (AP2.isNullValue()) | ||||
1166 | return nullptr; | ||||
1167 | |||||
1168 | bool IsAShr = isa<AShrOperator>(I.getOperand(0)); | ||||
1169 | if (IsAShr) { | ||||
1170 | if (AP2.isAllOnesValue()) | ||||
1171 | return nullptr; | ||||
1172 | if (AP2.isNegative() != AP1.isNegative()) | ||||
1173 | return nullptr; | ||||
1174 | if (AP2.sgt(AP1)) | ||||
1175 | return nullptr; | ||||
1176 | } | ||||
1177 | |||||
1178 | if (!AP1) | ||||
1179 | // 'A' must be large enough to shift out the highest set bit. | ||||
1180 | return getICmp(I.ICMP_UGT, A, | ||||
1181 | ConstantInt::get(A->getType(), AP2.logBase2())); | ||||
1182 | |||||
1183 | if (AP1 == AP2) | ||||
1184 | return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType())); | ||||
1185 | |||||
1186 | int Shift; | ||||
1187 | if (IsAShr && AP1.isNegative()) | ||||
1188 | Shift = AP1.countLeadingOnes() - AP2.countLeadingOnes(); | ||||
1189 | else | ||||
1190 | Shift = AP1.countLeadingZeros() - AP2.countLeadingZeros(); | ||||
1191 | |||||
1192 | if (Shift > 0) { | ||||
1193 | if (IsAShr && AP1 == AP2.ashr(Shift)) { | ||||
1194 | // There are multiple solutions if we are comparing against -1 and the LHS | ||||
1195 | // of the ashr is not a power of two. | ||||
1196 | if (AP1.isAllOnesValue() && !AP2.isPowerOf2()) | ||||
1197 | return getICmp(I.ICMP_UGE, A, ConstantInt::get(A->getType(), Shift)); | ||||
1198 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); | ||||
1199 | } else if (AP1 == AP2.lshr(Shift)) { | ||||
1200 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); | ||||
1201 | } | ||||
1202 | } | ||||
1203 | |||||
1204 | // Shifting const2 will never be equal to const1. | ||||
1205 | // FIXME: This should always be handled by InstSimplify? | ||||
1206 | auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE); | ||||
1207 | return replaceInstUsesWith(I, TorF); | ||||
1208 | } | ||||
1209 | |||||
1210 | /// Handle "(icmp eq/ne (shl AP2, A), AP1)" -> | ||||
1211 | /// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)). | ||||
1212 | Instruction *InstCombinerImpl::foldICmpShlConstConst(ICmpInst &I, Value *A, | ||||
1213 | const APInt &AP1, | ||||
1214 | const APInt &AP2) { | ||||
1215 | assert(I.isEquality() && "Cannot fold icmp gt/lt")((I.isEquality() && "Cannot fold icmp gt/lt") ? static_cast <void> (0) : __assert_fail ("I.isEquality() && \"Cannot fold icmp gt/lt\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1215, __PRETTY_FUNCTION__)); | ||||
1216 | |||||
1217 | auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { | ||||
1218 | if (I.getPredicate() == I.ICMP_NE) | ||||
1219 | Pred = CmpInst::getInversePredicate(Pred); | ||||
1220 | return new ICmpInst(Pred, LHS, RHS); | ||||
1221 | }; | ||||
1222 | |||||
1223 | // Don't bother doing any work for cases which InstSimplify handles. | ||||
1224 | if (AP2.isNullValue()) | ||||
1225 | return nullptr; | ||||
1226 | |||||
1227 | unsigned AP2TrailingZeros = AP2.countTrailingZeros(); | ||||
1228 | |||||
1229 | if (!AP1 && AP2TrailingZeros != 0) | ||||
1230 | return getICmp( | ||||
1231 | I.ICMP_UGE, A, | ||||
1232 | ConstantInt::get(A->getType(), AP2.getBitWidth() - AP2TrailingZeros)); | ||||
1233 | |||||
1234 | if (AP1 == AP2) | ||||
1235 | return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType())); | ||||
1236 | |||||
1237 | // Get the distance between the lowest bits that are set. | ||||
1238 | int Shift = AP1.countTrailingZeros() - AP2TrailingZeros; | ||||
1239 | |||||
1240 | if (Shift > 0 && AP2.shl(Shift) == AP1) | ||||
1241 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); | ||||
1242 | |||||
1243 | // Shifting const2 will never be equal to const1. | ||||
1244 | // FIXME: This should always be handled by InstSimplify? | ||||
1245 | auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE); | ||||
1246 | return replaceInstUsesWith(I, TorF); | ||||
1247 | } | ||||
1248 | |||||
1249 | /// The caller has matched a pattern of the form: | ||||
1250 | /// I = icmp ugt (add (add A, B), CI2), CI1 | ||||
1251 | /// If this is of the form: | ||||
1252 | /// sum = a + b | ||||
1253 | /// if (sum+128 >u 255) | ||||
1254 | /// Then replace it with llvm.sadd.with.overflow.i8. | ||||
1255 | /// | ||||
1256 | static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B, | ||||
1257 | ConstantInt *CI2, ConstantInt *CI1, | ||||
1258 | InstCombinerImpl &IC) { | ||||
1259 | // The transformation we're trying to do here is to transform this into an | ||||
1260 | // llvm.sadd.with.overflow. To do this, we have to replace the original add | ||||
1261 | // with a narrower add, and discard the add-with-constant that is part of the | ||||
1262 | // range check (if we can't eliminate it, this isn't profitable). | ||||
1263 | |||||
1264 | // In order to eliminate the add-with-constant, the compare can be its only | ||||
1265 | // use. | ||||
1266 | Instruction *AddWithCst = cast<Instruction>(I.getOperand(0)); | ||||
1267 | if (!AddWithCst->hasOneUse()) | ||||
1268 | return nullptr; | ||||
1269 | |||||
1270 | // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow. | ||||
1271 | if (!CI2->getValue().isPowerOf2()) | ||||
1272 | return nullptr; | ||||
1273 | unsigned NewWidth = CI2->getValue().countTrailingZeros(); | ||||
1274 | if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31) | ||||
1275 | return nullptr; | ||||
1276 | |||||
1277 | // The width of the new add formed is 1 more than the bias. | ||||
1278 | ++NewWidth; | ||||
1279 | |||||
1280 | // Check to see that CI1 is an all-ones value with NewWidth bits. | ||||
1281 | if (CI1->getBitWidth() == NewWidth || | ||||
1282 | CI1->getValue() != APInt::getLowBitsSet(CI1->getBitWidth(), NewWidth)) | ||||
1283 | return nullptr; | ||||
1284 | |||||
1285 | // This is only really a signed overflow check if the inputs have been | ||||
1286 | // sign-extended; check for that condition. For example, if CI2 is 2^31 and | ||||
1287 | // the operands of the add are 64 bits wide, we need at least 33 sign bits. | ||||
1288 | unsigned NeededSignBits = CI1->getBitWidth() - NewWidth + 1; | ||||
1289 | if (IC.ComputeNumSignBits(A, 0, &I) < NeededSignBits || | ||||
1290 | IC.ComputeNumSignBits(B, 0, &I) < NeededSignBits) | ||||
1291 | return nullptr; | ||||
1292 | |||||
1293 | // In order to replace the original add with a narrower | ||||
1294 | // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant | ||||
1295 | // and truncates that discard the high bits of the add. Verify that this is | ||||
1296 | // the case. | ||||
1297 | Instruction *OrigAdd = cast<Instruction>(AddWithCst->getOperand(0)); | ||||
1298 | for (User *U : OrigAdd->users()) { | ||||
1299 | if (U == AddWithCst) | ||||
1300 | continue; | ||||
1301 | |||||
1302 | // Only accept truncates for now. We would really like a nice recursive | ||||
1303 | // predicate like SimplifyDemandedBits, but which goes downwards the use-def | ||||
1304 | // chain to see which bits of a value are actually demanded. If the | ||||
1305 | // original add had another add which was then immediately truncated, we | ||||
1306 | // could still do the transformation. | ||||
1307 | TruncInst *TI = dyn_cast<TruncInst>(U); | ||||
1308 | if (!TI || TI->getType()->getPrimitiveSizeInBits() > NewWidth) | ||||
1309 | return nullptr; | ||||
1310 | } | ||||
1311 | |||||
1312 | // If the pattern matches, truncate the inputs to the narrower type and | ||||
1313 | // use the sadd_with_overflow intrinsic to efficiently compute both the | ||||
1314 | // result and the overflow bit. | ||||
1315 | Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth); | ||||
1316 | Function *F = Intrinsic::getDeclaration( | ||||
1317 | I.getModule(), Intrinsic::sadd_with_overflow, NewType); | ||||
1318 | |||||
1319 | InstCombiner::BuilderTy &Builder = IC.Builder; | ||||
1320 | |||||
1321 | // Put the new code above the original add, in case there are any uses of the | ||||
1322 | // add between the add and the compare. | ||||
1323 | Builder.SetInsertPoint(OrigAdd); | ||||
1324 | |||||
1325 | Value *TruncA = Builder.CreateTrunc(A, NewType, A->getName() + ".trunc"); | ||||
1326 | Value *TruncB = Builder.CreateTrunc(B, NewType, B->getName() + ".trunc"); | ||||
1327 | CallInst *Call = Builder.CreateCall(F, {TruncA, TruncB}, "sadd"); | ||||
1328 | Value *Add = Builder.CreateExtractValue(Call, 0, "sadd.result"); | ||||
1329 | Value *ZExt = Builder.CreateZExt(Add, OrigAdd->getType()); | ||||
1330 | |||||
1331 | // The inner add was the result of the narrow add, zero extended to the | ||||
1332 | // wider type. Replace it with the result computed by the intrinsic. | ||||
1333 | IC.replaceInstUsesWith(*OrigAdd, ZExt); | ||||
1334 | IC.eraseInstFromFunction(*OrigAdd); | ||||
1335 | |||||
1336 | // The original icmp gets replaced with the overflow value. | ||||
1337 | return ExtractValueInst::Create(Call, 1, "sadd.overflow"); | ||||
1338 | } | ||||
1339 | |||||
1340 | /// If we have: | ||||
1341 | /// icmp eq/ne (urem/srem %x, %y), 0 | ||||
1342 | /// iff %y is a power-of-two, we can replace this with a bit test: | ||||
1343 | /// icmp eq/ne (and %x, (add %y, -1)), 0 | ||||
1344 | Instruction *InstCombinerImpl::foldIRemByPowerOfTwoToBitTest(ICmpInst &I) { | ||||
1345 | // This fold is only valid for equality predicates. | ||||
1346 | if (!I.isEquality()) | ||||
1347 | return nullptr; | ||||
1348 | ICmpInst::Predicate Pred; | ||||
1349 | Value *X, *Y, *Zero; | ||||
1350 | if (!match(&I, m_ICmp(Pred, m_OneUse(m_IRem(m_Value(X), m_Value(Y))), | ||||
1351 | m_CombineAnd(m_Zero(), m_Value(Zero))))) | ||||
1352 | return nullptr; | ||||
1353 | if (!isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, 0, &I)) | ||||
1354 | return nullptr; | ||||
1355 | // This may increase instruction count, we don't enforce that Y is a constant. | ||||
1356 | Value *Mask = Builder.CreateAdd(Y, Constant::getAllOnesValue(Y->getType())); | ||||
1357 | Value *Masked = Builder.CreateAnd(X, Mask); | ||||
1358 | return ICmpInst::Create(Instruction::ICmp, Pred, Masked, Zero); | ||||
1359 | } | ||||
1360 | |||||
1361 | /// Fold equality-comparison between zero and any (maybe truncated) right-shift | ||||
1362 | /// by one-less-than-bitwidth into a sign test on the original value. | ||||
1363 | Instruction *InstCombinerImpl::foldSignBitTest(ICmpInst &I) { | ||||
1364 | Instruction *Val; | ||||
1365 | ICmpInst::Predicate Pred; | ||||
1366 | if (!I.isEquality() || !match(&I, m_ICmp(Pred, m_Instruction(Val), m_Zero()))) | ||||
1367 | return nullptr; | ||||
1368 | |||||
1369 | Value *X; | ||||
1370 | Type *XTy; | ||||
1371 | |||||
1372 | Constant *C; | ||||
1373 | if (match(Val, m_TruncOrSelf(m_Shr(m_Value(X), m_Constant(C))))) { | ||||
1374 | XTy = X->getType(); | ||||
1375 | unsigned XBitWidth = XTy->getScalarSizeInBits(); | ||||
1376 | if (!match(C, m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_EQ, | ||||
1377 | APInt(XBitWidth, XBitWidth - 1)))) | ||||
1378 | return nullptr; | ||||
1379 | } else if (isa<BinaryOperator>(Val) && | ||||
1380 | (X = reassociateShiftAmtsOfTwoSameDirectionShifts( | ||||
1381 | cast<BinaryOperator>(Val), SQ.getWithInstruction(Val), | ||||
1382 | /*AnalyzeForSignBitExtraction=*/true))) { | ||||
1383 | XTy = X->getType(); | ||||
1384 | } else | ||||
1385 | return nullptr; | ||||
1386 | |||||
1387 | return ICmpInst::Create(Instruction::ICmp, | ||||
1388 | Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_SGE | ||||
1389 | : ICmpInst::ICMP_SLT, | ||||
1390 | X, ConstantInt::getNullValue(XTy)); | ||||
1391 | } | ||||
1392 | |||||
1393 | // Handle icmp pred X, 0 | ||||
1394 | Instruction *InstCombinerImpl::foldICmpWithZero(ICmpInst &Cmp) { | ||||
1395 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
1396 | if (!match(Cmp.getOperand(1), m_Zero())) | ||||
1397 | return nullptr; | ||||
1398 | |||||
1399 | // (icmp sgt smin(PosA, B) 0) -> (icmp sgt B 0) | ||||
1400 | if (Pred == ICmpInst::ICMP_SGT) { | ||||
1401 | Value *A, *B; | ||||
1402 | SelectPatternResult SPR = matchSelectPattern(Cmp.getOperand(0), A, B); | ||||
1403 | if (SPR.Flavor == SPF_SMIN) { | ||||
1404 | if (isKnownPositive(A, DL, 0, &AC, &Cmp, &DT)) | ||||
1405 | return new ICmpInst(Pred, B, Cmp.getOperand(1)); | ||||
1406 | if (isKnownPositive(B, DL, 0, &AC, &Cmp, &DT)) | ||||
1407 | return new ICmpInst(Pred, A, Cmp.getOperand(1)); | ||||
1408 | } | ||||
1409 | } | ||||
1410 | |||||
1411 | if (Instruction *New = foldIRemByPowerOfTwoToBitTest(Cmp)) | ||||
1412 | return New; | ||||
1413 | |||||
1414 | // Given: | ||||
1415 | // icmp eq/ne (urem %x, %y), 0 | ||||
1416 | // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem': | ||||
1417 | // icmp eq/ne %x, 0 | ||||
1418 | Value *X, *Y; | ||||
1419 | if (match(Cmp.getOperand(0), m_URem(m_Value(X), m_Value(Y))) && | ||||
1420 | ICmpInst::isEquality(Pred)) { | ||||
1421 | KnownBits XKnown = computeKnownBits(X, 0, &Cmp); | ||||
1422 | KnownBits YKnown = computeKnownBits(Y, 0, &Cmp); | ||||
1423 | if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2) | ||||
1424 | return new ICmpInst(Pred, X, Cmp.getOperand(1)); | ||||
1425 | } | ||||
1426 | |||||
1427 | return nullptr; | ||||
1428 | } | ||||
1429 | |||||
1430 | /// Fold icmp Pred X, C. | ||||
1431 | /// TODO: This code structure does not make sense. The saturating add fold | ||||
1432 | /// should be moved to some other helper and extended as noted below (it is also | ||||
1433 | /// possible that code has been made unnecessary - do we canonicalize IR to | ||||
1434 | /// overflow/saturating intrinsics or not?). | ||||
1435 | Instruction *InstCombinerImpl::foldICmpWithConstant(ICmpInst &Cmp) { | ||||
1436 | // Match the following pattern, which is a common idiom when writing | ||||
1437 | // overflow-safe integer arithmetic functions. The source performs an addition | ||||
1438 | // in wider type and explicitly checks for overflow using comparisons against | ||||
1439 | // INT_MIN and INT_MAX. Simplify by using the sadd_with_overflow intrinsic. | ||||
1440 | // | ||||
1441 | // TODO: This could probably be generalized to handle other overflow-safe | ||||
1442 | // operations if we worked out the formulas to compute the appropriate magic | ||||
1443 | // constants. | ||||
1444 | // | ||||
1445 | // sum = a + b | ||||
1446 | // if (sum+128 >u 255) ... -> llvm.sadd.with.overflow.i8 | ||||
1447 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
1448 | Value *Op0 = Cmp.getOperand(0), *Op1 = Cmp.getOperand(1); | ||||
1449 | Value *A, *B; | ||||
1450 | ConstantInt *CI, *CI2; // I = icmp ugt (add (add A, B), CI2), CI | ||||
1451 | if (Pred == ICmpInst::ICMP_UGT && match(Op1, m_ConstantInt(CI)) && | ||||
1452 | match(Op0, m_Add(m_Add(m_Value(A), m_Value(B)), m_ConstantInt(CI2)))) | ||||
1453 | if (Instruction *Res = processUGT_ADDCST_ADD(Cmp, A, B, CI2, CI, *this)) | ||||
1454 | return Res; | ||||
1455 | |||||
1456 | // icmp(phi(C1, C2, ...), C) -> phi(icmp(C1, C), icmp(C2, C), ...). | ||||
1457 | Constant *C = dyn_cast<Constant>(Op1); | ||||
1458 | if (!C) | ||||
1459 | return nullptr; | ||||
1460 | |||||
1461 | if (auto *Phi = dyn_cast<PHINode>(Op0)) | ||||
1462 | if (all_of(Phi->operands(), [](Value *V) { return isa<Constant>(V); })) { | ||||
1463 | Type *Ty = Cmp.getType(); | ||||
1464 | Builder.SetInsertPoint(Phi); | ||||
1465 | PHINode *NewPhi = | ||||
1466 | Builder.CreatePHI(Ty, Phi->getNumOperands()); | ||||
1467 | for (BasicBlock *Predecessor : predecessors(Phi->getParent())) { | ||||
1468 | auto *Input = | ||||
1469 | cast<Constant>(Phi->getIncomingValueForBlock(Predecessor)); | ||||
1470 | auto *BoolInput = ConstantExpr::getCompare(Pred, Input, C); | ||||
1471 | NewPhi->addIncoming(BoolInput, Predecessor); | ||||
1472 | } | ||||
1473 | NewPhi->takeName(&Cmp); | ||||
1474 | return replaceInstUsesWith(Cmp, NewPhi); | ||||
1475 | } | ||||
1476 | |||||
1477 | return nullptr; | ||||
1478 | } | ||||
1479 | |||||
1480 | /// Canonicalize icmp instructions based on dominating conditions. | ||||
1481 | Instruction *InstCombinerImpl::foldICmpWithDominatingICmp(ICmpInst &Cmp) { | ||||
1482 | // This is a cheap/incomplete check for dominance - just match a single | ||||
1483 | // predecessor with a conditional branch. | ||||
1484 | BasicBlock *CmpBB = Cmp.getParent(); | ||||
1485 | BasicBlock *DomBB = CmpBB->getSinglePredecessor(); | ||||
1486 | if (!DomBB) | ||||
1487 | return nullptr; | ||||
1488 | |||||
1489 | Value *DomCond; | ||||
1490 | BasicBlock *TrueBB, *FalseBB; | ||||
1491 | if (!match(DomBB->getTerminator(), m_Br(m_Value(DomCond), TrueBB, FalseBB))) | ||||
1492 | return nullptr; | ||||
1493 | |||||
1494 | assert((TrueBB == CmpBB || FalseBB == CmpBB) &&(((TrueBB == CmpBB || FalseBB == CmpBB) && "Predecessor block does not point to successor?" ) ? static_cast<void> (0) : __assert_fail ("(TrueBB == CmpBB || FalseBB == CmpBB) && \"Predecessor block does not point to successor?\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1495, __PRETTY_FUNCTION__)) | ||||
1495 | "Predecessor block does not point to successor?")(((TrueBB == CmpBB || FalseBB == CmpBB) && "Predecessor block does not point to successor?" ) ? static_cast<void> (0) : __assert_fail ("(TrueBB == CmpBB || FalseBB == CmpBB) && \"Predecessor block does not point to successor?\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1495, __PRETTY_FUNCTION__)); | ||||
1496 | |||||
1497 | // The branch should get simplified. Don't bother simplifying this condition. | ||||
1498 | if (TrueBB == FalseBB) | ||||
1499 | return nullptr; | ||||
1500 | |||||
1501 | // Try to simplify this compare to T/F based on the dominating condition. | ||||
1502 | Optional<bool> Imp = isImpliedCondition(DomCond, &Cmp, DL, TrueBB == CmpBB); | ||||
1503 | if (Imp) | ||||
1504 | return replaceInstUsesWith(Cmp, ConstantInt::get(Cmp.getType(), *Imp)); | ||||
1505 | |||||
1506 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
1507 | Value *X = Cmp.getOperand(0), *Y = Cmp.getOperand(1); | ||||
1508 | ICmpInst::Predicate DomPred; | ||||
1509 | const APInt *C, *DomC; | ||||
1510 | if (match(DomCond, m_ICmp(DomPred, m_Specific(X), m_APInt(DomC))) && | ||||
1511 | match(Y, m_APInt(C))) { | ||||
1512 | // We have 2 compares of a variable with constants. Calculate the constant | ||||
1513 | // ranges of those compares to see if we can transform the 2nd compare: | ||||
1514 | // DomBB: | ||||
1515 | // DomCond = icmp DomPred X, DomC | ||||
1516 | // br DomCond, CmpBB, FalseBB | ||||
1517 | // CmpBB: | ||||
1518 | // Cmp = icmp Pred X, C | ||||
1519 | ConstantRange CR = ConstantRange::makeAllowedICmpRegion(Pred, *C); | ||||
1520 | ConstantRange DominatingCR = | ||||
1521 | (CmpBB == TrueBB) ? ConstantRange::makeExactICmpRegion(DomPred, *DomC) | ||||
1522 | : ConstantRange::makeExactICmpRegion( | ||||
1523 | CmpInst::getInversePredicate(DomPred), *DomC); | ||||
1524 | ConstantRange Intersection = DominatingCR.intersectWith(CR); | ||||
1525 | ConstantRange Difference = DominatingCR.difference(CR); | ||||
1526 | if (Intersection.isEmptySet()) | ||||
1527 | return replaceInstUsesWith(Cmp, Builder.getFalse()); | ||||
1528 | if (Difference.isEmptySet()) | ||||
1529 | return replaceInstUsesWith(Cmp, Builder.getTrue()); | ||||
1530 | |||||
1531 | // Canonicalizing a sign bit comparison that gets used in a branch, | ||||
1532 | // pessimizes codegen by generating branch on zero instruction instead | ||||
1533 | // of a test and branch. So we avoid canonicalizing in such situations | ||||
1534 | // because test and branch instruction has better branch displacement | ||||
1535 | // than compare and branch instruction. | ||||
1536 | bool UnusedBit; | ||||
1537 | bool IsSignBit = isSignBitCheck(Pred, *C, UnusedBit); | ||||
1538 | if (Cmp.isEquality() || (IsSignBit && hasBranchUse(Cmp))) | ||||
1539 | return nullptr; | ||||
1540 | |||||
1541 | if (const APInt *EqC = Intersection.getSingleElement()) | ||||
1542 | return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(*EqC)); | ||||
1543 | if (const APInt *NeC = Difference.getSingleElement()) | ||||
1544 | return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(*NeC)); | ||||
1545 | } | ||||
1546 | |||||
1547 | return nullptr; | ||||
1548 | } | ||||
1549 | |||||
1550 | /// Fold icmp (trunc X, Y), C. | ||||
1551 | Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp, | ||||
1552 | TruncInst *Trunc, | ||||
1553 | const APInt &C) { | ||||
1554 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
1555 | Value *X = Trunc->getOperand(0); | ||||
1556 | if (C.isOneValue() && C.getBitWidth() > 1) { | ||||
1557 | // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1 | ||||
1558 | Value *V = nullptr; | ||||
1559 | if (Pred == ICmpInst::ICMP_SLT && match(X, m_Signum(m_Value(V)))) | ||||
1560 | return new ICmpInst(ICmpInst::ICMP_SLT, V, | ||||
1561 | ConstantInt::get(V->getType(), 1)); | ||||
1562 | } | ||||
1563 | |||||
1564 | if (Cmp.isEquality() && Trunc->hasOneUse()) { | ||||
1565 | // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all | ||||
1566 | // of the high bits truncated out of x are known. | ||||
1567 | unsigned DstBits = Trunc->getType()->getScalarSizeInBits(), | ||||
1568 | SrcBits = X->getType()->getScalarSizeInBits(); | ||||
1569 | KnownBits Known = computeKnownBits(X, 0, &Cmp); | ||||
1570 | |||||
1571 | // If all the high bits are known, we can do this xform. | ||||
1572 | if ((Known.Zero | Known.One).countLeadingOnes() >= SrcBits - DstBits) { | ||||
1573 | // Pull in the high bits from known-ones set. | ||||
1574 | APInt NewRHS = C.zext(SrcBits); | ||||
1575 | NewRHS |= Known.One & APInt::getHighBitsSet(SrcBits, SrcBits - DstBits); | ||||
1576 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), NewRHS)); | ||||
1577 | } | ||||
1578 | } | ||||
1579 | |||||
1580 | return nullptr; | ||||
1581 | } | ||||
1582 | |||||
1583 | /// Fold icmp (xor X, Y), C. | ||||
1584 | Instruction *InstCombinerImpl::foldICmpXorConstant(ICmpInst &Cmp, | ||||
1585 | BinaryOperator *Xor, | ||||
1586 | const APInt &C) { | ||||
1587 | Value *X = Xor->getOperand(0); | ||||
1588 | Value *Y = Xor->getOperand(1); | ||||
1589 | const APInt *XorC; | ||||
1590 | if (!match(Y, m_APInt(XorC))) | ||||
1591 | return nullptr; | ||||
1592 | |||||
1593 | // If this is a comparison that tests the signbit (X < 0) or (x > -1), | ||||
1594 | // fold the xor. | ||||
1595 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
1596 | bool TrueIfSigned = false; | ||||
1597 | if (isSignBitCheck(Cmp.getPredicate(), C, TrueIfSigned)) { | ||||
1598 | |||||
1599 | // If the sign bit of the XorCst is not set, there is no change to | ||||
1600 | // the operation, just stop using the Xor. | ||||
1601 | if (!XorC->isNegative()) | ||||
1602 | return replaceOperand(Cmp, 0, X); | ||||
1603 | |||||
1604 | // Emit the opposite comparison. | ||||
1605 | if (TrueIfSigned) | ||||
1606 | return new ICmpInst(ICmpInst::ICMP_SGT, X, | ||||
1607 | ConstantInt::getAllOnesValue(X->getType())); | ||||
1608 | else | ||||
1609 | return new ICmpInst(ICmpInst::ICMP_SLT, X, | ||||
1610 | ConstantInt::getNullValue(X->getType())); | ||||
1611 | } | ||||
1612 | |||||
1613 | if (Xor->hasOneUse()) { | ||||
1614 | // (icmp u/s (xor X SignMask), C) -> (icmp s/u X, (xor C SignMask)) | ||||
1615 | if (!Cmp.isEquality() && XorC->isSignMask()) { | ||||
1616 | Pred = Cmp.isSigned() ? Cmp.getUnsignedPredicate() | ||||
1617 | : Cmp.getSignedPredicate(); | ||||
1618 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC)); | ||||
1619 | } | ||||
1620 | |||||
1621 | // (icmp u/s (xor X ~SignMask), C) -> (icmp s/u X, (xor C ~SignMask)) | ||||
1622 | if (!Cmp.isEquality() && XorC->isMaxSignedValue()) { | ||||
1623 | Pred = Cmp.isSigned() ? Cmp.getUnsignedPredicate() | ||||
1624 | : Cmp.getSignedPredicate(); | ||||
1625 | Pred = Cmp.getSwappedPredicate(Pred); | ||||
1626 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC)); | ||||
1627 | } | ||||
1628 | } | ||||
1629 | |||||
1630 | // Mask constant magic can eliminate an 'xor' with unsigned compares. | ||||
1631 | if (Pred == ICmpInst::ICMP_UGT) { | ||||
1632 | // (xor X, ~C) >u C --> X <u ~C (when C+1 is a power of 2) | ||||
1633 | if (*XorC == ~C && (C + 1).isPowerOf2()) | ||||
1634 | return new ICmpInst(ICmpInst::ICMP_ULT, X, Y); | ||||
1635 | // (xor X, C) >u C --> X >u C (when C+1 is a power of 2) | ||||
1636 | if (*XorC == C && (C + 1).isPowerOf2()) | ||||
1637 | return new ICmpInst(ICmpInst::ICMP_UGT, X, Y); | ||||
1638 | } | ||||
1639 | if (Pred == ICmpInst::ICMP_ULT) { | ||||
1640 | // (xor X, -C) <u C --> X >u ~C (when C is a power of 2) | ||||
1641 | if (*XorC == -C && C.isPowerOf2()) | ||||
1642 | return new ICmpInst(ICmpInst::ICMP_UGT, X, | ||||
1643 | ConstantInt::get(X->getType(), ~C)); | ||||
1644 | // (xor X, C) <u C --> X >u ~C (when -C is a power of 2) | ||||
1645 | if (*XorC == C && (-C).isPowerOf2()) | ||||
1646 | return new ICmpInst(ICmpInst::ICMP_UGT, X, | ||||
1647 | ConstantInt::get(X->getType(), ~C)); | ||||
1648 | } | ||||
1649 | return nullptr; | ||||
1650 | } | ||||
1651 | |||||
1652 | /// Fold icmp (and (sh X, Y), C2), C1. | ||||
1653 | Instruction *InstCombinerImpl::foldICmpAndShift(ICmpInst &Cmp, | ||||
1654 | BinaryOperator *And, | ||||
1655 | const APInt &C1, | ||||
1656 | const APInt &C2) { | ||||
1657 | BinaryOperator *Shift = dyn_cast<BinaryOperator>(And->getOperand(0)); | ||||
1658 | if (!Shift || !Shift->isShift()) | ||||
1659 | return nullptr; | ||||
1660 | |||||
1661 | // If this is: (X >> C3) & C2 != C1 (where any shift and any compare could | ||||
1662 | // exist), turn it into (X & (C2 << C3)) != (C1 << C3). This happens a LOT in | ||||
1663 | // code produced by the clang front-end, for bitfield access. | ||||
1664 | // This seemingly simple opportunity to fold away a shift turns out to be | ||||
1665 | // rather complicated. See PR17827 for details. | ||||
1666 | unsigned ShiftOpcode = Shift->getOpcode(); | ||||
1667 | bool IsShl = ShiftOpcode == Instruction::Shl; | ||||
1668 | const APInt *C3; | ||||
1669 | if (match(Shift->getOperand(1), m_APInt(C3))) { | ||||
1670 | APInt NewAndCst, NewCmpCst; | ||||
1671 | bool AnyCmpCstBitsShiftedOut; | ||||
1672 | if (ShiftOpcode == Instruction::Shl) { | ||||
1673 | // For a left shift, we can fold if the comparison is not signed. We can | ||||
1674 | // also fold a signed comparison if the mask value and comparison value | ||||
1675 | // are not negative. These constraints may not be obvious, but we can | ||||
1676 | // prove that they are correct using an SMT solver. | ||||
1677 | if (Cmp.isSigned() && (C2.isNegative() || C1.isNegative())) | ||||
1678 | return nullptr; | ||||
1679 | |||||
1680 | NewCmpCst = C1.lshr(*C3); | ||||
1681 | NewAndCst = C2.lshr(*C3); | ||||
1682 | AnyCmpCstBitsShiftedOut = NewCmpCst.shl(*C3) != C1; | ||||
1683 | } else if (ShiftOpcode == Instruction::LShr) { | ||||
1684 | // For a logical right shift, we can fold if the comparison is not signed. | ||||
1685 | // We can also fold a signed comparison if the shifted mask value and the | ||||
1686 | // shifted comparison value are not negative. These constraints may not be | ||||
1687 | // obvious, but we can prove that they are correct using an SMT solver. | ||||
1688 | NewCmpCst = C1.shl(*C3); | ||||
1689 | NewAndCst = C2.shl(*C3); | ||||
1690 | AnyCmpCstBitsShiftedOut = NewCmpCst.lshr(*C3) != C1; | ||||
1691 | if (Cmp.isSigned() && (NewAndCst.isNegative() || NewCmpCst.isNegative())) | ||||
1692 | return nullptr; | ||||
1693 | } else { | ||||
1694 | // For an arithmetic shift, check that both constants don't use (in a | ||||
1695 | // signed sense) the top bits being shifted out. | ||||
1696 | assert(ShiftOpcode == Instruction::AShr && "Unknown shift opcode")((ShiftOpcode == Instruction::AShr && "Unknown shift opcode" ) ? static_cast<void> (0) : __assert_fail ("ShiftOpcode == Instruction::AShr && \"Unknown shift opcode\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1696, __PRETTY_FUNCTION__)); | ||||
1697 | NewCmpCst = C1.shl(*C3); | ||||
1698 | NewAndCst = C2.shl(*C3); | ||||
1699 | AnyCmpCstBitsShiftedOut = NewCmpCst.ashr(*C3) != C1; | ||||
1700 | if (NewAndCst.ashr(*C3) != C2) | ||||
1701 | return nullptr; | ||||
1702 | } | ||||
1703 | |||||
1704 | if (AnyCmpCstBitsShiftedOut) { | ||||
1705 | // If we shifted bits out, the fold is not going to work out. As a | ||||
1706 | // special case, check to see if this means that the result is always | ||||
1707 | // true or false now. | ||||
1708 | if (Cmp.getPredicate() == ICmpInst::ICMP_EQ) | ||||
1709 | return replaceInstUsesWith(Cmp, ConstantInt::getFalse(Cmp.getType())); | ||||
1710 | if (Cmp.getPredicate() == ICmpInst::ICMP_NE) | ||||
1711 | return replaceInstUsesWith(Cmp, ConstantInt::getTrue(Cmp.getType())); | ||||
1712 | } else { | ||||
1713 | Value *NewAnd = Builder.CreateAnd( | ||||
1714 | Shift->getOperand(0), ConstantInt::get(And->getType(), NewAndCst)); | ||||
1715 | return new ICmpInst(Cmp.getPredicate(), | ||||
1716 | NewAnd, ConstantInt::get(And->getType(), NewCmpCst)); | ||||
1717 | } | ||||
1718 | } | ||||
1719 | |||||
1720 | // Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is | ||||
1721 | // preferable because it allows the C2 << Y expression to be hoisted out of a | ||||
1722 | // loop if Y is invariant and X is not. | ||||
1723 | if (Shift->hasOneUse() && C1.isNullValue() && Cmp.isEquality() && | ||||
1724 | !Shift->isArithmeticShift() && !isa<Constant>(Shift->getOperand(0))) { | ||||
1725 | // Compute C2 << Y. | ||||
1726 | Value *NewShift = | ||||
1727 | IsShl ? Builder.CreateLShr(And->getOperand(1), Shift->getOperand(1)) | ||||
1728 | : Builder.CreateShl(And->getOperand(1), Shift->getOperand(1)); | ||||
1729 | |||||
1730 | // Compute X & (C2 << Y). | ||||
1731 | Value *NewAnd = Builder.CreateAnd(Shift->getOperand(0), NewShift); | ||||
1732 | return replaceOperand(Cmp, 0, NewAnd); | ||||
1733 | } | ||||
1734 | |||||
1735 | return nullptr; | ||||
1736 | } | ||||
1737 | |||||
1738 | /// Fold icmp (and X, C2), C1. | ||||
1739 | Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp, | ||||
1740 | BinaryOperator *And, | ||||
1741 | const APInt &C1) { | ||||
1742 | bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE; | ||||
1743 | |||||
1744 | // For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1 | ||||
1745 | // TODO: We canonicalize to the longer form for scalars because we have | ||||
1746 | // better analysis/folds for icmp, and codegen may be better with icmp. | ||||
1747 | if (isICMP_NE && Cmp.getType()->isVectorTy() && C1.isNullValue() && | ||||
1748 | match(And->getOperand(1), m_One())) | ||||
1749 | return new TruncInst(And->getOperand(0), Cmp.getType()); | ||||
1750 | |||||
1751 | const APInt *C2; | ||||
1752 | Value *X; | ||||
1753 | if (!match(And, m_And(m_Value(X), m_APInt(C2)))) | ||||
1754 | return nullptr; | ||||
1755 | |||||
1756 | // Don't perform the following transforms if the AND has multiple uses | ||||
1757 | if (!And->hasOneUse()) | ||||
1758 | return nullptr; | ||||
1759 | |||||
1760 | if (Cmp.isEquality() && C1.isNullValue()) { | ||||
1761 | // Restrict this fold to single-use 'and' (PR10267). | ||||
1762 | // Replace (and X, (1 << size(X)-1) != 0) with X s< 0 | ||||
1763 | if (C2->isSignMask()) { | ||||
1764 | Constant *Zero = Constant::getNullValue(X->getType()); | ||||
1765 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE; | ||||
1766 | return new ICmpInst(NewPred, X, Zero); | ||||
1767 | } | ||||
1768 | |||||
1769 | // Restrict this fold only for single-use 'and' (PR10267). | ||||
1770 | // ((%x & C) == 0) --> %x u< (-C) iff (-C) is power of two. | ||||
1771 | if ((~(*C2) + 1).isPowerOf2()) { | ||||
1772 | Constant *NegBOC = | ||||
1773 | ConstantExpr::getNeg(cast<Constant>(And->getOperand(1))); | ||||
1774 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; | ||||
1775 | return new ICmpInst(NewPred, X, NegBOC); | ||||
1776 | } | ||||
1777 | } | ||||
1778 | |||||
1779 | // If the LHS is an 'and' of a truncate and we can widen the and/compare to | ||||
1780 | // the input width without changing the value produced, eliminate the cast: | ||||
1781 | // | ||||
1782 | // icmp (and (trunc W), C2), C1 -> icmp (and W, C2'), C1' | ||||
1783 | // | ||||
1784 | // We can do this transformation if the constants do not have their sign bits | ||||
1785 | // set or if it is an equality comparison. Extending a relational comparison | ||||
1786 | // when we're checking the sign bit would not work. | ||||
1787 | Value *W; | ||||
1788 | if (match(And->getOperand(0), m_OneUse(m_Trunc(m_Value(W)))) && | ||||
1789 | (Cmp.isEquality() || (!C1.isNegative() && !C2->isNegative()))) { | ||||
1790 | // TODO: Is this a good transform for vectors? Wider types may reduce | ||||
1791 | // throughput. Should this transform be limited (even for scalars) by using | ||||
1792 | // shouldChangeType()? | ||||
1793 | if (!Cmp.getType()->isVectorTy()) { | ||||
1794 | Type *WideType = W->getType(); | ||||
1795 | unsigned WideScalarBits = WideType->getScalarSizeInBits(); | ||||
1796 | Constant *ZextC1 = ConstantInt::get(WideType, C1.zext(WideScalarBits)); | ||||
1797 | Constant *ZextC2 = ConstantInt::get(WideType, C2->zext(WideScalarBits)); | ||||
1798 | Value *NewAnd = Builder.CreateAnd(W, ZextC2, And->getName()); | ||||
1799 | return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1); | ||||
1800 | } | ||||
1801 | } | ||||
1802 | |||||
1803 | if (Instruction *I = foldICmpAndShift(Cmp, And, C1, *C2)) | ||||
1804 | return I; | ||||
1805 | |||||
1806 | // (icmp pred (and (or (lshr A, B), A), 1), 0) --> | ||||
1807 | // (icmp pred (and A, (or (shl 1, B), 1), 0)) | ||||
1808 | // | ||||
1809 | // iff pred isn't signed | ||||
1810 | if (!Cmp.isSigned() && C1.isNullValue() && And->getOperand(0)->hasOneUse() && | ||||
1811 | match(And->getOperand(1), m_One())) { | ||||
1812 | Constant *One = cast<Constant>(And->getOperand(1)); | ||||
1813 | Value *Or = And->getOperand(0); | ||||
1814 | Value *A, *B, *LShr; | ||||
1815 | if (match(Or, m_Or(m_Value(LShr), m_Value(A))) && | ||||
1816 | match(LShr, m_LShr(m_Specific(A), m_Value(B)))) { | ||||
1817 | unsigned UsesRemoved = 0; | ||||
1818 | if (And->hasOneUse()) | ||||
1819 | ++UsesRemoved; | ||||
1820 | if (Or->hasOneUse()) | ||||
1821 | ++UsesRemoved; | ||||
1822 | if (LShr->hasOneUse()) | ||||
1823 | ++UsesRemoved; | ||||
1824 | |||||
1825 | // Compute A & ((1 << B) | 1) | ||||
1826 | Value *NewOr = nullptr; | ||||
1827 | if (auto *C = dyn_cast<Constant>(B)) { | ||||
1828 | if (UsesRemoved >= 1) | ||||
1829 | NewOr = ConstantExpr::getOr(ConstantExpr::getNUWShl(One, C), One); | ||||
1830 | } else { | ||||
1831 | if (UsesRemoved >= 3) | ||||
1832 | NewOr = Builder.CreateOr(Builder.CreateShl(One, B, LShr->getName(), | ||||
1833 | /*HasNUW=*/true), | ||||
1834 | One, Or->getName()); | ||||
1835 | } | ||||
1836 | if (NewOr) { | ||||
1837 | Value *NewAnd = Builder.CreateAnd(A, NewOr, And->getName()); | ||||
1838 | return replaceOperand(Cmp, 0, NewAnd); | ||||
1839 | } | ||||
1840 | } | ||||
1841 | } | ||||
1842 | |||||
1843 | return nullptr; | ||||
1844 | } | ||||
1845 | |||||
1846 | /// Fold icmp (and X, Y), C. | ||||
1847 | Instruction *InstCombinerImpl::foldICmpAndConstant(ICmpInst &Cmp, | ||||
1848 | BinaryOperator *And, | ||||
1849 | const APInt &C) { | ||||
1850 | if (Instruction *I = foldICmpAndConstConst(Cmp, And, C)) | ||||
1851 | return I; | ||||
1852 | |||||
1853 | // TODO: These all require that Y is constant too, so refactor with the above. | ||||
1854 | |||||
1855 | // Try to optimize things like "A[i] & 42 == 0" to index computations. | ||||
1856 | Value *X = And->getOperand(0); | ||||
1857 | Value *Y = And->getOperand(1); | ||||
1858 | if (auto *LI = dyn_cast<LoadInst>(X)) | ||||
1859 | if (auto *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0))) | ||||
1860 | if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) | ||||
1861 | if (GV->isConstant() && GV->hasDefinitiveInitializer() && | ||||
1862 | !LI->isVolatile() && isa<ConstantInt>(Y)) { | ||||
1863 | ConstantInt *C2 = cast<ConstantInt>(Y); | ||||
1864 | if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, Cmp, C2)) | ||||
1865 | return Res; | ||||
1866 | } | ||||
1867 | |||||
1868 | if (!Cmp.isEquality()) | ||||
1869 | return nullptr; | ||||
1870 | |||||
1871 | // X & -C == -C -> X > u ~C | ||||
1872 | // X & -C != -C -> X <= u ~C | ||||
1873 | // iff C is a power of 2 | ||||
1874 | if (Cmp.getOperand(1) == Y && (-C).isPowerOf2()) { | ||||
1875 | auto NewPred = Cmp.getPredicate() == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGT | ||||
1876 | : CmpInst::ICMP_ULE; | ||||
1877 | return new ICmpInst(NewPred, X, SubOne(cast<Constant>(Cmp.getOperand(1)))); | ||||
1878 | } | ||||
1879 | |||||
1880 | // (X & C2) == 0 -> (trunc X) >= 0 | ||||
1881 | // (X & C2) != 0 -> (trunc X) < 0 | ||||
1882 | // iff C2 is a power of 2 and it masks the sign bit of a legal integer type. | ||||
1883 | const APInt *C2; | ||||
1884 | if (And->hasOneUse() && C.isNullValue() && match(Y, m_APInt(C2))) { | ||||
1885 | int32_t ExactLogBase2 = C2->exactLogBase2(); | ||||
1886 | if (ExactLogBase2 != -1 && DL.isLegalInteger(ExactLogBase2 + 1)) { | ||||
1887 | Type *NTy = IntegerType::get(Cmp.getContext(), ExactLogBase2 + 1); | ||||
1888 | if (auto *AndVTy = dyn_cast<VectorType>(And->getType())) | ||||
1889 | NTy = FixedVectorType::get( | ||||
1890 | NTy, cast<FixedVectorType>(AndVTy)->getNumElements()); | ||||
1891 | Value *Trunc = Builder.CreateTrunc(X, NTy); | ||||
1892 | auto NewPred = Cmp.getPredicate() == CmpInst::ICMP_EQ ? CmpInst::ICMP_SGE | ||||
1893 | : CmpInst::ICMP_SLT; | ||||
1894 | return new ICmpInst(NewPred, Trunc, Constant::getNullValue(NTy)); | ||||
1895 | } | ||||
1896 | } | ||||
1897 | |||||
1898 | return nullptr; | ||||
1899 | } | ||||
1900 | |||||
1901 | /// Fold icmp (or X, Y), C. | ||||
1902 | Instruction *InstCombinerImpl::foldICmpOrConstant(ICmpInst &Cmp, | ||||
1903 | BinaryOperator *Or, | ||||
1904 | const APInt &C) { | ||||
1905 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
1906 | if (C.isOneValue()) { | ||||
1907 | // icmp slt signum(V) 1 --> icmp slt V, 1 | ||||
1908 | Value *V = nullptr; | ||||
1909 | if (Pred == ICmpInst::ICMP_SLT && match(Or, m_Signum(m_Value(V)))) | ||||
1910 | return new ICmpInst(ICmpInst::ICMP_SLT, V, | ||||
1911 | ConstantInt::get(V->getType(), 1)); | ||||
1912 | } | ||||
1913 | |||||
1914 | Value *OrOp0 = Or->getOperand(0), *OrOp1 = Or->getOperand(1); | ||||
1915 | const APInt *MaskC; | ||||
1916 | if (match(OrOp1, m_APInt(MaskC)) && Cmp.isEquality()) { | ||||
1917 | if (*MaskC == C && (C + 1).isPowerOf2()) { | ||||
1918 | // X | C == C --> X <=u C | ||||
1919 | // X | C != C --> X >u C | ||||
1920 | // iff C+1 is a power of 2 (C is a bitmask of the low bits) | ||||
1921 | Pred = (Pred == CmpInst::ICMP_EQ) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT; | ||||
1922 | return new ICmpInst(Pred, OrOp0, OrOp1); | ||||
1923 | } | ||||
1924 | |||||
1925 | // More general: canonicalize 'equality with set bits mask' to | ||||
1926 | // 'equality with clear bits mask'. | ||||
1927 | // (X | MaskC) == C --> (X & ~MaskC) == C ^ MaskC | ||||
1928 | // (X | MaskC) != C --> (X & ~MaskC) != C ^ MaskC | ||||
1929 | if (Or->hasOneUse()) { | ||||
1930 | Value *And = Builder.CreateAnd(OrOp0, ~(*MaskC)); | ||||
1931 | Constant *NewC = ConstantInt::get(Or->getType(), C ^ (*MaskC)); | ||||
1932 | return new ICmpInst(Pred, And, NewC); | ||||
1933 | } | ||||
1934 | } | ||||
1935 | |||||
1936 | if (!Cmp.isEquality() || !C.isNullValue() || !Or->hasOneUse()) | ||||
1937 | return nullptr; | ||||
1938 | |||||
1939 | Value *P, *Q; | ||||
1940 | if (match(Or, m_Or(m_PtrToInt(m_Value(P)), m_PtrToInt(m_Value(Q))))) { | ||||
1941 | // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0 | ||||
1942 | // -> and (icmp eq P, null), (icmp eq Q, null). | ||||
1943 | Value *CmpP = | ||||
1944 | Builder.CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType())); | ||||
1945 | Value *CmpQ = | ||||
1946 | Builder.CreateICmp(Pred, Q, ConstantInt::getNullValue(Q->getType())); | ||||
1947 | auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; | ||||
1948 | return BinaryOperator::Create(BOpc, CmpP, CmpQ); | ||||
1949 | } | ||||
1950 | |||||
1951 | // Are we using xors to bitwise check for a pair of (in)equalities? Convert to | ||||
1952 | // a shorter form that has more potential to be folded even further. | ||||
1953 | Value *X1, *X2, *X3, *X4; | ||||
1954 | if (match(OrOp0, m_OneUse(m_Xor(m_Value(X1), m_Value(X2)))) && | ||||
1955 | match(OrOp1, m_OneUse(m_Xor(m_Value(X3), m_Value(X4))))) { | ||||
1956 | // ((X1 ^ X2) || (X3 ^ X4)) == 0 --> (X1 == X2) && (X3 == X4) | ||||
1957 | // ((X1 ^ X2) || (X3 ^ X4)) != 0 --> (X1 != X2) || (X3 != X4) | ||||
1958 | Value *Cmp12 = Builder.CreateICmp(Pred, X1, X2); | ||||
1959 | Value *Cmp34 = Builder.CreateICmp(Pred, X3, X4); | ||||
1960 | auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; | ||||
1961 | return BinaryOperator::Create(BOpc, Cmp12, Cmp34); | ||||
1962 | } | ||||
1963 | |||||
1964 | return nullptr; | ||||
1965 | } | ||||
1966 | |||||
1967 | /// Fold icmp (mul X, Y), C. | ||||
1968 | Instruction *InstCombinerImpl::foldICmpMulConstant(ICmpInst &Cmp, | ||||
1969 | BinaryOperator *Mul, | ||||
1970 | const APInt &C) { | ||||
1971 | const APInt *MulC; | ||||
1972 | if (!match(Mul->getOperand(1), m_APInt(MulC))) | ||||
1973 | return nullptr; | ||||
1974 | |||||
1975 | // If this is a test of the sign bit and the multiply is sign-preserving with | ||||
1976 | // a constant operand, use the multiply LHS operand instead. | ||||
1977 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
1978 | if (isSignTest(Pred, C) && Mul->hasNoSignedWrap()) { | ||||
1979 | if (MulC->isNegative()) | ||||
1980 | Pred = ICmpInst::getSwappedPredicate(Pred); | ||||
1981 | return new ICmpInst(Pred, Mul->getOperand(0), | ||||
1982 | Constant::getNullValue(Mul->getType())); | ||||
1983 | } | ||||
1984 | |||||
1985 | // If the multiply does not wrap, try to divide the compare constant by the | ||||
1986 | // multiplication factor. | ||||
1987 | if (Cmp.isEquality() && !MulC->isNullValue()) { | ||||
1988 | // (mul nsw X, MulC) == C --> X == C /s MulC | ||||
1989 | if (Mul->hasNoSignedWrap() && C.srem(*MulC).isNullValue()) { | ||||
1990 | Constant *NewC = ConstantInt::get(Mul->getType(), C.sdiv(*MulC)); | ||||
1991 | return new ICmpInst(Pred, Mul->getOperand(0), NewC); | ||||
1992 | } | ||||
1993 | // (mul nuw X, MulC) == C --> X == C /u MulC | ||||
1994 | if (Mul->hasNoUnsignedWrap() && C.urem(*MulC).isNullValue()) { | ||||
1995 | Constant *NewC = ConstantInt::get(Mul->getType(), C.udiv(*MulC)); | ||||
1996 | return new ICmpInst(Pred, Mul->getOperand(0), NewC); | ||||
1997 | } | ||||
1998 | } | ||||
1999 | |||||
2000 | return nullptr; | ||||
2001 | } | ||||
2002 | |||||
2003 | /// Fold icmp (shl 1, Y), C. | ||||
2004 | static Instruction *foldICmpShlOne(ICmpInst &Cmp, Instruction *Shl, | ||||
2005 | const APInt &C) { | ||||
2006 | Value *Y; | ||||
2007 | if (!match(Shl, m_Shl(m_One(), m_Value(Y)))) | ||||
2008 | return nullptr; | ||||
2009 | |||||
2010 | Type *ShiftType = Shl->getType(); | ||||
2011 | unsigned TypeBits = C.getBitWidth(); | ||||
2012 | bool CIsPowerOf2 = C.isPowerOf2(); | ||||
2013 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
2014 | if (Cmp.isUnsigned()) { | ||||
2015 | // (1 << Y) pred C -> Y pred Log2(C) | ||||
2016 | if (!CIsPowerOf2) { | ||||
2017 | // (1 << Y) < 30 -> Y <= 4 | ||||
2018 | // (1 << Y) <= 30 -> Y <= 4 | ||||
2019 | // (1 << Y) >= 30 -> Y > 4 | ||||
2020 | // (1 << Y) > 30 -> Y > 4 | ||||
2021 | if (Pred == ICmpInst::ICMP_ULT) | ||||
2022 | Pred = ICmpInst::ICMP_ULE; | ||||
2023 | else if (Pred == ICmpInst::ICMP_UGE) | ||||
2024 | Pred = ICmpInst::ICMP_UGT; | ||||
2025 | } | ||||
2026 | |||||
2027 | // (1 << Y) >= 2147483648 -> Y >= 31 -> Y == 31 | ||||
2028 | // (1 << Y) < 2147483648 -> Y < 31 -> Y != 31 | ||||
2029 | unsigned CLog2 = C.logBase2(); | ||||
2030 | if (CLog2 == TypeBits - 1) { | ||||
2031 | if (Pred == ICmpInst::ICMP_UGE) | ||||
2032 | Pred = ICmpInst::ICMP_EQ; | ||||
2033 | else if (Pred == ICmpInst::ICMP_ULT) | ||||
2034 | Pred = ICmpInst::ICMP_NE; | ||||
2035 | } | ||||
2036 | return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, CLog2)); | ||||
2037 | } else if (Cmp.isSigned()) { | ||||
2038 | Constant *BitWidthMinusOne = ConstantInt::get(ShiftType, TypeBits - 1); | ||||
2039 | if (C.isAllOnesValue()) { | ||||
2040 | // (1 << Y) <= -1 -> Y == 31 | ||||
2041 | if (Pred == ICmpInst::ICMP_SLE) | ||||
2042 | return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); | ||||
2043 | |||||
2044 | // (1 << Y) > -1 -> Y != 31 | ||||
2045 | if (Pred == ICmpInst::ICMP_SGT) | ||||
2046 | return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); | ||||
2047 | } else if (!C) { | ||||
2048 | // (1 << Y) < 0 -> Y == 31 | ||||
2049 | // (1 << Y) <= 0 -> Y == 31 | ||||
2050 | if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) | ||||
2051 | return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); | ||||
2052 | |||||
2053 | // (1 << Y) >= 0 -> Y != 31 | ||||
2054 | // (1 << Y) > 0 -> Y != 31 | ||||
2055 | if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) | ||||
2056 | return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); | ||||
2057 | } | ||||
2058 | } else if (Cmp.isEquality() && CIsPowerOf2) { | ||||
2059 | return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, C.logBase2())); | ||||
2060 | } | ||||
2061 | |||||
2062 | return nullptr; | ||||
2063 | } | ||||
2064 | |||||
2065 | /// Fold icmp (shl X, Y), C. | ||||
2066 | Instruction *InstCombinerImpl::foldICmpShlConstant(ICmpInst &Cmp, | ||||
2067 | BinaryOperator *Shl, | ||||
2068 | const APInt &C) { | ||||
2069 | const APInt *ShiftVal; | ||||
2070 | if (Cmp.isEquality() && match(Shl->getOperand(0), m_APInt(ShiftVal))) | ||||
2071 | return foldICmpShlConstConst(Cmp, Shl->getOperand(1), C, *ShiftVal); | ||||
2072 | |||||
2073 | const APInt *ShiftAmt; | ||||
2074 | if (!match(Shl->getOperand(1), m_APInt(ShiftAmt))) | ||||
2075 | return foldICmpShlOne(Cmp, Shl, C); | ||||
2076 | |||||
2077 | // Check that the shift amount is in range. If not, don't perform undefined | ||||
2078 | // shifts. When the shift is visited, it will be simplified. | ||||
2079 | unsigned TypeBits = C.getBitWidth(); | ||||
2080 | if (ShiftAmt->uge(TypeBits)) | ||||
2081 | return nullptr; | ||||
2082 | |||||
2083 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
2084 | Value *X = Shl->getOperand(0); | ||||
2085 | Type *ShType = Shl->getType(); | ||||
2086 | |||||
2087 | // NSW guarantees that we are only shifting out sign bits from the high bits, | ||||
2088 | // so we can ASHR the compare constant without needing a mask and eliminate | ||||
2089 | // the shift. | ||||
2090 | if (Shl->hasNoSignedWrap()) { | ||||
2091 | if (Pred == ICmpInst::ICMP_SGT) { | ||||
2092 | // icmp Pred (shl nsw X, ShiftAmt), C --> icmp Pred X, (C >>s ShiftAmt) | ||||
2093 | APInt ShiftedC = C.ashr(*ShiftAmt); | ||||
2094 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||
2095 | } | ||||
2096 | if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && | ||||
2097 | C.ashr(*ShiftAmt).shl(*ShiftAmt) == C) { | ||||
2098 | APInt ShiftedC = C.ashr(*ShiftAmt); | ||||
2099 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||
2100 | } | ||||
2101 | if (Pred == ICmpInst::ICMP_SLT) { | ||||
2102 | // SLE is the same as above, but SLE is canonicalized to SLT, so convert: | ||||
2103 | // (X << S) <=s C is equiv to X <=s (C >> S) for all C | ||||
2104 | // (X << S) <s (C + 1) is equiv to X <s (C >> S) + 1 if C <s SMAX | ||||
2105 | // (X << S) <s C is equiv to X <s ((C - 1) >> S) + 1 if C >s SMIN | ||||
2106 | assert(!C.isMinSignedValue() && "Unexpected icmp slt")((!C.isMinSignedValue() && "Unexpected icmp slt") ? static_cast <void> (0) : __assert_fail ("!C.isMinSignedValue() && \"Unexpected icmp slt\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2106, __PRETTY_FUNCTION__)); | ||||
2107 | APInt ShiftedC = (C - 1).ashr(*ShiftAmt) + 1; | ||||
2108 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||
2109 | } | ||||
2110 | // If this is a signed comparison to 0 and the shift is sign preserving, | ||||
2111 | // use the shift LHS operand instead; isSignTest may change 'Pred', so only | ||||
2112 | // do that if we're sure to not continue on in this function. | ||||
2113 | if (isSignTest(Pred, C)) | ||||
2114 | return new ICmpInst(Pred, X, Constant::getNullValue(ShType)); | ||||
2115 | } | ||||
2116 | |||||
2117 | // NUW guarantees that we are only shifting out zero bits from the high bits, | ||||
2118 | // so we can LSHR the compare constant without needing a mask and eliminate | ||||
2119 | // the shift. | ||||
2120 | if (Shl->hasNoUnsignedWrap()) { | ||||
2121 | if (Pred == ICmpInst::ICMP_UGT) { | ||||
2122 | // icmp Pred (shl nuw X, ShiftAmt), C --> icmp Pred X, (C >>u ShiftAmt) | ||||
2123 | APInt ShiftedC = C.lshr(*ShiftAmt); | ||||
2124 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||
2125 | } | ||||
2126 | if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && | ||||
2127 | C.lshr(*ShiftAmt).shl(*ShiftAmt) == C) { | ||||
2128 | APInt ShiftedC = C.lshr(*ShiftAmt); | ||||
2129 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||
2130 | } | ||||
2131 | if (Pred == ICmpInst::ICMP_ULT) { | ||||
2132 | // ULE is the same as above, but ULE is canonicalized to ULT, so convert: | ||||
2133 | // (X << S) <=u C is equiv to X <=u (C >> S) for all C | ||||
2134 | // (X << S) <u (C + 1) is equiv to X <u (C >> S) + 1 if C <u ~0u | ||||
2135 | // (X << S) <u C is equiv to X <u ((C - 1) >> S) + 1 if C >u 0 | ||||
2136 | assert(C.ugt(0) && "ult 0 should have been eliminated")((C.ugt(0) && "ult 0 should have been eliminated") ? static_cast <void> (0) : __assert_fail ("C.ugt(0) && \"ult 0 should have been eliminated\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2136, __PRETTY_FUNCTION__)); | ||||
2137 | APInt ShiftedC = (C - 1).lshr(*ShiftAmt) + 1; | ||||
2138 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||
2139 | } | ||||
2140 | } | ||||
2141 | |||||
2142 | if (Cmp.isEquality() && Shl->hasOneUse()) { | ||||
2143 | // Strength-reduce the shift into an 'and'. | ||||
2144 | Constant *Mask = ConstantInt::get( | ||||
2145 | ShType, | ||||
2146 | APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt->getZExtValue())); | ||||
2147 | Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask"); | ||||
2148 | Constant *LShrC = ConstantInt::get(ShType, C.lshr(*ShiftAmt)); | ||||
2149 | return new ICmpInst(Pred, And, LShrC); | ||||
2150 | } | ||||
2151 | |||||
2152 | // Otherwise, if this is a comparison of the sign bit, simplify to and/test. | ||||
2153 | bool TrueIfSigned = false; | ||||
2154 | if (Shl->hasOneUse() && isSignBitCheck(Pred, C, TrueIfSigned)) { | ||||
2155 | // (X << 31) <s 0 --> (X & 1) != 0 | ||||
2156 | Constant *Mask = ConstantInt::get( | ||||
2157 | ShType, | ||||
2158 | APInt::getOneBitSet(TypeBits, TypeBits - ShiftAmt->getZExtValue() - 1)); | ||||
2159 | Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask"); | ||||
2160 | return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ, | ||||
2161 | And, Constant::getNullValue(ShType)); | ||||
2162 | } | ||||
2163 | |||||
2164 | // Simplify 'shl' inequality test into 'and' equality test. | ||||
2165 | if (Cmp.isUnsigned() && Shl->hasOneUse()) { | ||||
2166 | // (X l<< C2) u<=/u> C1 iff C1+1 is power of two -> X & (~C1 l>> C2) ==/!= 0 | ||||
2167 | if ((C + 1).isPowerOf2() && | ||||
2168 | (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT)) { | ||||
2169 | Value *And = Builder.CreateAnd(X, (~C).lshr(ShiftAmt->getZExtValue())); | ||||
2170 | return new ICmpInst(Pred == ICmpInst::ICMP_ULE ? ICmpInst::ICMP_EQ | ||||
2171 | : ICmpInst::ICMP_NE, | ||||
2172 | And, Constant::getNullValue(ShType)); | ||||
2173 | } | ||||
2174 | // (X l<< C2) u</u>= C1 iff C1 is power of two -> X & (-C1 l>> C2) ==/!= 0 | ||||
2175 | if (C.isPowerOf2() && | ||||
2176 | (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) { | ||||
2177 | Value *And = | ||||
2178 | Builder.CreateAnd(X, (~(C - 1)).lshr(ShiftAmt->getZExtValue())); | ||||
2179 | return new ICmpInst(Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_EQ | ||||
2180 | : ICmpInst::ICMP_NE, | ||||
2181 | And, Constant::getNullValue(ShType)); | ||||
2182 | } | ||||
2183 | } | ||||
2184 | |||||
2185 | // Transform (icmp pred iM (shl iM %v, N), C) | ||||
2186 | // -> (icmp pred i(M-N) (trunc %v iM to i(M-N)), (trunc (C>>N)) | ||||
2187 | // Transform the shl to a trunc if (trunc (C>>N)) has no loss and M-N. | ||||
2188 | // This enables us to get rid of the shift in favor of a trunc that may be | ||||
2189 | // free on the target. It has the additional benefit of comparing to a | ||||
2190 | // smaller constant that may be more target-friendly. | ||||
2191 | unsigned Amt = ShiftAmt->getLimitedValue(TypeBits - 1); | ||||
2192 | if (Shl->hasOneUse() && Amt != 0 && C.countTrailingZeros() >= Amt && | ||||
2193 | DL.isLegalInteger(TypeBits - Amt)) { | ||||
2194 | Type *TruncTy = IntegerType::get(Cmp.getContext(), TypeBits - Amt); | ||||
2195 | if (auto *ShVTy = dyn_cast<VectorType>(ShType)) | ||||
2196 | TruncTy = FixedVectorType::get( | ||||
2197 | TruncTy, cast<FixedVectorType>(ShVTy)->getNumElements()); | ||||
2198 | Constant *NewC = | ||||
2199 | ConstantInt::get(TruncTy, C.ashr(*ShiftAmt).trunc(TypeBits - Amt)); | ||||
2200 | return new ICmpInst(Pred, Builder.CreateTrunc(X, TruncTy), NewC); | ||||
2201 | } | ||||
2202 | |||||
2203 | return nullptr; | ||||
2204 | } | ||||
2205 | |||||
2206 | /// Fold icmp ({al}shr X, Y), C. | ||||
2207 | Instruction *InstCombinerImpl::foldICmpShrConstant(ICmpInst &Cmp, | ||||
2208 | BinaryOperator *Shr, | ||||
2209 | const APInt &C) { | ||||
2210 | // An exact shr only shifts out zero bits, so: | ||||
2211 | // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0 | ||||
2212 | Value *X = Shr->getOperand(0); | ||||
2213 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
2214 | if (Cmp.isEquality() && Shr->isExact() && Shr->hasOneUse() && | ||||
2215 | C.isNullValue()) | ||||
2216 | return new ICmpInst(Pred, X, Cmp.getOperand(1)); | ||||
2217 | |||||
2218 | const APInt *ShiftVal; | ||||
2219 | if (Cmp.isEquality() && match(Shr->getOperand(0), m_APInt(ShiftVal))) | ||||
2220 | return foldICmpShrConstConst(Cmp, Shr->getOperand(1), C, *ShiftVal); | ||||
2221 | |||||
2222 | const APInt *ShiftAmt; | ||||
2223 | if (!match(Shr->getOperand(1), m_APInt(ShiftAmt))) | ||||
2224 | return nullptr; | ||||
2225 | |||||
2226 | // Check that the shift amount is in range. If not, don't perform undefined | ||||
2227 | // shifts. When the shift is visited it will be simplified. | ||||
2228 | unsigned TypeBits = C.getBitWidth(); | ||||
2229 | unsigned ShAmtVal = ShiftAmt->getLimitedValue(TypeBits); | ||||
2230 | if (ShAmtVal >= TypeBits || ShAmtVal == 0) | ||||
2231 | return nullptr; | ||||
2232 | |||||
2233 | bool IsAShr = Shr->getOpcode() == Instruction::AShr; | ||||
2234 | bool IsExact = Shr->isExact(); | ||||
2235 | Type *ShrTy = Shr->getType(); | ||||
2236 | // TODO: If we could guarantee that InstSimplify would handle all of the | ||||
2237 | // constant-value-based preconditions in the folds below, then we could assert | ||||
2238 | // those conditions rather than checking them. This is difficult because of | ||||
2239 | // undef/poison (PR34838). | ||||
2240 | if (IsAShr) { | ||||
2241 | if (Pred == CmpInst::ICMP_SLT || (Pred == CmpInst::ICMP_SGT && IsExact)) { | ||||
2242 | // icmp slt (ashr X, ShAmtC), C --> icmp slt X, (C << ShAmtC) | ||||
2243 | // icmp sgt (ashr exact X, ShAmtC), C --> icmp sgt X, (C << ShAmtC) | ||||
2244 | APInt ShiftedC = C.shl(ShAmtVal); | ||||
2245 | if (ShiftedC.ashr(ShAmtVal) == C) | ||||
2246 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); | ||||
2247 | } | ||||
2248 | if (Pred == CmpInst::ICMP_SGT) { | ||||
2249 | // icmp sgt (ashr X, ShAmtC), C --> icmp sgt X, ((C + 1) << ShAmtC) - 1 | ||||
2250 | APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1; | ||||
2251 | if (!C.isMaxSignedValue() && !(C + 1).shl(ShAmtVal).isMinSignedValue() && | ||||
2252 | (ShiftedC + 1).ashr(ShAmtVal) == (C + 1)) | ||||
2253 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); | ||||
2254 | } | ||||
2255 | } else { | ||||
2256 | if (Pred == CmpInst::ICMP_ULT || (Pred == CmpInst::ICMP_UGT && IsExact)) { | ||||
2257 | // icmp ult (lshr X, ShAmtC), C --> icmp ult X, (C << ShAmtC) | ||||
2258 | // icmp ugt (lshr exact X, ShAmtC), C --> icmp ugt X, (C << ShAmtC) | ||||
2259 | APInt ShiftedC = C.shl(ShAmtVal); | ||||
2260 | if (ShiftedC.lshr(ShAmtVal) == C) | ||||
2261 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); | ||||
2262 | } | ||||
2263 | if (Pred == CmpInst::ICMP_UGT) { | ||||
2264 | // icmp ugt (lshr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1 | ||||
2265 | APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1; | ||||
2266 | if ((ShiftedC + 1).lshr(ShAmtVal) == (C + 1)) | ||||
2267 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); | ||||
2268 | } | ||||
2269 | } | ||||
2270 | |||||
2271 | if (!Cmp.isEquality()) | ||||
2272 | return nullptr; | ||||
2273 | |||||
2274 | // Handle equality comparisons of shift-by-constant. | ||||
2275 | |||||
2276 | // If the comparison constant changes with the shift, the comparison cannot | ||||
2277 | // succeed (bits of the comparison constant cannot match the shifted value). | ||||
2278 | // This should be known by InstSimplify and already be folded to true/false. | ||||
2279 | assert(((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) ||((((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && "Expected icmp+shr simplify did not occur.") ? static_cast< void> (0) : __assert_fail ("((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && \"Expected icmp+shr simplify did not occur.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2281, __PRETTY_FUNCTION__)) | ||||
2280 | (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) &&((((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && "Expected icmp+shr simplify did not occur.") ? static_cast< void> (0) : __assert_fail ("((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && \"Expected icmp+shr simplify did not occur.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2281, __PRETTY_FUNCTION__)) | ||||
2281 | "Expected icmp+shr simplify did not occur.")((((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && "Expected icmp+shr simplify did not occur.") ? static_cast< void> (0) : __assert_fail ("((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && \"Expected icmp+shr simplify did not occur.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2281, __PRETTY_FUNCTION__)); | ||||
2282 | |||||
2283 | // If the bits shifted out are known zero, compare the unshifted value: | ||||
2284 | // (X & 4) >> 1 == 2 --> (X & 4) == 4. | ||||
2285 | if (Shr->isExact()) | ||||
2286 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, C << ShAmtVal)); | ||||
2287 | |||||
2288 | if (Shr->hasOneUse()) { | ||||
2289 | // Canonicalize the shift into an 'and': | ||||
2290 | // icmp eq/ne (shr X, ShAmt), C --> icmp eq/ne (and X, HiMask), (C << ShAmt) | ||||
2291 | APInt Val(APInt::getHighBitsSet(TypeBits, TypeBits - ShAmtVal)); | ||||
2292 | Constant *Mask = ConstantInt::get(ShrTy, Val); | ||||
2293 | Value *And = Builder.CreateAnd(X, Mask, Shr->getName() + ".mask"); | ||||
2294 | return new ICmpInst(Pred, And, ConstantInt::get(ShrTy, C << ShAmtVal)); | ||||
2295 | } | ||||
2296 | |||||
2297 | return nullptr; | ||||
2298 | } | ||||
2299 | |||||
2300 | Instruction *InstCombinerImpl::foldICmpSRemConstant(ICmpInst &Cmp, | ||||
2301 | BinaryOperator *SRem, | ||||
2302 | const APInt &C) { | ||||
2303 | // Match an 'is positive' or 'is negative' comparison of remainder by a | ||||
2304 | // constant power-of-2 value: | ||||
2305 | // (X % pow2C) sgt/slt 0 | ||||
2306 | const ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
2307 | if (Pred != ICmpInst::ICMP_SGT && Pred != ICmpInst::ICMP_SLT) | ||||
2308 | return nullptr; | ||||
2309 | |||||
2310 | // TODO: The one-use check is standard because we do not typically want to | ||||
2311 | // create longer instruction sequences, but this might be a special-case | ||||
2312 | // because srem is not good for analysis or codegen. | ||||
2313 | if (!SRem->hasOneUse()) | ||||
2314 | return nullptr; | ||||
2315 | |||||
2316 | const APInt *DivisorC; | ||||
2317 | if (!C.isNullValue() || !match(SRem->getOperand(1), m_Power2(DivisorC))) | ||||
2318 | return nullptr; | ||||
2319 | |||||
2320 | // Mask off the sign bit and the modulo bits (low-bits). | ||||
2321 | Type *Ty = SRem->getType(); | ||||
2322 | APInt SignMask = APInt::getSignMask(Ty->getScalarSizeInBits()); | ||||
2323 | Constant *MaskC = ConstantInt::get(Ty, SignMask | (*DivisorC - 1)); | ||||
2324 | Value *And = Builder.CreateAnd(SRem->getOperand(0), MaskC); | ||||
2325 | |||||
2326 | // For 'is positive?' check that the sign-bit is clear and at least 1 masked | ||||
2327 | // bit is set. Example: | ||||
2328 | // (i8 X % 32) s> 0 --> (X & 159) s> 0 | ||||
2329 | if (Pred == ICmpInst::ICMP_SGT) | ||||
2330 | return new ICmpInst(ICmpInst::ICMP_SGT, And, ConstantInt::getNullValue(Ty)); | ||||
2331 | |||||
2332 | // For 'is negative?' check that the sign-bit is set and at least 1 masked | ||||
2333 | // bit is set. Example: | ||||
2334 | // (i16 X % 4) s< 0 --> (X & 32771) u> 32768 | ||||
2335 | return new ICmpInst(ICmpInst::ICMP_UGT, And, ConstantInt::get(Ty, SignMask)); | ||||
2336 | } | ||||
2337 | |||||
2338 | /// Fold icmp (udiv X, Y), C. | ||||
2339 | Instruction *InstCombinerImpl::foldICmpUDivConstant(ICmpInst &Cmp, | ||||
2340 | BinaryOperator *UDiv, | ||||
2341 | const APInt &C) { | ||||
2342 | const APInt *C2; | ||||
2343 | if (!match(UDiv->getOperand(0), m_APInt(C2))) | ||||
2344 | return nullptr; | ||||
2345 | |||||
2346 | assert(*C2 != 0 && "udiv 0, X should have been simplified already.")((*C2 != 0 && "udiv 0, X should have been simplified already." ) ? static_cast<void> (0) : __assert_fail ("*C2 != 0 && \"udiv 0, X should have been simplified already.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2346, __PRETTY_FUNCTION__)); | ||||
2347 | |||||
2348 | // (icmp ugt (udiv C2, Y), C) -> (icmp ule Y, C2/(C+1)) | ||||
2349 | Value *Y = UDiv->getOperand(1); | ||||
2350 | if (Cmp.getPredicate() == ICmpInst::ICMP_UGT) { | ||||
2351 | assert(!C.isMaxValue() &&((!C.isMaxValue() && "icmp ugt X, UINT_MAX should have been simplified already." ) ? static_cast<void> (0) : __assert_fail ("!C.isMaxValue() && \"icmp ugt X, UINT_MAX should have been simplified already.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2352, __PRETTY_FUNCTION__)) | ||||
2352 | "icmp ugt X, UINT_MAX should have been simplified already.")((!C.isMaxValue() && "icmp ugt X, UINT_MAX should have been simplified already." ) ? static_cast<void> (0) : __assert_fail ("!C.isMaxValue() && \"icmp ugt X, UINT_MAX should have been simplified already.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2352, __PRETTY_FUNCTION__)); | ||||
2353 | return new ICmpInst(ICmpInst::ICMP_ULE, Y, | ||||
2354 | ConstantInt::get(Y->getType(), C2->udiv(C + 1))); | ||||
2355 | } | ||||
2356 | |||||
2357 | // (icmp ult (udiv C2, Y), C) -> (icmp ugt Y, C2/C) | ||||
2358 | if (Cmp.getPredicate() == ICmpInst::ICMP_ULT) { | ||||
2359 | assert(C != 0 && "icmp ult X, 0 should have been simplified already.")((C != 0 && "icmp ult X, 0 should have been simplified already." ) ? static_cast<void> (0) : __assert_fail ("C != 0 && \"icmp ult X, 0 should have been simplified already.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2359, __PRETTY_FUNCTION__)); | ||||
2360 | return new ICmpInst(ICmpInst::ICMP_UGT, Y, | ||||
2361 | ConstantInt::get(Y->getType(), C2->udiv(C))); | ||||
2362 | } | ||||
2363 | |||||
2364 | return nullptr; | ||||
2365 | } | ||||
2366 | |||||
2367 | /// Fold icmp ({su}div X, Y), C. | ||||
2368 | Instruction *InstCombinerImpl::foldICmpDivConstant(ICmpInst &Cmp, | ||||
2369 | BinaryOperator *Div, | ||||
2370 | const APInt &C) { | ||||
2371 | // Fold: icmp pred ([us]div X, C2), C -> range test | ||||
2372 | // Fold this div into the comparison, producing a range check. | ||||
2373 | // Determine, based on the divide type, what the range is being | ||||
2374 | // checked. If there is an overflow on the low or high side, remember | ||||
2375 | // it, otherwise compute the range [low, hi) bounding the new value. | ||||
2376 | // See: InsertRangeTest above for the kinds of replacements possible. | ||||
2377 | const APInt *C2; | ||||
2378 | if (!match(Div->getOperand(1), m_APInt(C2))) | ||||
2379 | return nullptr; | ||||
2380 | |||||
2381 | // FIXME: If the operand types don't match the type of the divide | ||||
2382 | // then don't attempt this transform. The code below doesn't have the | ||||
2383 | // logic to deal with a signed divide and an unsigned compare (and | ||||
2384 | // vice versa). This is because (x /s C2) <s C produces different | ||||
2385 | // results than (x /s C2) <u C or (x /u C2) <s C or even | ||||
2386 | // (x /u C2) <u C. Simply casting the operands and result won't | ||||
2387 | // work. :( The if statement below tests that condition and bails | ||||
2388 | // if it finds it. | ||||
2389 | bool DivIsSigned = Div->getOpcode() == Instruction::SDiv; | ||||
2390 | if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned()) | ||||
2391 | return nullptr; | ||||
2392 | |||||
2393 | // The ProdOV computation fails on divide by 0 and divide by -1. Cases with | ||||
2394 | // INT_MIN will also fail if the divisor is 1. Although folds of all these | ||||
2395 | // division-by-constant cases should be present, we can not assert that they | ||||
2396 | // have happened before we reach this icmp instruction. | ||||
2397 | if (C2->isNullValue() || C2->isOneValue() || | ||||
2398 | (DivIsSigned && C2->isAllOnesValue())) | ||||
2399 | return nullptr; | ||||
2400 | |||||
2401 | // Compute Prod = C * C2. We are essentially solving an equation of | ||||
2402 | // form X / C2 = C. We solve for X by multiplying C2 and C. | ||||
2403 | // By solving for X, we can turn this into a range check instead of computing | ||||
2404 | // a divide. | ||||
2405 | APInt Prod = C * *C2; | ||||
2406 | |||||
2407 | // Determine if the product overflows by seeing if the product is not equal to | ||||
2408 | // the divide. Make sure we do the same kind of divide as in the LHS | ||||
2409 | // instruction that we're folding. | ||||
2410 | bool ProdOV = (DivIsSigned ? Prod.sdiv(*C2) : Prod.udiv(*C2)) != C; | ||||
2411 | |||||
2412 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
2413 | |||||
2414 | // If the division is known to be exact, then there is no remainder from the | ||||
2415 | // divide, so the covered range size is unit, otherwise it is the divisor. | ||||
2416 | APInt RangeSize = Div->isExact() ? APInt(C2->getBitWidth(), 1) : *C2; | ||||
2417 | |||||
2418 | // Figure out the interval that is being checked. For example, a comparison | ||||
2419 | // like "X /u 5 == 0" is really checking that X is in the interval [0, 5). | ||||
2420 | // Compute this interval based on the constants involved and the signedness of | ||||
2421 | // the compare/divide. This computes a half-open interval, keeping track of | ||||
2422 | // whether either value in the interval overflows. After analysis each | ||||
2423 | // overflow variable is set to 0 if it's corresponding bound variable is valid | ||||
2424 | // -1 if overflowed off the bottom end, or +1 if overflowed off the top end. | ||||
2425 | int LoOverflow = 0, HiOverflow = 0; | ||||
2426 | APInt LoBound, HiBound; | ||||
2427 | |||||
2428 | if (!DivIsSigned) { // udiv | ||||
2429 | // e.g. X/5 op 3 --> [15, 20) | ||||
2430 | LoBound = Prod; | ||||
2431 | HiOverflow = LoOverflow = ProdOV; | ||||
2432 | if (!HiOverflow) { | ||||
2433 | // If this is not an exact divide, then many values in the range collapse | ||||
2434 | // to the same result value. | ||||
2435 | HiOverflow = addWithOverflow(HiBound, LoBound, RangeSize, false); | ||||
2436 | } | ||||
2437 | } else if (C2->isStrictlyPositive()) { // Divisor is > 0. | ||||
2438 | if (C.isNullValue()) { // (X / pos) op 0 | ||||
2439 | // Can't overflow. e.g. X/2 op 0 --> [-1, 2) | ||||
2440 | LoBound = -(RangeSize - 1); | ||||
2441 | HiBound = RangeSize; | ||||
2442 | } else if (C.isStrictlyPositive()) { // (X / pos) op pos | ||||
2443 | LoBound = Prod; // e.g. X/5 op 3 --> [15, 20) | ||||
2444 | HiOverflow = LoOverflow = ProdOV; | ||||
2445 | if (!HiOverflow) | ||||
2446 | HiOverflow = addWithOverflow(HiBound, Prod, RangeSize, true); | ||||
2447 | } else { // (X / pos) op neg | ||||
2448 | // e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14) | ||||
2449 | HiBound = Prod + 1; | ||||
2450 | LoOverflow = HiOverflow = ProdOV ? -1 : 0; | ||||
2451 | if (!LoOverflow) { | ||||
2452 | APInt DivNeg = -RangeSize; | ||||
2453 | LoOverflow = addWithOverflow(LoBound, HiBound, DivNeg, true) ? -1 : 0; | ||||
2454 | } | ||||
2455 | } | ||||
2456 | } else if (C2->isNegative()) { // Divisor is < 0. | ||||
2457 | if (Div->isExact()) | ||||
2458 | RangeSize.negate(); | ||||
2459 | if (C.isNullValue()) { // (X / neg) op 0 | ||||
2460 | // e.g. X/-5 op 0 --> [-4, 5) | ||||
2461 | LoBound = RangeSize + 1; | ||||
2462 | HiBound = -RangeSize; | ||||
2463 | if (HiBound == *C2) { // -INTMIN = INTMIN | ||||
2464 | HiOverflow = 1; // [INTMIN+1, overflow) | ||||
2465 | HiBound = APInt(); // e.g. X/INTMIN = 0 --> X > INTMIN | ||||
2466 | } | ||||
2467 | } else if (C.isStrictlyPositive()) { // (X / neg) op pos | ||||
2468 | // e.g. X/-5 op 3 --> [-19, -14) | ||||
2469 | HiBound = Prod + 1; | ||||
2470 | HiOverflow = LoOverflow = ProdOV ? -1 : 0; | ||||
2471 | if (!LoOverflow) | ||||
2472 | LoOverflow = addWithOverflow(LoBound, HiBound, RangeSize, true) ? -1:0; | ||||
2473 | } else { // (X / neg) op neg | ||||
2474 | LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20) | ||||
2475 | LoOverflow = HiOverflow = ProdOV; | ||||
2476 | if (!HiOverflow) | ||||
2477 | HiOverflow = subWithOverflow(HiBound, Prod, RangeSize, true); | ||||
2478 | } | ||||
2479 | |||||
2480 | // Dividing by a negative swaps the condition. LT <-> GT | ||||
2481 | Pred = ICmpInst::getSwappedPredicate(Pred); | ||||
2482 | } | ||||
2483 | |||||
2484 | Value *X = Div->getOperand(0); | ||||
2485 | switch (Pred) { | ||||
2486 | default: llvm_unreachable("Unhandled icmp opcode!")::llvm::llvm_unreachable_internal("Unhandled icmp opcode!", "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2486); | ||||
2487 | case ICmpInst::ICMP_EQ: | ||||
2488 | if (LoOverflow && HiOverflow) | ||||
2489 | return replaceInstUsesWith(Cmp, Builder.getFalse()); | ||||
2490 | if (HiOverflow) | ||||
2491 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : | ||||
2492 | ICmpInst::ICMP_UGE, X, | ||||
2493 | ConstantInt::get(Div->getType(), LoBound)); | ||||
2494 | if (LoOverflow) | ||||
2495 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : | ||||
2496 | ICmpInst::ICMP_ULT, X, | ||||
2497 | ConstantInt::get(Div->getType(), HiBound)); | ||||
2498 | return replaceInstUsesWith( | ||||
2499 | Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, true)); | ||||
2500 | case ICmpInst::ICMP_NE: | ||||
2501 | if (LoOverflow && HiOverflow) | ||||
2502 | return replaceInstUsesWith(Cmp, Builder.getTrue()); | ||||
2503 | if (HiOverflow) | ||||
2504 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : | ||||
2505 | ICmpInst::ICMP_ULT, X, | ||||
2506 | ConstantInt::get(Div->getType(), LoBound)); | ||||
2507 | if (LoOverflow) | ||||
2508 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : | ||||
2509 | ICmpInst::ICMP_UGE, X, | ||||
2510 | ConstantInt::get(Div->getType(), HiBound)); | ||||
2511 | return replaceInstUsesWith(Cmp, | ||||
2512 | insertRangeTest(X, LoBound, HiBound, | ||||
2513 | DivIsSigned, false)); | ||||
2514 | case ICmpInst::ICMP_ULT: | ||||
2515 | case ICmpInst::ICMP_SLT: | ||||
2516 | if (LoOverflow == +1) // Low bound is greater than input range. | ||||
2517 | return replaceInstUsesWith(Cmp, Builder.getTrue()); | ||||
2518 | if (LoOverflow == -1) // Low bound is less than input range. | ||||
2519 | return replaceInstUsesWith(Cmp, Builder.getFalse()); | ||||
2520 | return new ICmpInst(Pred, X, ConstantInt::get(Div->getType(), LoBound)); | ||||
2521 | case ICmpInst::ICMP_UGT: | ||||
2522 | case ICmpInst::ICMP_SGT: | ||||
2523 | if (HiOverflow == +1) // High bound greater than input range. | ||||
2524 | return replaceInstUsesWith(Cmp, Builder.getFalse()); | ||||
2525 | if (HiOverflow == -1) // High bound less than input range. | ||||
2526 | return replaceInstUsesWith(Cmp, Builder.getTrue()); | ||||
2527 | if (Pred == ICmpInst::ICMP_UGT) | ||||
2528 | return new ICmpInst(ICmpInst::ICMP_UGE, X, | ||||
2529 | ConstantInt::get(Div->getType(), HiBound)); | ||||
2530 | return new ICmpInst(ICmpInst::ICMP_SGE, X, | ||||
2531 | ConstantInt::get(Div->getType(), HiBound)); | ||||
2532 | } | ||||
2533 | |||||
2534 | return nullptr; | ||||
2535 | } | ||||
2536 | |||||
2537 | /// Fold icmp (sub X, Y), C. | ||||
2538 | Instruction *InstCombinerImpl::foldICmpSubConstant(ICmpInst &Cmp, | ||||
2539 | BinaryOperator *Sub, | ||||
2540 | const APInt &C) { | ||||
2541 | Value *X = Sub->getOperand(0), *Y = Sub->getOperand(1); | ||||
2542 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
2543 | const APInt *C2; | ||||
2544 | APInt SubResult; | ||||
2545 | |||||
2546 | // icmp eq/ne (sub C, Y), C -> icmp eq/ne Y, 0 | ||||
2547 | if (match(X, m_APInt(C2)) && *C2 == C && Cmp.isEquality()) | ||||
2548 | return new ICmpInst(Cmp.getPredicate(), Y, | ||||
2549 | ConstantInt::get(Y->getType(), 0)); | ||||
2550 | |||||
2551 | // (icmp P (sub nuw|nsw C2, Y), C) -> (icmp swap(P) Y, C2-C) | ||||
2552 | if (match(X, m_APInt(C2)) && | ||||
2553 | ((Cmp.isUnsigned() && Sub->hasNoUnsignedWrap()) || | ||||
2554 | (Cmp.isSigned() && Sub->hasNoSignedWrap())) && | ||||
2555 | !subWithOverflow(SubResult, *C2, C, Cmp.isSigned())) | ||||
2556 | return new ICmpInst(Cmp.getSwappedPredicate(), Y, | ||||
2557 | ConstantInt::get(Y->getType(), SubResult)); | ||||
2558 | |||||
2559 | // The following transforms are only worth it if the only user of the subtract | ||||
2560 | // is the icmp. | ||||
2561 | if (!Sub->hasOneUse()) | ||||
2562 | return nullptr; | ||||
2563 | |||||
2564 | if (Sub->hasNoSignedWrap()) { | ||||
2565 | // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y) | ||||
2566 | if (Pred == ICmpInst::ICMP_SGT && C.isAllOnesValue()) | ||||
2567 | return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); | ||||
2568 | |||||
2569 | // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y) | ||||
2570 | if (Pred == ICmpInst::ICMP_SGT && C.isNullValue()) | ||||
2571 | return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); | ||||
2572 | |||||
2573 | // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y) | ||||
2574 | if (Pred == ICmpInst::ICMP_SLT && C.isNullValue()) | ||||
2575 | return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); | ||||
2576 | |||||
2577 | // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y) | ||||
2578 | if (Pred == ICmpInst::ICMP_SLT && C.isOneValue()) | ||||
2579 | return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); | ||||
2580 | } | ||||
2581 | |||||
2582 | if (!match(X, m_APInt(C2))) | ||||
2583 | return nullptr; | ||||
2584 | |||||
2585 | // C2 - Y <u C -> (Y | (C - 1)) == C2 | ||||
2586 | // iff (C2 & (C - 1)) == C - 1 and C is a power of 2 | ||||
2587 | if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && | ||||
2588 | (*C2 & (C - 1)) == (C - 1)) | ||||
2589 | return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateOr(Y, C - 1), X); | ||||
2590 | |||||
2591 | // C2 - Y >u C -> (Y | C) != C2 | ||||
2592 | // iff C2 & C == C and C + 1 is a power of 2 | ||||
2593 | if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == C) | ||||
2594 | return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateOr(Y, C), X); | ||||
2595 | |||||
2596 | return nullptr; | ||||
2597 | } | ||||
2598 | |||||
2599 | /// Fold icmp (add X, Y), C. | ||||
2600 | Instruction *InstCombinerImpl::foldICmpAddConstant(ICmpInst &Cmp, | ||||
2601 | BinaryOperator *Add, | ||||
2602 | const APInt &C) { | ||||
2603 | Value *Y = Add->getOperand(1); | ||||
2604 | const APInt *C2; | ||||
2605 | if (Cmp.isEquality() || !match(Y, m_APInt(C2))) | ||||
2606 | return nullptr; | ||||
2607 | |||||
2608 | // Fold icmp pred (add X, C2), C. | ||||
2609 | Value *X = Add->getOperand(0); | ||||
2610 | Type *Ty = Add->getType(); | ||||
2611 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
2612 | |||||
2613 | // If the add does not wrap, we can always adjust the compare by subtracting | ||||
2614 | // the constants. Equality comparisons are handled elsewhere. SGE/SLE/UGE/ULE | ||||
2615 | // are canonicalized to SGT/SLT/UGT/ULT. | ||||
2616 | if ((Add->hasNoSignedWrap() && | ||||
2617 | (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT)) || | ||||
2618 | (Add->hasNoUnsignedWrap() && | ||||
2619 | (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULT))) { | ||||
2620 | bool Overflow; | ||||
2621 | APInt NewC = | ||||
2622 | Cmp.isSigned() ? C.ssub_ov(*C2, Overflow) : C.usub_ov(*C2, Overflow); | ||||
2623 | // If there is overflow, the result must be true or false. | ||||
2624 | // TODO: Can we assert there is no overflow because InstSimplify always | ||||
2625 | // handles those cases? | ||||
2626 | if (!Overflow) | ||||
2627 | // icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2) | ||||
2628 | return new ICmpInst(Pred, X, ConstantInt::get(Ty, NewC)); | ||||
2629 | } | ||||
2630 | |||||
2631 | auto CR = ConstantRange::makeExactICmpRegion(Pred, C).subtract(*C2); | ||||
2632 | const APInt &Upper = CR.getUpper(); | ||||
2633 | const APInt &Lower = CR.getLower(); | ||||
2634 | if (Cmp.isSigned()) { | ||||
2635 | if (Lower.isSignMask()) | ||||
2636 | return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, Upper)); | ||||
2637 | if (Upper.isSignMask()) | ||||
2638 | return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, Lower)); | ||||
2639 | } else { | ||||
2640 | if (Lower.isMinValue()) | ||||
2641 | return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, Upper)); | ||||
2642 | if (Upper.isMinValue()) | ||||
2643 | return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, Lower)); | ||||
2644 | } | ||||
2645 | |||||
2646 | if (!Add->hasOneUse()) | ||||
2647 | return nullptr; | ||||
2648 | |||||
2649 | // X+C <u C2 -> (X & -C2) == C | ||||
2650 | // iff C & (C2-1) == 0 | ||||
2651 | // C2 is a power of 2 | ||||
2652 | if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && (*C2 & (C - 1)) == 0) | ||||
2653 | return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateAnd(X, -C), | ||||
2654 | ConstantExpr::getNeg(cast<Constant>(Y))); | ||||
2655 | |||||
2656 | // X+C >u C2 -> (X & ~C2) != C | ||||
2657 | // iff C & C2 == 0 | ||||
2658 | // C2+1 is a power of 2 | ||||
2659 | if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == 0) | ||||
2660 | return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateAnd(X, ~C), | ||||
2661 | ConstantExpr::getNeg(cast<Constant>(Y))); | ||||
2662 | |||||
2663 | return nullptr; | ||||
2664 | } | ||||
2665 | |||||
2666 | bool InstCombinerImpl::matchThreeWayIntCompare(SelectInst *SI, Value *&LHS, | ||||
2667 | Value *&RHS, ConstantInt *&Less, | ||||
2668 | ConstantInt *&Equal, | ||||
2669 | ConstantInt *&Greater) { | ||||
2670 | // TODO: Generalize this to work with other comparison idioms or ensure | ||||
2671 | // they get canonicalized into this form. | ||||
2672 | |||||
2673 | // select i1 (a == b), | ||||
2674 | // i32 Equal, | ||||
2675 | // i32 (select i1 (a < b), i32 Less, i32 Greater) | ||||
2676 | // where Equal, Less and Greater are placeholders for any three constants. | ||||
2677 | ICmpInst::Predicate PredA; | ||||
2678 | if (!match(SI->getCondition(), m_ICmp(PredA, m_Value(LHS), m_Value(RHS))) || | ||||
2679 | !ICmpInst::isEquality(PredA)) | ||||
2680 | return false; | ||||
2681 | Value *EqualVal = SI->getTrueValue(); | ||||
2682 | Value *UnequalVal = SI->getFalseValue(); | ||||
2683 | // We still can get non-canonical predicate here, so canonicalize. | ||||
2684 | if (PredA == ICmpInst::ICMP_NE) | ||||
2685 | std::swap(EqualVal, UnequalVal); | ||||
2686 | if (!match(EqualVal, m_ConstantInt(Equal))) | ||||
2687 | return false; | ||||
2688 | ICmpInst::Predicate PredB; | ||||
2689 | Value *LHS2, *RHS2; | ||||
2690 | if (!match(UnequalVal, m_Select(m_ICmp(PredB, m_Value(LHS2), m_Value(RHS2)), | ||||
2691 | m_ConstantInt(Less), m_ConstantInt(Greater)))) | ||||
2692 | return false; | ||||
2693 | // We can get predicate mismatch here, so canonicalize if possible: | ||||
2694 | // First, ensure that 'LHS' match. | ||||
2695 | if (LHS2 != LHS) { | ||||
2696 | // x sgt y <--> y slt x | ||||
2697 | std::swap(LHS2, RHS2); | ||||
2698 | PredB = ICmpInst::getSwappedPredicate(PredB); | ||||
2699 | } | ||||
2700 | if (LHS2 != LHS) | ||||
2701 | return false; | ||||
2702 | // We also need to canonicalize 'RHS'. | ||||
2703 | if (PredB == ICmpInst::ICMP_SGT && isa<Constant>(RHS2)) { | ||||
2704 | // x sgt C-1 <--> x sge C <--> not(x slt C) | ||||
2705 | auto FlippedStrictness = | ||||
2706 | InstCombiner::getFlippedStrictnessPredicateAndConstant( | ||||
2707 | PredB, cast<Constant>(RHS2)); | ||||
2708 | if (!FlippedStrictness) | ||||
2709 | return false; | ||||
2710 | assert(FlippedStrictness->first == ICmpInst::ICMP_SGE && "Sanity check")((FlippedStrictness->first == ICmpInst::ICMP_SGE && "Sanity check") ? static_cast<void> (0) : __assert_fail ("FlippedStrictness->first == ICmpInst::ICMP_SGE && \"Sanity check\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2710, __PRETTY_FUNCTION__)); | ||||
2711 | RHS2 = FlippedStrictness->second; | ||||
2712 | // And kind-of perform the result swap. | ||||
2713 | std::swap(Less, Greater); | ||||
2714 | PredB = ICmpInst::ICMP_SLT; | ||||
2715 | } | ||||
2716 | return PredB == ICmpInst::ICMP_SLT && RHS == RHS2; | ||||
2717 | } | ||||
2718 | |||||
2719 | Instruction *InstCombinerImpl::foldICmpSelectConstant(ICmpInst &Cmp, | ||||
2720 | SelectInst *Select, | ||||
2721 | ConstantInt *C) { | ||||
2722 | |||||
2723 | assert(C && "Cmp RHS should be a constant int!")((C && "Cmp RHS should be a constant int!") ? static_cast <void> (0) : __assert_fail ("C && \"Cmp RHS should be a constant int!\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2723, __PRETTY_FUNCTION__)); | ||||
2724 | // If we're testing a constant value against the result of a three way | ||||
2725 | // comparison, the result can be expressed directly in terms of the | ||||
2726 | // original values being compared. Note: We could possibly be more | ||||
2727 | // aggressive here and remove the hasOneUse test. The original select is | ||||
2728 | // really likely to simplify or sink when we remove a test of the result. | ||||
2729 | Value *OrigLHS, *OrigRHS; | ||||
2730 | ConstantInt *C1LessThan, *C2Equal, *C3GreaterThan; | ||||
2731 | if (Cmp.hasOneUse() && | ||||
2732 | matchThreeWayIntCompare(Select, OrigLHS, OrigRHS, C1LessThan, C2Equal, | ||||
2733 | C3GreaterThan)) { | ||||
2734 | assert(C1LessThan && C2Equal && C3GreaterThan)((C1LessThan && C2Equal && C3GreaterThan) ? static_cast <void> (0) : __assert_fail ("C1LessThan && C2Equal && C3GreaterThan" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2734, __PRETTY_FUNCTION__)); | ||||
2735 | |||||
2736 | bool TrueWhenLessThan = | ||||
2737 | ConstantExpr::getCompare(Cmp.getPredicate(), C1LessThan, C) | ||||
2738 | ->isAllOnesValue(); | ||||
2739 | bool TrueWhenEqual = | ||||
2740 | ConstantExpr::getCompare(Cmp.getPredicate(), C2Equal, C) | ||||
2741 | ->isAllOnesValue(); | ||||
2742 | bool TrueWhenGreaterThan = | ||||
2743 | ConstantExpr::getCompare(Cmp.getPredicate(), C3GreaterThan, C) | ||||
2744 | ->isAllOnesValue(); | ||||
2745 | |||||
2746 | // This generates the new instruction that will replace the original Cmp | ||||
2747 | // Instruction. Instead of enumerating the various combinations when | ||||
2748 | // TrueWhenLessThan, TrueWhenEqual and TrueWhenGreaterThan are true versus | ||||
2749 | // false, we rely on chaining of ORs and future passes of InstCombine to | ||||
2750 | // simplify the OR further (i.e. a s< b || a == b becomes a s<= b). | ||||
2751 | |||||
2752 | // When none of the three constants satisfy the predicate for the RHS (C), | ||||
2753 | // the entire original Cmp can be simplified to a false. | ||||
2754 | Value *Cond = Builder.getFalse(); | ||||
2755 | if (TrueWhenLessThan) | ||||
2756 | Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SLT, | ||||
2757 | OrigLHS, OrigRHS)); | ||||
2758 | if (TrueWhenEqual) | ||||
2759 | Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_EQ, | ||||
2760 | OrigLHS, OrigRHS)); | ||||
2761 | if (TrueWhenGreaterThan) | ||||
2762 | Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SGT, | ||||
2763 | OrigLHS, OrigRHS)); | ||||
2764 | |||||
2765 | return replaceInstUsesWith(Cmp, Cond); | ||||
2766 | } | ||||
2767 | return nullptr; | ||||
2768 | } | ||||
2769 | |||||
2770 | static Instruction *foldICmpBitCast(ICmpInst &Cmp, | ||||
2771 | InstCombiner::BuilderTy &Builder) { | ||||
2772 | auto *Bitcast = dyn_cast<BitCastInst>(Cmp.getOperand(0)); | ||||
2773 | if (!Bitcast) | ||||
2774 | return nullptr; | ||||
2775 | |||||
2776 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
2777 | Value *Op1 = Cmp.getOperand(1); | ||||
2778 | Value *BCSrcOp = Bitcast->getOperand(0); | ||||
2779 | |||||
2780 | // Make sure the bitcast doesn't change the number of vector elements. | ||||
2781 | if (Bitcast->getSrcTy()->getScalarSizeInBits() == | ||||
2782 | Bitcast->getDestTy()->getScalarSizeInBits()) { | ||||
2783 | // Zero-equality and sign-bit checks are preserved through sitofp + bitcast. | ||||
2784 | Value *X; | ||||
2785 | if (match(BCSrcOp, m_SIToFP(m_Value(X)))) { | ||||
2786 | // icmp eq (bitcast (sitofp X)), 0 --> icmp eq X, 0 | ||||
2787 | // icmp ne (bitcast (sitofp X)), 0 --> icmp ne X, 0 | ||||
2788 | // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0 | ||||
2789 | // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0 | ||||
2790 | if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT || | ||||
2791 | Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) && | ||||
2792 | match(Op1, m_Zero())) | ||||
2793 | return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType())); | ||||
2794 | |||||
2795 | // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1 | ||||
2796 | if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_One())) | ||||
2797 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), 1)); | ||||
2798 | |||||
2799 | // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1 | ||||
2800 | if (Pred == ICmpInst::ICMP_SGT && match(Op1, m_AllOnes())) | ||||
2801 | return new ICmpInst(Pred, X, | ||||
2802 | ConstantInt::getAllOnesValue(X->getType())); | ||||
2803 | } | ||||
2804 | |||||
2805 | // Zero-equality checks are preserved through unsigned floating-point casts: | ||||
2806 | // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0 | ||||
2807 | // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0 | ||||
2808 | if (match(BCSrcOp, m_UIToFP(m_Value(X)))) | ||||
2809 | if (Cmp.isEquality() && match(Op1, m_Zero())) | ||||
2810 | return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType())); | ||||
2811 | |||||
2812 | // If this is a sign-bit test of a bitcast of a casted FP value, eliminate | ||||
2813 | // the FP extend/truncate because that cast does not change the sign-bit. | ||||
2814 | // This is true for all standard IEEE-754 types and the X86 80-bit type. | ||||
2815 | // The sign-bit is always the most significant bit in those types. | ||||
2816 | const APInt *C; | ||||
2817 | bool TrueIfSigned; | ||||
2818 | if (match(Op1, m_APInt(C)) && Bitcast->hasOneUse() && | ||||
2819 | InstCombiner::isSignBitCheck(Pred, *C, TrueIfSigned)) { | ||||
2820 | if (match(BCSrcOp, m_FPExt(m_Value(X))) || | ||||
2821 | match(BCSrcOp, m_FPTrunc(m_Value(X)))) { | ||||
2822 | // (bitcast (fpext/fptrunc X)) to iX) < 0 --> (bitcast X to iY) < 0 | ||||
2823 | // (bitcast (fpext/fptrunc X)) to iX) > -1 --> (bitcast X to iY) > -1 | ||||
2824 | Type *XType = X->getType(); | ||||
2825 | |||||
2826 | // We can't currently handle Power style floating point operations here. | ||||
2827 | if (!(XType->isPPC_FP128Ty() || BCSrcOp->getType()->isPPC_FP128Ty())) { | ||||
2828 | |||||
2829 | Type *NewType = Builder.getIntNTy(XType->getScalarSizeInBits()); | ||||
2830 | if (auto *XVTy = dyn_cast<VectorType>(XType)) | ||||
2831 | NewType = FixedVectorType::get( | ||||
2832 | NewType, cast<FixedVectorType>(XVTy)->getNumElements()); | ||||
2833 | Value *NewBitcast = Builder.CreateBitCast(X, NewType); | ||||
2834 | if (TrueIfSigned) | ||||
2835 | return new ICmpInst(ICmpInst::ICMP_SLT, NewBitcast, | ||||
2836 | ConstantInt::getNullValue(NewType)); | ||||
2837 | else | ||||
2838 | return new ICmpInst(ICmpInst::ICMP_SGT, NewBitcast, | ||||
2839 | ConstantInt::getAllOnesValue(NewType)); | ||||
2840 | } | ||||
2841 | } | ||||
2842 | } | ||||
2843 | } | ||||
2844 | |||||
2845 | // Test to see if the operands of the icmp are casted versions of other | ||||
2846 | // values. If the ptr->ptr cast can be stripped off both arguments, do so. | ||||
2847 | if (Bitcast->getType()->isPointerTy() && | ||||
2848 | (isa<Constant>(Op1) || isa<BitCastInst>(Op1))) { | ||||
2849 | // If operand #1 is a bitcast instruction, it must also be a ptr->ptr cast | ||||
2850 | // so eliminate it as well. | ||||
2851 | if (auto *BC2 = dyn_cast<BitCastInst>(Op1)) | ||||
2852 | Op1 = BC2->getOperand(0); | ||||
2853 | |||||
2854 | Op1 = Builder.CreateBitCast(Op1, BCSrcOp->getType()); | ||||
2855 | return new ICmpInst(Pred, BCSrcOp, Op1); | ||||
2856 | } | ||||
2857 | |||||
2858 | // Folding: icmp <pred> iN X, C | ||||
2859 | // where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN | ||||
2860 | // and C is a splat of a K-bit pattern | ||||
2861 | // and SC is a constant vector = <C', C', C', ..., C'> | ||||
2862 | // Into: | ||||
2863 | // %E = extractelement <M x iK> %vec, i32 C' | ||||
2864 | // icmp <pred> iK %E, trunc(C) | ||||
2865 | const APInt *C; | ||||
2866 | if (!match(Cmp.getOperand(1), m_APInt(C)) || | ||||
2867 | !Bitcast->getType()->isIntegerTy() || | ||||
2868 | !Bitcast->getSrcTy()->isIntOrIntVectorTy()) | ||||
2869 | return nullptr; | ||||
2870 | |||||
2871 | Value *Vec; | ||||
2872 | ArrayRef<int> Mask; | ||||
2873 | if (match(BCSrcOp, m_Shuffle(m_Value(Vec), m_Undef(), m_Mask(Mask)))) { | ||||
2874 | // Check whether every element of Mask is the same constant | ||||
2875 | if (is_splat(Mask)) { | ||||
2876 | auto *VecTy = cast<VectorType>(BCSrcOp->getType()); | ||||
2877 | auto *EltTy = cast<IntegerType>(VecTy->getElementType()); | ||||
2878 | if (C->isSplat(EltTy->getBitWidth())) { | ||||
2879 | // Fold the icmp based on the value of C | ||||
2880 | // If C is M copies of an iK sized bit pattern, | ||||
2881 | // then: | ||||
2882 | // => %E = extractelement <N x iK> %vec, i32 Elem | ||||
2883 | // icmp <pred> iK %SplatVal, <pattern> | ||||
2884 | Value *Elem = Builder.getInt32(Mask[0]); | ||||
2885 | Value *Extract = Builder.CreateExtractElement(Vec, Elem); | ||||
2886 | Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth())); | ||||
2887 | return new ICmpInst(Pred, Extract, NewC); | ||||
2888 | } | ||||
2889 | } | ||||
2890 | } | ||||
2891 | return nullptr; | ||||
2892 | } | ||||
2893 | |||||
2894 | /// Try to fold integer comparisons with a constant operand: icmp Pred X, C | ||||
2895 | /// where X is some kind of instruction. | ||||
2896 | Instruction *InstCombinerImpl::foldICmpInstWithConstant(ICmpInst &Cmp) { | ||||
2897 | const APInt *C; | ||||
2898 | if (!match(Cmp.getOperand(1), m_APInt(C))) | ||||
2899 | return nullptr; | ||||
2900 | |||||
2901 | if (auto *BO = dyn_cast<BinaryOperator>(Cmp.getOperand(0))) { | ||||
2902 | switch (BO->getOpcode()) { | ||||
2903 | case Instruction::Xor: | ||||
2904 | if (Instruction *I = foldICmpXorConstant(Cmp, BO, *C)) | ||||
2905 | return I; | ||||
2906 | break; | ||||
2907 | case Instruction::And: | ||||
2908 | if (Instruction *I = foldICmpAndConstant(Cmp, BO, *C)) | ||||
2909 | return I; | ||||
2910 | break; | ||||
2911 | case Instruction::Or: | ||||
2912 | if (Instruction *I = foldICmpOrConstant(Cmp, BO, *C)) | ||||
2913 | return I; | ||||
2914 | break; | ||||
2915 | case Instruction::Mul: | ||||
2916 | if (Instruction *I = foldICmpMulConstant(Cmp, BO, *C)) | ||||
2917 | return I; | ||||
2918 | break; | ||||
2919 | case Instruction::Shl: | ||||
2920 | if (Instruction *I = foldICmpShlConstant(Cmp, BO, *C)) | ||||
2921 | return I; | ||||
2922 | break; | ||||
2923 | case Instruction::LShr: | ||||
2924 | case Instruction::AShr: | ||||
2925 | if (Instruction *I = foldICmpShrConstant(Cmp, BO, *C)) | ||||
2926 | return I; | ||||
2927 | break; | ||||
2928 | case Instruction::SRem: | ||||
2929 | if (Instruction *I = foldICmpSRemConstant(Cmp, BO, *C)) | ||||
2930 | return I; | ||||
2931 | break; | ||||
2932 | case Instruction::UDiv: | ||||
2933 | if (Instruction *I = foldICmpUDivConstant(Cmp, BO, *C)) | ||||
2934 | return I; | ||||
2935 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||
2936 | case Instruction::SDiv: | ||||
2937 | if (Instruction *I = foldICmpDivConstant(Cmp, BO, *C)) | ||||
2938 | return I; | ||||
2939 | break; | ||||
2940 | case Instruction::Sub: | ||||
2941 | if (Instruction *I = foldICmpSubConstant(Cmp, BO, *C)) | ||||
2942 | return I; | ||||
2943 | break; | ||||
2944 | case Instruction::Add: | ||||
2945 | if (Instruction *I = foldICmpAddConstant(Cmp, BO, *C)) | ||||
2946 | return I; | ||||
2947 | break; | ||||
2948 | default: | ||||
2949 | break; | ||||
2950 | } | ||||
2951 | // TODO: These folds could be refactored to be part of the above calls. | ||||
2952 | if (Instruction *I = foldICmpBinOpEqualityWithConstant(Cmp, BO, *C)) | ||||
2953 | return I; | ||||
2954 | } | ||||
2955 | |||||
2956 | // Match against CmpInst LHS being instructions other than binary operators. | ||||
2957 | |||||
2958 | if (auto *SI = dyn_cast<SelectInst>(Cmp.getOperand(0))) { | ||||
2959 | // For now, we only support constant integers while folding the | ||||
2960 | // ICMP(SELECT)) pattern. We can extend this to support vector of integers | ||||
2961 | // similar to the cases handled by binary ops above. | ||||
2962 | if (ConstantInt *ConstRHS = dyn_cast<ConstantInt>(Cmp.getOperand(1))) | ||||
2963 | if (Instruction *I = foldICmpSelectConstant(Cmp, SI, ConstRHS)) | ||||
2964 | return I; | ||||
2965 | } | ||||
2966 | |||||
2967 | if (auto *TI = dyn_cast<TruncInst>(Cmp.getOperand(0))) { | ||||
2968 | if (Instruction *I = foldICmpTruncConstant(Cmp, TI, *C)) | ||||
2969 | return I; | ||||
2970 | } | ||||
2971 | |||||
2972 | if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0))) | ||||
2973 | if (Instruction *I = foldICmpIntrinsicWithConstant(Cmp, II, *C)) | ||||
2974 | return I; | ||||
2975 | |||||
2976 | return nullptr; | ||||
2977 | } | ||||
2978 | |||||
2979 | /// Fold an icmp equality instruction with binary operator LHS and constant RHS: | ||||
2980 | /// icmp eq/ne BO, C. | ||||
2981 | Instruction *InstCombinerImpl::foldICmpBinOpEqualityWithConstant( | ||||
2982 | ICmpInst &Cmp, BinaryOperator *BO, const APInt &C) { | ||||
2983 | // TODO: Some of these folds could work with arbitrary constants, but this | ||||
2984 | // function is limited to scalar and vector splat constants. | ||||
2985 | if (!Cmp.isEquality()) | ||||
2986 | return nullptr; | ||||
2987 | |||||
2988 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
2989 | bool isICMP_NE = Pred == ICmpInst::ICMP_NE; | ||||
2990 | Constant *RHS = cast<Constant>(Cmp.getOperand(1)); | ||||
2991 | Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1); | ||||
2992 | |||||
2993 | switch (BO->getOpcode()) { | ||||
2994 | case Instruction::SRem: | ||||
2995 | // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one. | ||||
2996 | if (C.isNullValue() && BO->hasOneUse()) { | ||||
2997 | const APInt *BOC; | ||||
2998 | if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) { | ||||
2999 | Value *NewRem = Builder.CreateURem(BOp0, BOp1, BO->getName()); | ||||
3000 | return new ICmpInst(Pred, NewRem, | ||||
3001 | Constant::getNullValue(BO->getType())); | ||||
3002 | } | ||||
3003 | } | ||||
3004 | break; | ||||
3005 | case Instruction::Add: { | ||||
3006 | // Replace ((add A, B) != C) with (A != C-B) if B & C are constants. | ||||
3007 | if (Constant *BOC = dyn_cast<Constant>(BOp1)) { | ||||
3008 | if (BO->hasOneUse()) | ||||
3009 | return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(RHS, BOC)); | ||||
3010 | } else if (C.isNullValue()) { | ||||
3011 | // Replace ((add A, B) != 0) with (A != -B) if A or B is | ||||
3012 | // efficiently invertible, or if the add has just this one use. | ||||
3013 | if (Value *NegVal = dyn_castNegVal(BOp1)) | ||||
3014 | return new ICmpInst(Pred, BOp0, NegVal); | ||||
3015 | if (Value *NegVal = dyn_castNegVal(BOp0)) | ||||
3016 | return new ICmpInst(Pred, NegVal, BOp1); | ||||
3017 | if (BO->hasOneUse()) { | ||||
3018 | Value *Neg = Builder.CreateNeg(BOp1); | ||||
3019 | Neg->takeName(BO); | ||||
3020 | return new ICmpInst(Pred, BOp0, Neg); | ||||
3021 | } | ||||
3022 | } | ||||
3023 | break; | ||||
3024 | } | ||||
3025 | case Instruction::Xor: | ||||
3026 | if (BO->hasOneUse()) { | ||||
3027 | if (Constant *BOC = dyn_cast<Constant>(BOp1)) { | ||||
3028 | // For the xor case, we can xor two constants together, eliminating | ||||
3029 | // the explicit xor. | ||||
3030 | return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC)); | ||||
3031 | } else if (C.isNullValue()) { | ||||
3032 | // Replace ((xor A, B) != 0) with (A != B) | ||||
3033 | return new ICmpInst(Pred, BOp0, BOp1); | ||||
3034 | } | ||||
3035 | } | ||||
3036 | break; | ||||
3037 | case Instruction::Sub: | ||||
3038 | if (BO->hasOneUse()) { | ||||
3039 | // Only check for constant LHS here, as constant RHS will be canonicalized | ||||
3040 | // to add and use the fold above. | ||||
3041 | if (Constant *BOC = dyn_cast<Constant>(BOp0)) { | ||||
3042 | // Replace ((sub BOC, B) != C) with (B != BOC-C). | ||||
3043 | return new ICmpInst(Pred, BOp1, ConstantExpr::getSub(BOC, RHS)); | ||||
3044 | } else if (C.isNullValue()) { | ||||
3045 | // Replace ((sub A, B) != 0) with (A != B). | ||||
3046 | return new ICmpInst(Pred, BOp0, BOp1); | ||||
3047 | } | ||||
3048 | } | ||||
3049 | break; | ||||
3050 | case Instruction::Or: { | ||||
3051 | const APInt *BOC; | ||||
3052 | if (match(BOp1, m_APInt(BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) { | ||||
3053 | // Comparing if all bits outside of a constant mask are set? | ||||
3054 | // Replace (X | C) == -1 with (X & ~C) == ~C. | ||||
3055 | // This removes the -1 constant. | ||||
3056 | Constant *NotBOC = ConstantExpr::getNot(cast<Constant>(BOp1)); | ||||
3057 | Value *And = Builder.CreateAnd(BOp0, NotBOC); | ||||
3058 | return new ICmpInst(Pred, And, NotBOC); | ||||
3059 | } | ||||
3060 | break; | ||||
3061 | } | ||||
3062 | case Instruction::And: { | ||||
3063 | const APInt *BOC; | ||||
3064 | if (match(BOp1, m_APInt(BOC))) { | ||||
3065 | // If we have ((X & C) == C), turn it into ((X & C) != 0). | ||||
3066 | if (C == *BOC && C.isPowerOf2()) | ||||
3067 | return new ICmpInst(isICMP_NE ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE, | ||||
3068 | BO, Constant::getNullValue(RHS->getType())); | ||||
3069 | } | ||||
3070 | break; | ||||
3071 | } | ||||
3072 | case Instruction::UDiv: | ||||
3073 | if (C.isNullValue()) { | ||||
3074 | // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A) | ||||
3075 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; | ||||
3076 | return new ICmpInst(NewPred, BOp1, BOp0); | ||||
3077 | } | ||||
3078 | break; | ||||
3079 | default: | ||||
3080 | break; | ||||
3081 | } | ||||
3082 | return nullptr; | ||||
3083 | } | ||||
3084 | |||||
3085 | /// Fold an equality icmp with LLVM intrinsic and constant operand. | ||||
3086 | Instruction *InstCombinerImpl::foldICmpEqIntrinsicWithConstant( | ||||
3087 | ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) { | ||||
3088 | Type *Ty = II->getType(); | ||||
3089 | unsigned BitWidth = C.getBitWidth(); | ||||
3090 | switch (II->getIntrinsicID()) { | ||||
3091 | case Intrinsic::abs: | ||||
3092 | // abs(A) == 0 -> A == 0 | ||||
3093 | // abs(A) == INT_MIN -> A == INT_MIN | ||||
3094 | if (C.isNullValue() || C.isMinSignedValue()) | ||||
3095 | return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), | ||||
3096 | ConstantInt::get(Ty, C)); | ||||
3097 | break; | ||||
3098 | |||||
3099 | case Intrinsic::bswap: | ||||
3100 | // bswap(A) == C -> A == bswap(C) | ||||
3101 | return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), | ||||
3102 | ConstantInt::get(Ty, C.byteSwap())); | ||||
3103 | |||||
3104 | case Intrinsic::ctlz: | ||||
3105 | case Intrinsic::cttz: { | ||||
3106 | // ctz(A) == bitwidth(A) -> A == 0 and likewise for != | ||||
3107 | if (C == BitWidth) | ||||
3108 | return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), | ||||
3109 | ConstantInt::getNullValue(Ty)); | ||||
3110 | |||||
3111 | // ctz(A) == C -> A & Mask1 == Mask2, where Mask2 only has bit C set | ||||
3112 | // and Mask1 has bits 0..C+1 set. Similar for ctl, but for high bits. | ||||
3113 | // Limit to one use to ensure we don't increase instruction count. | ||||
3114 | unsigned Num = C.getLimitedValue(BitWidth); | ||||
3115 | if (Num != BitWidth && II->hasOneUse()) { | ||||
3116 | bool IsTrailing = II->getIntrinsicID() == Intrinsic::cttz; | ||||
3117 | APInt Mask1 = IsTrailing ? APInt::getLowBitsSet(BitWidth, Num + 1) | ||||
3118 | : APInt::getHighBitsSet(BitWidth, Num + 1); | ||||
3119 | APInt Mask2 = IsTrailing | ||||
3120 | ? APInt::getOneBitSet(BitWidth, Num) | ||||
3121 | : APInt::getOneBitSet(BitWidth, BitWidth - Num - 1); | ||||
3122 | return new ICmpInst(Cmp.getPredicate(), | ||||
3123 | Builder.CreateAnd(II->getArgOperand(0), Mask1), | ||||
3124 | ConstantInt::get(Ty, Mask2)); | ||||
3125 | } | ||||
3126 | break; | ||||
3127 | } | ||||
3128 | |||||
3129 | case Intrinsic::ctpop: { | ||||
3130 | // popcount(A) == 0 -> A == 0 and likewise for != | ||||
3131 | // popcount(A) == bitwidth(A) -> A == -1 and likewise for != | ||||
3132 | bool IsZero = C.isNullValue(); | ||||
3133 | if (IsZero || C == BitWidth) | ||||
3134 | return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), | ||||
3135 | IsZero ? Constant::getNullValue(Ty) : Constant::getAllOnesValue(Ty)); | ||||
3136 | |||||
3137 | break; | ||||
3138 | } | ||||
3139 | |||||
3140 | case Intrinsic::uadd_sat: { | ||||
3141 | // uadd.sat(a, b) == 0 -> (a | b) == 0 | ||||
3142 | if (C.isNullValue()) { | ||||
3143 | Value *Or = Builder.CreateOr(II->getArgOperand(0), II->getArgOperand(1)); | ||||
3144 | return new ICmpInst(Cmp.getPredicate(), Or, Constant::getNullValue(Ty)); | ||||
3145 | } | ||||
3146 | break; | ||||
3147 | } | ||||
3148 | |||||
3149 | case Intrinsic::usub_sat: { | ||||
3150 | // usub.sat(a, b) == 0 -> a <= b | ||||
3151 | if (C.isNullValue()) { | ||||
3152 | ICmpInst::Predicate NewPred = Cmp.getPredicate() == ICmpInst::ICMP_EQ | ||||
3153 | ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; | ||||
3154 | return new ICmpInst(NewPred, II->getArgOperand(0), II->getArgOperand(1)); | ||||
3155 | } | ||||
3156 | break; | ||||
3157 | } | ||||
3158 | default: | ||||
3159 | break; | ||||
3160 | } | ||||
3161 | |||||
3162 | return nullptr; | ||||
3163 | } | ||||
3164 | |||||
3165 | /// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C. | ||||
3166 | Instruction *InstCombinerImpl::foldICmpIntrinsicWithConstant(ICmpInst &Cmp, | ||||
3167 | IntrinsicInst *II, | ||||
3168 | const APInt &C) { | ||||
3169 | if (Cmp.isEquality()) | ||||
3170 | return foldICmpEqIntrinsicWithConstant(Cmp, II, C); | ||||
3171 | |||||
3172 | Type *Ty = II->getType(); | ||||
3173 | unsigned BitWidth = C.getBitWidth(); | ||||
3174 | switch (II->getIntrinsicID()) { | ||||
3175 | case Intrinsic::ctlz: { | ||||
3176 | // ctlz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX < 0b00010000 | ||||
3177 | if (Cmp.getPredicate() == ICmpInst::ICMP_UGT && C.ult(BitWidth)) { | ||||
3178 | unsigned Num = C.getLimitedValue(); | ||||
3179 | APInt Limit = APInt::getOneBitSet(BitWidth, BitWidth - Num - 1); | ||||
3180 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT, | ||||
3181 | II->getArgOperand(0), ConstantInt::get(Ty, Limit)); | ||||
3182 | } | ||||
3183 | |||||
3184 | // ctlz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX > 0b00011111 | ||||
3185 | if (Cmp.getPredicate() == ICmpInst::ICMP_ULT && | ||||
3186 | C.uge(1) && C.ule(BitWidth)) { | ||||
3187 | unsigned Num = C.getLimitedValue(); | ||||
3188 | APInt Limit = APInt::getLowBitsSet(BitWidth, BitWidth - Num); | ||||
3189 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, | ||||
3190 | II->getArgOperand(0), ConstantInt::get(Ty, Limit)); | ||||
3191 | } | ||||
3192 | break; | ||||
3193 | } | ||||
3194 | case Intrinsic::cttz: { | ||||
3195 | // Limit to one use to ensure we don't increase instruction count. | ||||
3196 | if (!II->hasOneUse()) | ||||
3197 | return nullptr; | ||||
3198 | |||||
3199 | // cttz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX & 0b00001111 == 0 | ||||
3200 | if (Cmp.getPredicate() == ICmpInst::ICMP_UGT && C.ult(BitWidth)) { | ||||
3201 | APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue() + 1); | ||||
3202 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, | ||||
3203 | Builder.CreateAnd(II->getArgOperand(0), Mask), | ||||
3204 | ConstantInt::getNullValue(Ty)); | ||||
3205 | } | ||||
3206 | |||||
3207 | // cttz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX & 0b00000111 != 0 | ||||
3208 | if (Cmp.getPredicate() == ICmpInst::ICMP_ULT && | ||||
3209 | C.uge(1) && C.ule(BitWidth)) { | ||||
3210 | APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue()); | ||||
3211 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, | ||||
3212 | Builder.CreateAnd(II->getArgOperand(0), Mask), | ||||
3213 | ConstantInt::getNullValue(Ty)); | ||||
3214 | } | ||||
3215 | break; | ||||
3216 | } | ||||
3217 | default: | ||||
3218 | break; | ||||
3219 | } | ||||
3220 | |||||
3221 | return nullptr; | ||||
3222 | } | ||||
3223 | |||||
3224 | /// Handle icmp with constant (but not simple integer constant) RHS. | ||||
3225 | Instruction *InstCombinerImpl::foldICmpInstWithConstantNotInt(ICmpInst &I) { | ||||
3226 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||
3227 | Constant *RHSC = dyn_cast<Constant>(Op1); | ||||
3228 | Instruction *LHSI = dyn_cast<Instruction>(Op0); | ||||
3229 | if (!RHSC || !LHSI) | ||||
3230 | return nullptr; | ||||
3231 | |||||
3232 | switch (LHSI->getOpcode()) { | ||||
3233 | case Instruction::GetElementPtr: | ||||
3234 | // icmp pred GEP (P, int 0, int 0, int 0), null -> icmp pred P, null | ||||
3235 | if (RHSC->isNullValue() && | ||||
3236 | cast<GetElementPtrInst>(LHSI)->hasAllZeroIndices()) | ||||
3237 | return new ICmpInst( | ||||
3238 | I.getPredicate(), LHSI->getOperand(0), | ||||
3239 | Constant::getNullValue(LHSI->getOperand(0)->getType())); | ||||
3240 | break; | ||||
3241 | case Instruction::PHI: | ||||
3242 | // Only fold icmp into the PHI if the phi and icmp are in the same | ||||
3243 | // block. If in the same block, we're encouraging jump threading. If | ||||
3244 | // not, we are just pessimizing the code by making an i1 phi. | ||||
3245 | if (LHSI->getParent() == I.getParent()) | ||||
3246 | if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI))) | ||||
3247 | return NV; | ||||
3248 | break; | ||||
3249 | case Instruction::Select: { | ||||
3250 | // If either operand of the select is a constant, we can fold the | ||||
3251 | // comparison into the select arms, which will cause one to be | ||||
3252 | // constant folded and the select turned into a bitwise or. | ||||
3253 | Value *Op1 = nullptr, *Op2 = nullptr; | ||||
3254 | ConstantInt *CI = nullptr; | ||||
3255 | if (Constant *C = dyn_cast<Constant>(LHSI->getOperand(1))) { | ||||
3256 | Op1 = ConstantExpr::getICmp(I.getPredicate(), C, RHSC); | ||||
3257 | CI = dyn_cast<ConstantInt>(Op1); | ||||
3258 | } | ||||
3259 | if (Constant *C = dyn_cast<Constant>(LHSI->getOperand(2))) { | ||||
3260 | Op2 = ConstantExpr::getICmp(I.getPredicate(), C, RHSC); | ||||
3261 | CI = dyn_cast<ConstantInt>(Op2); | ||||
3262 | } | ||||
3263 | |||||
3264 | // We only want to perform this transformation if it will not lead to | ||||
3265 | // additional code. This is true if either both sides of the select | ||||
3266 | // fold to a constant (in which case the icmp is replaced with a select | ||||
3267 | // which will usually simplify) or this is the only user of the | ||||
3268 | // select (in which case we are trading a select+icmp for a simpler | ||||
3269 | // select+icmp) or all uses of the select can be replaced based on | ||||
3270 | // dominance information ("Global cases"). | ||||
3271 | bool Transform = false; | ||||
3272 | if (Op1 && Op2) | ||||
3273 | Transform = true; | ||||
3274 | else if (Op1 || Op2) { | ||||
3275 | // Local case | ||||
3276 | if (LHSI->hasOneUse()) | ||||
3277 | Transform = true; | ||||
3278 | // Global cases | ||||
3279 | else if (CI && !CI->isZero()) | ||||
3280 | // When Op1 is constant try replacing select with second operand. | ||||
3281 | // Otherwise Op2 is constant and try replacing select with first | ||||
3282 | // operand. | ||||
3283 | Transform = | ||||
3284 | replacedSelectWithOperand(cast<SelectInst>(LHSI), &I, Op1 ? 2 : 1); | ||||
3285 | } | ||||
3286 | if (Transform) { | ||||
3287 | if (!Op1) | ||||
3288 | Op1 = Builder.CreateICmp(I.getPredicate(), LHSI->getOperand(1), RHSC, | ||||
3289 | I.getName()); | ||||
3290 | if (!Op2) | ||||
3291 | Op2 = Builder.CreateICmp(I.getPredicate(), LHSI->getOperand(2), RHSC, | ||||
3292 | I.getName()); | ||||
3293 | return SelectInst::Create(LHSI->getOperand(0), Op1, Op2); | ||||
3294 | } | ||||
3295 | break; | ||||
3296 | } | ||||
3297 | case Instruction::IntToPtr: | ||||
3298 | // icmp pred inttoptr(X), null -> icmp pred X, 0 | ||||
3299 | if (RHSC->isNullValue() && | ||||
3300 | DL.getIntPtrType(RHSC->getType()) == LHSI->getOperand(0)->getType()) | ||||
3301 | return new ICmpInst( | ||||
3302 | I.getPredicate(), LHSI->getOperand(0), | ||||
3303 | Constant::getNullValue(LHSI->getOperand(0)->getType())); | ||||
3304 | break; | ||||
3305 | |||||
3306 | case Instruction::Load: | ||||
3307 | // Try to optimize things like "A[i] > 4" to index computations. | ||||
3308 | if (GetElementPtrInst *GEP = | ||||
3309 | dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) { | ||||
3310 | if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) | ||||
3311 | if (GV->isConstant() && GV->hasDefinitiveInitializer() && | ||||
3312 | !cast<LoadInst>(LHSI)->isVolatile()) | ||||
3313 | if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, I)) | ||||
3314 | return Res; | ||||
3315 | } | ||||
3316 | break; | ||||
3317 | } | ||||
3318 | |||||
3319 | return nullptr; | ||||
3320 | } | ||||
3321 | |||||
3322 | /// Some comparisons can be simplified. | ||||
3323 | /// In this case, we are looking for comparisons that look like | ||||
3324 | /// a check for a lossy truncation. | ||||
3325 | /// Folds: | ||||
3326 | /// icmp SrcPred (x & Mask), x to icmp DstPred x, Mask | ||||
3327 | /// Where Mask is some pattern that produces all-ones in low bits: | ||||
3328 | /// (-1 >> y) | ||||
3329 | /// ((-1 << y) >> y) <- non-canonical, has extra uses | ||||
3330 | /// ~(-1 << y) | ||||
3331 | /// ((1 << y) + (-1)) <- non-canonical, has extra uses | ||||
3332 | /// The Mask can be a constant, too. | ||||
3333 | /// For some predicates, the operands are commutative. | ||||
3334 | /// For others, x can only be on a specific side. | ||||
3335 | static Value *foldICmpWithLowBitMaskedVal(ICmpInst &I, | ||||
3336 | InstCombiner::BuilderTy &Builder) { | ||||
3337 | ICmpInst::Predicate SrcPred; | ||||
3338 | Value *X, *M, *Y; | ||||
3339 | auto m_VariableMask = m_CombineOr( | ||||
3340 | m_CombineOr(m_Not(m_Shl(m_AllOnes(), m_Value())), | ||||
3341 | m_Add(m_Shl(m_One(), m_Value()), m_AllOnes())), | ||||
3342 | m_CombineOr(m_LShr(m_AllOnes(), m_Value()), | ||||
3343 | m_LShr(m_Shl(m_AllOnes(), m_Value(Y)), m_Deferred(Y)))); | ||||
3344 | auto m_Mask = m_CombineOr(m_VariableMask, m_LowBitMask()); | ||||
3345 | if (!match(&I, m_c_ICmp(SrcPred, | ||||
3346 | m_c_And(m_CombineAnd(m_Mask, m_Value(M)), m_Value(X)), | ||||
3347 | m_Deferred(X)))) | ||||
3348 | return nullptr; | ||||
3349 | |||||
3350 | ICmpInst::Predicate DstPred; | ||||
3351 | switch (SrcPred) { | ||||
3352 | case ICmpInst::Predicate::ICMP_EQ: | ||||
3353 | // x & (-1 >> y) == x -> x u<= (-1 >> y) | ||||
3354 | DstPred = ICmpInst::Predicate::ICMP_ULE; | ||||
3355 | break; | ||||
3356 | case ICmpInst::Predicate::ICMP_NE: | ||||
3357 | // x & (-1 >> y) != x -> x u> (-1 >> y) | ||||
3358 | DstPred = ICmpInst::Predicate::ICMP_UGT; | ||||
3359 | break; | ||||
3360 | case ICmpInst::Predicate::ICMP_ULT: | ||||
3361 | // x & (-1 >> y) u< x -> x u> (-1 >> y) | ||||
3362 | // x u> x & (-1 >> y) -> x u> (-1 >> y) | ||||
3363 | DstPred = ICmpInst::Predicate::ICMP_UGT; | ||||
3364 | break; | ||||
3365 | case ICmpInst::Predicate::ICMP_UGE: | ||||
3366 | // x & (-1 >> y) u>= x -> x u<= (-1 >> y) | ||||
3367 | // x u<= x & (-1 >> y) -> x u<= (-1 >> y) | ||||
3368 | DstPred = ICmpInst::Predicate::ICMP_ULE; | ||||
3369 | break; | ||||
3370 | case ICmpInst::Predicate::ICMP_SLT: | ||||
3371 | // x & (-1 >> y) s< x -> x s> (-1 >> y) | ||||
3372 | // x s> x & (-1 >> y) -> x s> (-1 >> y) | ||||
3373 | if (!match(M, m_Constant())) // Can not do this fold with non-constant. | ||||
3374 | return nullptr; | ||||
3375 | if (!match(M, m_NonNegative())) // Must not have any -1 vector elements. | ||||
3376 | return nullptr; | ||||
3377 | DstPred = ICmpInst::Predicate::ICMP_SGT; | ||||
3378 | break; | ||||
3379 | case ICmpInst::Predicate::ICMP_SGE: | ||||
3380 | // x & (-1 >> y) s>= x -> x s<= (-1 >> y) | ||||
3381 | // x s<= x & (-1 >> y) -> x s<= (-1 >> y) | ||||
3382 | if (!match(M, m_Constant())) // Can not do this fold with non-constant. | ||||
3383 | return nullptr; | ||||
3384 | if (!match(M, m_NonNegative())) // Must not have any -1 vector elements. | ||||
3385 | return nullptr; | ||||
3386 | DstPred = ICmpInst::Predicate::ICMP_SLE; | ||||
3387 | break; | ||||
3388 | case ICmpInst::Predicate::ICMP_SGT: | ||||
3389 | case ICmpInst::Predicate::ICMP_SLE: | ||||
3390 | return nullptr; | ||||
3391 | case ICmpInst::Predicate::ICMP_UGT: | ||||
3392 | case ICmpInst::Predicate::ICMP_ULE: | ||||
3393 | llvm_unreachable("Instsimplify took care of commut. variant")::llvm::llvm_unreachable_internal("Instsimplify took care of commut. variant" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3393); | ||||
3394 | break; | ||||
3395 | default: | ||||
3396 | llvm_unreachable("All possible folds are handled.")::llvm::llvm_unreachable_internal("All possible folds are handled." , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3396); | ||||
3397 | } | ||||
3398 | |||||
3399 | // The mask value may be a vector constant that has undefined elements. But it | ||||
3400 | // may not be safe to propagate those undefs into the new compare, so replace | ||||
3401 | // those elements by copying an existing, defined, and safe scalar constant. | ||||
3402 | Type *OpTy = M->getType(); | ||||
3403 | auto *VecC = dyn_cast<Constant>(M); | ||||
3404 | if (OpTy->isVectorTy() && VecC && VecC->containsUndefElement()) { | ||||
3405 | auto *OpVTy = cast<FixedVectorType>(OpTy); | ||||
3406 | Constant *SafeReplacementConstant = nullptr; | ||||
3407 | for (unsigned i = 0, e = OpVTy->getNumElements(); i != e; ++i) { | ||||
3408 | if (!isa<UndefValue>(VecC->getAggregateElement(i))) { | ||||
3409 | SafeReplacementConstant = VecC->getAggregateElement(i); | ||||
3410 | break; | ||||
3411 | } | ||||
3412 | } | ||||
3413 | assert(SafeReplacementConstant && "Failed to find undef replacement")((SafeReplacementConstant && "Failed to find undef replacement" ) ? static_cast<void> (0) : __assert_fail ("SafeReplacementConstant && \"Failed to find undef replacement\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3413, __PRETTY_FUNCTION__)); | ||||
3414 | M = Constant::replaceUndefsWith(VecC, SafeReplacementConstant); | ||||
3415 | } | ||||
3416 | |||||
3417 | return Builder.CreateICmp(DstPred, X, M); | ||||
3418 | } | ||||
3419 | |||||
3420 | /// Some comparisons can be simplified. | ||||
3421 | /// In this case, we are looking for comparisons that look like | ||||
3422 | /// a check for a lossy signed truncation. | ||||
3423 | /// Folds: (MaskedBits is a constant.) | ||||
3424 | /// ((%x << MaskedBits) a>> MaskedBits) SrcPred %x | ||||
3425 | /// Into: | ||||
3426 | /// (add %x, (1 << (KeptBits-1))) DstPred (1 << KeptBits) | ||||
3427 | /// Where KeptBits = bitwidth(%x) - MaskedBits | ||||
3428 | static Value * | ||||
3429 | foldICmpWithTruncSignExtendedVal(ICmpInst &I, | ||||
3430 | InstCombiner::BuilderTy &Builder) { | ||||
3431 | ICmpInst::Predicate SrcPred; | ||||
3432 | Value *X; | ||||
3433 | const APInt *C0, *C1; // FIXME: non-splats, potentially with undef. | ||||
3434 | // We are ok with 'shl' having multiple uses, but 'ashr' must be one-use. | ||||
3435 | if (!match(&I, m_c_ICmp(SrcPred, | ||||
3436 | m_OneUse(m_AShr(m_Shl(m_Value(X), m_APInt(C0)), | ||||
3437 | m_APInt(C1))), | ||||
3438 | m_Deferred(X)))) | ||||
3439 | return nullptr; | ||||
3440 | |||||
3441 | // Potential handling of non-splats: for each element: | ||||
3442 | // * if both are undef, replace with constant 0. | ||||
3443 | // Because (1<<0) is OK and is 1, and ((1<<0)>>1) is also OK and is 0. | ||||
3444 | // * if both are not undef, and are different, bailout. | ||||
3445 | // * else, only one is undef, then pick the non-undef one. | ||||
3446 | |||||
3447 | // The shift amount must be equal. | ||||
3448 | if (*C0 != *C1) | ||||
3449 | return nullptr; | ||||
3450 | const APInt &MaskedBits = *C0; | ||||
3451 | assert(MaskedBits != 0 && "shift by zero should be folded away already.")((MaskedBits != 0 && "shift by zero should be folded away already." ) ? static_cast<void> (0) : __assert_fail ("MaskedBits != 0 && \"shift by zero should be folded away already.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3451, __PRETTY_FUNCTION__)); | ||||
3452 | |||||
3453 | ICmpInst::Predicate DstPred; | ||||
3454 | switch (SrcPred) { | ||||
3455 | case ICmpInst::Predicate::ICMP_EQ: | ||||
3456 | // ((%x << MaskedBits) a>> MaskedBits) == %x | ||||
3457 | // => | ||||
3458 | // (add %x, (1 << (KeptBits-1))) u< (1 << KeptBits) | ||||
3459 | DstPred = ICmpInst::Predicate::ICMP_ULT; | ||||
3460 | break; | ||||
3461 | case ICmpInst::Predicate::ICMP_NE: | ||||
3462 | // ((%x << MaskedBits) a>> MaskedBits) != %x | ||||
3463 | // => | ||||
3464 | // (add %x, (1 << (KeptBits-1))) u>= (1 << KeptBits) | ||||
3465 | DstPred = ICmpInst::Predicate::ICMP_UGE; | ||||
3466 | break; | ||||
3467 | // FIXME: are more folds possible? | ||||
3468 | default: | ||||
3469 | return nullptr; | ||||
3470 | } | ||||
3471 | |||||
3472 | auto *XType = X->getType(); | ||||
3473 | const unsigned XBitWidth = XType->getScalarSizeInBits(); | ||||
3474 | const APInt BitWidth = APInt(XBitWidth, XBitWidth); | ||||
3475 | assert(BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched")((BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched" ) ? static_cast<void> (0) : __assert_fail ("BitWidth.ugt(MaskedBits) && \"shifts should leave some bits untouched\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3475, __PRETTY_FUNCTION__)); | ||||
3476 | |||||
3477 | // KeptBits = bitwidth(%x) - MaskedBits | ||||
3478 | const APInt KeptBits = BitWidth - MaskedBits; | ||||
3479 | assert(KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable")((KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable") ? static_cast<void> (0) : __assert_fail ("KeptBits.ugt(0) && KeptBits.ult(BitWidth) && \"unreachable\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3479, __PRETTY_FUNCTION__)); | ||||
3480 | // ICmpCst = (1 << KeptBits) | ||||
3481 | const APInt ICmpCst = APInt(XBitWidth, 1).shl(KeptBits); | ||||
3482 | assert(ICmpCst.isPowerOf2())((ICmpCst.isPowerOf2()) ? static_cast<void> (0) : __assert_fail ("ICmpCst.isPowerOf2()", "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3482, __PRETTY_FUNCTION__)); | ||||
3483 | // AddCst = (1 << (KeptBits-1)) | ||||
3484 | const APInt AddCst = ICmpCst.lshr(1); | ||||
3485 | assert(AddCst.ult(ICmpCst) && AddCst.isPowerOf2())((AddCst.ult(ICmpCst) && AddCst.isPowerOf2()) ? static_cast <void> (0) : __assert_fail ("AddCst.ult(ICmpCst) && AddCst.isPowerOf2()" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3485, __PRETTY_FUNCTION__)); | ||||
3486 | |||||
3487 | // T0 = add %x, AddCst | ||||
3488 | Value *T0 = Builder.CreateAdd(X, ConstantInt::get(XType, AddCst)); | ||||
3489 | // T1 = T0 DstPred ICmpCst | ||||
3490 | Value *T1 = Builder.CreateICmp(DstPred, T0, ConstantInt::get(XType, ICmpCst)); | ||||
3491 | |||||
3492 | return T1; | ||||
3493 | } | ||||
3494 | |||||
3495 | // Given pattern: | ||||
3496 | // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0 | ||||
3497 | // we should move shifts to the same hand of 'and', i.e. rewrite as | ||||
3498 | // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x) | ||||
3499 | // We are only interested in opposite logical shifts here. | ||||
3500 | // One of the shifts can be truncated. | ||||
3501 | // If we can, we want to end up creating 'lshr' shift. | ||||
3502 | static Value * | ||||
3503 | foldShiftIntoShiftInAnotherHandOfAndInICmp(ICmpInst &I, const SimplifyQuery SQ, | ||||
3504 | InstCombiner::BuilderTy &Builder) { | ||||
3505 | if (!I.isEquality() || !match(I.getOperand(1), m_Zero()) || | ||||
3506 | !I.getOperand(0)->hasOneUse()) | ||||
3507 | return nullptr; | ||||
3508 | |||||
3509 | auto m_AnyLogicalShift = m_LogicalShift(m_Value(), m_Value()); | ||||
3510 | |||||
3511 | // Look for an 'and' of two logical shifts, one of which may be truncated. | ||||
3512 | // We use m_TruncOrSelf() on the RHS to correctly handle commutative case. | ||||
3513 | Instruction *XShift, *MaybeTruncation, *YShift; | ||||
3514 | if (!match( | ||||
3515 | I.getOperand(0), | ||||
3516 | m_c_And(m_CombineAnd(m_AnyLogicalShift, m_Instruction(XShift)), | ||||
3517 | m_CombineAnd(m_TruncOrSelf(m_CombineAnd( | ||||
3518 | m_AnyLogicalShift, m_Instruction(YShift))), | ||||
3519 | m_Instruction(MaybeTruncation))))) | ||||
3520 | return nullptr; | ||||
3521 | |||||
3522 | // We potentially looked past 'trunc', but only when matching YShift, | ||||
3523 | // therefore YShift must have the widest type. | ||||
3524 | Instruction *WidestShift = YShift; | ||||
3525 | // Therefore XShift must have the shallowest type. | ||||
3526 | // Or they both have identical types if there was no truncation. | ||||
3527 | Instruction *NarrowestShift = XShift; | ||||
3528 | |||||
3529 | Type *WidestTy = WidestShift->getType(); | ||||
3530 | Type *NarrowestTy = NarrowestShift->getType(); | ||||
3531 | assert(NarrowestTy == I.getOperand(0)->getType() &&((NarrowestTy == I.getOperand(0)->getType() && "We did not look past any shifts while matching XShift though." ) ? static_cast<void> (0) : __assert_fail ("NarrowestTy == I.getOperand(0)->getType() && \"We did not look past any shifts while matching XShift though.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3532, __PRETTY_FUNCTION__)) | ||||
3532 | "We did not look past any shifts while matching XShift though.")((NarrowestTy == I.getOperand(0)->getType() && "We did not look past any shifts while matching XShift though." ) ? static_cast<void> (0) : __assert_fail ("NarrowestTy == I.getOperand(0)->getType() && \"We did not look past any shifts while matching XShift though.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3532, __PRETTY_FUNCTION__)); | ||||
3533 | bool HadTrunc = WidestTy != I.getOperand(0)->getType(); | ||||
3534 | |||||
3535 | // If YShift is a 'lshr', swap the shifts around. | ||||
3536 | if (match(YShift, m_LShr(m_Value(), m_Value()))) | ||||
3537 | std::swap(XShift, YShift); | ||||
3538 | |||||
3539 | // The shifts must be in opposite directions. | ||||
3540 | auto XShiftOpcode = XShift->getOpcode(); | ||||
3541 | if (XShiftOpcode == YShift->getOpcode()) | ||||
3542 | return nullptr; // Do not care about same-direction shifts here. | ||||
3543 | |||||
3544 | Value *X, *XShAmt, *Y, *YShAmt; | ||||
3545 | match(XShift, m_BinOp(m_Value(X), m_ZExtOrSelf(m_Value(XShAmt)))); | ||||
3546 | match(YShift, m_BinOp(m_Value(Y), m_ZExtOrSelf(m_Value(YShAmt)))); | ||||
3547 | |||||
3548 | // If one of the values being shifted is a constant, then we will end with | ||||
3549 | // and+icmp, and [zext+]shift instrs will be constant-folded. If they are not, | ||||
3550 | // however, we will need to ensure that we won't increase instruction count. | ||||
3551 | if (!isa<Constant>(X) && !isa<Constant>(Y)) { | ||||
3552 | // At least one of the hands of the 'and' should be one-use shift. | ||||
3553 | if (!match(I.getOperand(0), | ||||
3554 | m_c_And(m_OneUse(m_AnyLogicalShift), m_Value()))) | ||||
3555 | return nullptr; | ||||
3556 | if (HadTrunc) { | ||||
3557 | // Due to the 'trunc', we will need to widen X. For that either the old | ||||
3558 | // 'trunc' or the shift amt in the non-truncated shift should be one-use. | ||||
3559 | if (!MaybeTruncation->hasOneUse() && | ||||
3560 | !NarrowestShift->getOperand(1)->hasOneUse()) | ||||
3561 | return nullptr; | ||||
3562 | } | ||||
3563 | } | ||||
3564 | |||||
3565 | // We have two shift amounts from two different shifts. The types of those | ||||
3566 | // shift amounts may not match. If that's the case let's bailout now. | ||||
3567 | if (XShAmt->getType() != YShAmt->getType()) | ||||
3568 | return nullptr; | ||||
3569 | |||||
3570 | // As input, we have the following pattern: | ||||
3571 | // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0 | ||||
3572 | // We want to rewrite that as: | ||||
3573 | // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x) | ||||
3574 | // While we know that originally (Q+K) would not overflow | ||||
3575 | // (because 2 * (N-1) u<= iN -1), we have looked past extensions of | ||||
3576 | // shift amounts. so it may now overflow in smaller bitwidth. | ||||
3577 | // To ensure that does not happen, we need to ensure that the total maximal | ||||
3578 | // shift amount is still representable in that smaller bit width. | ||||
3579 | unsigned MaximalPossibleTotalShiftAmount = | ||||
3580 | (WidestTy->getScalarSizeInBits() - 1) + | ||||
3581 | (NarrowestTy->getScalarSizeInBits() - 1); | ||||
3582 | APInt MaximalRepresentableShiftAmount = | ||||
3583 | APInt::getAllOnesValue(XShAmt->getType()->getScalarSizeInBits()); | ||||
3584 | if (MaximalRepresentableShiftAmount.ult(MaximalPossibleTotalShiftAmount)) | ||||
3585 | return nullptr; | ||||
3586 | |||||
3587 | // Can we fold (XShAmt+YShAmt) ? | ||||
3588 | auto *NewShAmt = dyn_cast_or_null<Constant>( | ||||
3589 | SimplifyAddInst(XShAmt, YShAmt, /*isNSW=*/false, | ||||
3590 | /*isNUW=*/false, SQ.getWithInstruction(&I))); | ||||
3591 | if (!NewShAmt) | ||||
3592 | return nullptr; | ||||
3593 | NewShAmt = ConstantExpr::getZExtOrBitCast(NewShAmt, WidestTy); | ||||
3594 | unsigned WidestBitWidth = WidestTy->getScalarSizeInBits(); | ||||
3595 | |||||
3596 | // Is the new shift amount smaller than the bit width? | ||||
3597 | // FIXME: could also rely on ConstantRange. | ||||
3598 | if (!match(NewShAmt, | ||||
3599 | m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_ULT, | ||||
3600 | APInt(WidestBitWidth, WidestBitWidth)))) | ||||
3601 | return nullptr; | ||||
3602 | |||||
3603 | // An extra legality check is needed if we had trunc-of-lshr. | ||||
3604 | if (HadTrunc && match(WidestShift, m_LShr(m_Value(), m_Value()))) { | ||||
3605 | auto CanFold = [NewShAmt, WidestBitWidth, NarrowestShift, SQ, | ||||
3606 | WidestShift]() { | ||||
3607 | // It isn't obvious whether it's worth it to analyze non-constants here. | ||||
3608 | // Also, let's basically give up on non-splat cases, pessimizing vectors. | ||||
3609 | // If *any* of these preconditions matches we can perform the fold. | ||||
3610 | Constant *NewShAmtSplat = NewShAmt->getType()->isVectorTy() | ||||
3611 | ? NewShAmt->getSplatValue() | ||||
3612 | : NewShAmt; | ||||
3613 | // If it's edge-case shift (by 0 or by WidestBitWidth-1) we can fold. | ||||
3614 | if (NewShAmtSplat && | ||||
3615 | (NewShAmtSplat->isNullValue() || | ||||
3616 | NewShAmtSplat->getUniqueInteger() == WidestBitWidth - 1)) | ||||
3617 | return true; | ||||
3618 | // We consider *min* leading zeros so a single outlier | ||||
3619 | // blocks the transform as opposed to allowing it. | ||||
3620 | if (auto *C = dyn_cast<Constant>(NarrowestShift->getOperand(0))) { | ||||
3621 | KnownBits Known = computeKnownBits(C, SQ.DL); | ||||
3622 | unsigned MinLeadZero = Known.countMinLeadingZeros(); | ||||
3623 | // If the value being shifted has at most lowest bit set we can fold. | ||||
3624 | unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero; | ||||
3625 | if (MaxActiveBits <= 1) | ||||
3626 | return true; | ||||
3627 | // Precondition: NewShAmt u<= countLeadingZeros(C) | ||||
3628 | if (NewShAmtSplat && NewShAmtSplat->getUniqueInteger().ule(MinLeadZero)) | ||||
3629 | return true; | ||||
3630 | } | ||||
3631 | if (auto *C = dyn_cast<Constant>(WidestShift->getOperand(0))) { | ||||
3632 | KnownBits Known = computeKnownBits(C, SQ.DL); | ||||
3633 | unsigned MinLeadZero = Known.countMinLeadingZeros(); | ||||
3634 | // If the value being shifted has at most lowest bit set we can fold. | ||||
3635 | unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero; | ||||
3636 | if (MaxActiveBits <= 1) | ||||
3637 | return true; | ||||
3638 | // Precondition: ((WidestBitWidth-1)-NewShAmt) u<= countLeadingZeros(C) | ||||
3639 | if (NewShAmtSplat) { | ||||
3640 | APInt AdjNewShAmt = | ||||
3641 | (WidestBitWidth - 1) - NewShAmtSplat->getUniqueInteger(); | ||||
3642 | if (AdjNewShAmt.ule(MinLeadZero)) | ||||
3643 | return true; | ||||
3644 | } | ||||
3645 | } | ||||
3646 | return false; // Can't tell if it's ok. | ||||
3647 | }; | ||||
3648 | if (!CanFold()) | ||||
3649 | return nullptr; | ||||
3650 | } | ||||
3651 | |||||
3652 | // All good, we can do this fold. | ||||
3653 | X = Builder.CreateZExt(X, WidestTy); | ||||
3654 | Y = Builder.CreateZExt(Y, WidestTy); | ||||
3655 | // The shift is the same that was for X. | ||||
3656 | Value *T0 = XShiftOpcode == Instruction::BinaryOps::LShr | ||||
3657 | ? Builder.CreateLShr(X, NewShAmt) | ||||
3658 | : Builder.CreateShl(X, NewShAmt); | ||||
3659 | Value *T1 = Builder.CreateAnd(T0, Y); | ||||
3660 | return Builder.CreateICmp(I.getPredicate(), T1, | ||||
3661 | Constant::getNullValue(WidestTy)); | ||||
3662 | } | ||||
3663 | |||||
3664 | /// Fold | ||||
3665 | /// (-1 u/ x) u< y | ||||
3666 | /// ((x * y) u/ x) != y | ||||
3667 | /// to | ||||
3668 | /// @llvm.umul.with.overflow(x, y) plus extraction of overflow bit | ||||
3669 | /// Note that the comparison is commutative, while inverted (u>=, ==) predicate | ||||
3670 | /// will mean that we are looking for the opposite answer. | ||||
3671 | Value *InstCombinerImpl::foldUnsignedMultiplicationOverflowCheck(ICmpInst &I) { | ||||
3672 | ICmpInst::Predicate Pred; | ||||
3673 | Value *X, *Y; | ||||
3674 | Instruction *Mul; | ||||
3675 | bool NeedNegation; | ||||
3676 | // Look for: (-1 u/ x) u</u>= y | ||||
3677 | if (!I.isEquality() && | ||||
3678 | match(&I, m_c_ICmp(Pred, m_OneUse(m_UDiv(m_AllOnes(), m_Value(X))), | ||||
3679 | m_Value(Y)))) { | ||||
3680 | Mul = nullptr; | ||||
3681 | |||||
3682 | // Are we checking that overflow does not happen, or does happen? | ||||
3683 | switch (Pred) { | ||||
3684 | case ICmpInst::Predicate::ICMP_ULT: | ||||
3685 | NeedNegation = false; | ||||
3686 | break; // OK | ||||
3687 | case ICmpInst::Predicate::ICMP_UGE: | ||||
3688 | NeedNegation = true; | ||||
3689 | break; // OK | ||||
3690 | default: | ||||
3691 | return nullptr; // Wrong predicate. | ||||
3692 | } | ||||
3693 | } else // Look for: ((x * y) u/ x) !=/== y | ||||
3694 | if (I.isEquality() && | ||||
3695 | match(&I, m_c_ICmp(Pred, m_Value(Y), | ||||
3696 | m_OneUse(m_UDiv(m_CombineAnd(m_c_Mul(m_Deferred(Y), | ||||
3697 | m_Value(X)), | ||||
3698 | m_Instruction(Mul)), | ||||
3699 | m_Deferred(X)))))) { | ||||
3700 | NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ; | ||||
3701 | } else | ||||
3702 | return nullptr; | ||||
3703 | |||||
3704 | BuilderTy::InsertPointGuard Guard(Builder); | ||||
3705 | // If the pattern included (x * y), we'll want to insert new instructions | ||||
3706 | // right before that original multiplication so that we can replace it. | ||||
3707 | bool MulHadOtherUses = Mul && !Mul->hasOneUse(); | ||||
3708 | if (MulHadOtherUses) | ||||
3709 | Builder.SetInsertPoint(Mul); | ||||
3710 | |||||
3711 | Function *F = Intrinsic::getDeclaration( | ||||
3712 | I.getModule(), Intrinsic::umul_with_overflow, X->getType()); | ||||
3713 | CallInst *Call = Builder.CreateCall(F, {X, Y}, "umul"); | ||||
3714 | |||||
3715 | // If the multiplication was used elsewhere, to ensure that we don't leave | ||||
3716 | // "duplicate" instructions, replace uses of that original multiplication | ||||
3717 | // with the multiplication result from the with.overflow intrinsic. | ||||
3718 | if (MulHadOtherUses) | ||||
3719 | replaceInstUsesWith(*Mul, Builder.CreateExtractValue(Call, 0, "umul.val")); | ||||
3720 | |||||
3721 | Value *Res = Builder.CreateExtractValue(Call, 1, "umul.ov"); | ||||
3722 | if (NeedNegation) // This technically increases instruction count. | ||||
3723 | Res = Builder.CreateNot(Res, "umul.not.ov"); | ||||
3724 | |||||
3725 | // If we replaced the mul, erase it. Do this after all uses of Builder, | ||||
3726 | // as the mul is used as insertion point. | ||||
3727 | if (MulHadOtherUses) | ||||
3728 | eraseInstFromFunction(*Mul); | ||||
3729 | |||||
3730 | return Res; | ||||
3731 | } | ||||
3732 | |||||
3733 | static Instruction *foldICmpXNegX(ICmpInst &I) { | ||||
3734 | CmpInst::Predicate Pred; | ||||
3735 | Value *X; | ||||
3736 | if (!match(&I, m_c_ICmp(Pred, m_NSWNeg(m_Value(X)), m_Deferred(X)))) | ||||
3737 | return nullptr; | ||||
3738 | |||||
3739 | if (ICmpInst::isSigned(Pred)) | ||||
3740 | Pred = ICmpInst::getSwappedPredicate(Pred); | ||||
3741 | else if (ICmpInst::isUnsigned(Pred)) | ||||
3742 | Pred = ICmpInst::getSignedPredicate(Pred); | ||||
3743 | // else for equality-comparisons just keep the predicate. | ||||
3744 | |||||
3745 | return ICmpInst::Create(Instruction::ICmp, Pred, X, | ||||
3746 | Constant::getNullValue(X->getType()), I.getName()); | ||||
3747 | } | ||||
3748 | |||||
3749 | /// Try to fold icmp (binop), X or icmp X, (binop). | ||||
3750 | /// TODO: A large part of this logic is duplicated in InstSimplify's | ||||
3751 | /// simplifyICmpWithBinOp(). We should be able to share that and avoid the code | ||||
3752 | /// duplication. | ||||
3753 | Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I, | ||||
3754 | const SimplifyQuery &SQ) { | ||||
3755 | const SimplifyQuery Q = SQ.getWithInstruction(&I); | ||||
3756 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||
3757 | |||||
3758 | // Special logic for binary operators. | ||||
3759 | BinaryOperator *BO0 = dyn_cast<BinaryOperator>(Op0); | ||||
3760 | BinaryOperator *BO1 = dyn_cast<BinaryOperator>(Op1); | ||||
3761 | if (!BO0 && !BO1) | ||||
3762 | return nullptr; | ||||
3763 | |||||
3764 | if (Instruction *NewICmp = foldICmpXNegX(I)) | ||||
3765 | return NewICmp; | ||||
3766 | |||||
3767 | const CmpInst::Predicate Pred = I.getPredicate(); | ||||
3768 | Value *X; | ||||
3769 | |||||
3770 | // Convert add-with-unsigned-overflow comparisons into a 'not' with compare. | ||||
3771 | // (Op1 + X) u</u>= Op1 --> ~Op1 u</u>= X | ||||
3772 | if (match(Op0, m_OneUse(m_c_Add(m_Specific(Op1), m_Value(X)))) && | ||||
3773 | (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) | ||||
3774 | return new ICmpInst(Pred, Builder.CreateNot(Op1), X); | ||||
3775 | // Op0 u>/u<= (Op0 + X) --> X u>/u<= ~Op0 | ||||
3776 | if (match(Op1, m_OneUse(m_c_Add(m_Specific(Op0), m_Value(X)))) && | ||||
3777 | (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) | ||||
3778 | return new ICmpInst(Pred, X, Builder.CreateNot(Op0)); | ||||
3779 | |||||
3780 | bool NoOp0WrapProblem = false, NoOp1WrapProblem = false; | ||||
3781 | if (BO0 && isa<OverflowingBinaryOperator>(BO0)) | ||||
3782 | NoOp0WrapProblem = | ||||
3783 | ICmpInst::isEquality(Pred) || | ||||
3784 | (CmpInst::isUnsigned(Pred) && BO0->hasNoUnsignedWrap()) || | ||||
3785 | (CmpInst::isSigned(Pred) && BO0->hasNoSignedWrap()); | ||||
3786 | if (BO1 && isa<OverflowingBinaryOperator>(BO1)) | ||||
3787 | NoOp1WrapProblem = | ||||
3788 | ICmpInst::isEquality(Pred) || | ||||
3789 | (CmpInst::isUnsigned(Pred) && BO1->hasNoUnsignedWrap()) || | ||||
3790 | (CmpInst::isSigned(Pred) && BO1->hasNoSignedWrap()); | ||||
3791 | |||||
3792 | // Analyze the case when either Op0 or Op1 is an add instruction. | ||||
3793 | // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null). | ||||
3794 | Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr; | ||||
3795 | if (BO0 && BO0->getOpcode() == Instruction::Add) { | ||||
3796 | A = BO0->getOperand(0); | ||||
3797 | B = BO0->getOperand(1); | ||||
3798 | } | ||||
3799 | if (BO1 && BO1->getOpcode() == Instruction::Add) { | ||||
3800 | C = BO1->getOperand(0); | ||||
3801 | D = BO1->getOperand(1); | ||||
3802 | } | ||||
3803 | |||||
3804 | // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow. | ||||
3805 | // icmp (A+B), B -> icmp A, 0 for equalities or if there is no overflow. | ||||
3806 | if ((A == Op1 || B == Op1) && NoOp0WrapProblem) | ||||
3807 | return new ICmpInst(Pred, A == Op1 ? B : A, | ||||
3808 | Constant::getNullValue(Op1->getType())); | ||||
3809 | |||||
3810 | // icmp C, (C+D) -> icmp 0, D for equalities or if there is no overflow. | ||||
3811 | // icmp D, (C+D) -> icmp 0, C for equalities or if there is no overflow. | ||||
3812 | if ((C == Op0 || D == Op0) && NoOp1WrapProblem) | ||||
3813 | return new ICmpInst(Pred, Constant::getNullValue(Op0->getType()), | ||||
3814 | C == Op0 ? D : C); | ||||
3815 | |||||
3816 | // icmp (A+B), (A+D) -> icmp B, D for equalities or if there is no overflow. | ||||
3817 | if (A && C && (A == C || A == D || B == C || B == D) && NoOp0WrapProblem && | ||||
3818 | NoOp1WrapProblem) { | ||||
3819 | // Determine Y and Z in the form icmp (X+Y), (X+Z). | ||||
3820 | Value *Y, *Z; | ||||
3821 | if (A == C) { | ||||
3822 | // C + B == C + D -> B == D | ||||
3823 | Y = B; | ||||
3824 | Z = D; | ||||
3825 | } else if (A == D) { | ||||
3826 | // D + B == C + D -> B == C | ||||
3827 | Y = B; | ||||
3828 | Z = C; | ||||
3829 | } else if (B == C) { | ||||
3830 | // A + C == C + D -> A == D | ||||
3831 | Y = A; | ||||
3832 | Z = D; | ||||
3833 | } else { | ||||
3834 | assert(B == D)((B == D) ? static_cast<void> (0) : __assert_fail ("B == D" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3834, __PRETTY_FUNCTION__)); | ||||
3835 | // A + D == C + D -> A == C | ||||
3836 | Y = A; | ||||
3837 | Z = C; | ||||
3838 | } | ||||
3839 | return new ICmpInst(Pred, Y, Z); | ||||
3840 | } | ||||
3841 | |||||
3842 | // icmp slt (A + -1), Op1 -> icmp sle A, Op1 | ||||
3843 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLT && | ||||
3844 | match(B, m_AllOnes())) | ||||
3845 | return new ICmpInst(CmpInst::ICMP_SLE, A, Op1); | ||||
3846 | |||||
3847 | // icmp sge (A + -1), Op1 -> icmp sgt A, Op1 | ||||
3848 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGE && | ||||
3849 | match(B, m_AllOnes())) | ||||
3850 | return new ICmpInst(CmpInst::ICMP_SGT, A, Op1); | ||||
3851 | |||||
3852 | // icmp sle (A + 1), Op1 -> icmp slt A, Op1 | ||||
3853 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLE && match(B, m_One())) | ||||
3854 | return new ICmpInst(CmpInst::ICMP_SLT, A, Op1); | ||||
3855 | |||||
3856 | // icmp sgt (A + 1), Op1 -> icmp sge A, Op1 | ||||
3857 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGT && match(B, m_One())) | ||||
3858 | return new ICmpInst(CmpInst::ICMP_SGE, A, Op1); | ||||
3859 | |||||
3860 | // icmp sgt Op0, (C + -1) -> icmp sge Op0, C | ||||
3861 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGT && | ||||
3862 | match(D, m_AllOnes())) | ||||
3863 | return new ICmpInst(CmpInst::ICMP_SGE, Op0, C); | ||||
3864 | |||||
3865 | // icmp sle Op0, (C + -1) -> icmp slt Op0, C | ||||
3866 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLE && | ||||
3867 | match(D, m_AllOnes())) | ||||
3868 | return new ICmpInst(CmpInst::ICMP_SLT, Op0, C); | ||||
3869 | |||||
3870 | // icmp sge Op0, (C + 1) -> icmp sgt Op0, C | ||||
3871 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGE && match(D, m_One())) | ||||
3872 | return new ICmpInst(CmpInst::ICMP_SGT, Op0, C); | ||||
3873 | |||||
3874 | // icmp slt Op0, (C + 1) -> icmp sle Op0, C | ||||
3875 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLT && match(D, m_One())) | ||||
3876 | return new ICmpInst(CmpInst::ICMP_SLE, Op0, C); | ||||
3877 | |||||
3878 | // TODO: The subtraction-related identities shown below also hold, but | ||||
3879 | // canonicalization from (X -nuw 1) to (X + -1) means that the combinations | ||||
3880 | // wouldn't happen even if they were implemented. | ||||
3881 | // | ||||
3882 | // icmp ult (A - 1), Op1 -> icmp ule A, Op1 | ||||
3883 | // icmp uge (A - 1), Op1 -> icmp ugt A, Op1 | ||||
3884 | // icmp ugt Op0, (C - 1) -> icmp uge Op0, C | ||||
3885 | // icmp ule Op0, (C - 1) -> icmp ult Op0, C | ||||
3886 | |||||
3887 | // icmp ule (A + 1), Op0 -> icmp ult A, Op1 | ||||
3888 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_ULE && match(B, m_One())) | ||||
3889 | return new ICmpInst(CmpInst::ICMP_ULT, A, Op1); | ||||
3890 | |||||
3891 | // icmp ugt (A + 1), Op0 -> icmp uge A, Op1 | ||||
3892 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_UGT && match(B, m_One())) | ||||
3893 | return new ICmpInst(CmpInst::ICMP_UGE, A, Op1); | ||||
3894 | |||||
3895 | // icmp uge Op0, (C + 1) -> icmp ugt Op0, C | ||||
3896 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_UGE && match(D, m_One())) | ||||
3897 | return new ICmpInst(CmpInst::ICMP_UGT, Op0, C); | ||||
3898 | |||||
3899 | // icmp ult Op0, (C + 1) -> icmp ule Op0, C | ||||
3900 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_ULT && match(D, m_One())) | ||||
3901 | return new ICmpInst(CmpInst::ICMP_ULE, Op0, C); | ||||
3902 | |||||
3903 | // if C1 has greater magnitude than C2: | ||||
3904 | // icmp (A + C1), (C + C2) -> icmp (A + C3), C | ||||
3905 | // s.t. C3 = C1 - C2 | ||||
3906 | // | ||||
3907 | // if C2 has greater magnitude than C1: | ||||
3908 | // icmp (A + C1), (C + C2) -> icmp A, (C + C3) | ||||
3909 | // s.t. C3 = C2 - C1 | ||||
3910 | if (A && C && NoOp0WrapProblem && NoOp1WrapProblem && | ||||
3911 | (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned()) | ||||
3912 | if (ConstantInt *C1 = dyn_cast<ConstantInt>(B)) | ||||
3913 | if (ConstantInt *C2 = dyn_cast<ConstantInt>(D)) { | ||||
3914 | const APInt &AP1 = C1->getValue(); | ||||
3915 | const APInt &AP2 = C2->getValue(); | ||||
3916 | if (AP1.isNegative() == AP2.isNegative()) { | ||||
3917 | APInt AP1Abs = C1->getValue().abs(); | ||||
3918 | APInt AP2Abs = C2->getValue().abs(); | ||||
3919 | if (AP1Abs.uge(AP2Abs)) { | ||||
3920 | ConstantInt *C3 = Builder.getInt(AP1 - AP2); | ||||
3921 | Value *NewAdd = Builder.CreateNSWAdd(A, C3); | ||||
3922 | return new ICmpInst(Pred, NewAdd, C); | ||||
3923 | } else { | ||||
3924 | ConstantInt *C3 = Builder.getInt(AP2 - AP1); | ||||
3925 | Value *NewAdd = Builder.CreateNSWAdd(C, C3); | ||||
3926 | return new ICmpInst(Pred, A, NewAdd); | ||||
3927 | } | ||||
3928 | } | ||||
3929 | } | ||||
3930 | |||||
3931 | // Analyze the case when either Op0 or Op1 is a sub instruction. | ||||
3932 | // Op0 = A - B (or A and B are null); Op1 = C - D (or C and D are null). | ||||
3933 | A = nullptr; | ||||
3934 | B = nullptr; | ||||
3935 | C = nullptr; | ||||
3936 | D = nullptr; | ||||
3937 | if (BO0 && BO0->getOpcode() == Instruction::Sub) { | ||||
3938 | A = BO0->getOperand(0); | ||||
3939 | B = BO0->getOperand(1); | ||||
3940 | } | ||||
3941 | if (BO1 && BO1->getOpcode() == Instruction::Sub) { | ||||
3942 | C = BO1->getOperand(0); | ||||
3943 | D = BO1->getOperand(1); | ||||
3944 | } | ||||
3945 | |||||
3946 | // icmp (A-B), A -> icmp 0, B for equalities or if there is no overflow. | ||||
3947 | if (A == Op1 && NoOp0WrapProblem) | ||||
3948 | return new ICmpInst(Pred, Constant::getNullValue(Op1->getType()), B); | ||||
3949 | // icmp C, (C-D) -> icmp D, 0 for equalities or if there is no overflow. | ||||
3950 | if (C == Op0 && NoOp1WrapProblem) | ||||
3951 | return new ICmpInst(Pred, D, Constant::getNullValue(Op0->getType())); | ||||
3952 | |||||
3953 | // Convert sub-with-unsigned-overflow comparisons into a comparison of args. | ||||
3954 | // (A - B) u>/u<= A --> B u>/u<= A | ||||
3955 | if (A == Op1 && (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) | ||||
3956 | return new ICmpInst(Pred, B, A); | ||||
3957 | // C u</u>= (C - D) --> C u</u>= D | ||||
3958 | if (C == Op0 && (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) | ||||
3959 | return new ICmpInst(Pred, C, D); | ||||
3960 | // (A - B) u>=/u< A --> B u>/u<= A iff B != 0 | ||||
3961 | if (A == Op1 && (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) && | ||||
3962 | isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) | ||||
3963 | return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), B, A); | ||||
3964 | // C u<=/u> (C - D) --> C u</u>= D iff B != 0 | ||||
3965 | if (C == Op0 && (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) && | ||||
3966 | isKnownNonZero(D, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) | ||||
3967 | return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), C, D); | ||||
3968 | |||||
3969 | // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow. | ||||
3970 | if (B && D && B == D && NoOp0WrapProblem && NoOp1WrapProblem) | ||||
3971 | return new ICmpInst(Pred, A, C); | ||||
3972 | |||||
3973 | // icmp (A-B), (A-D) -> icmp D, B for equalities or if there is no overflow. | ||||
3974 | if (A && C && A == C && NoOp0WrapProblem && NoOp1WrapProblem) | ||||
3975 | return new ICmpInst(Pred, D, B); | ||||
3976 | |||||
3977 | // icmp (0-X) < cst --> x > -cst | ||||
3978 | if (NoOp0WrapProblem && ICmpInst::isSigned(Pred)) { | ||||
3979 | Value *X; | ||||
3980 | if (match(BO0, m_Neg(m_Value(X)))) | ||||
3981 | if (Constant *RHSC = dyn_cast<Constant>(Op1)) | ||||
3982 | if (RHSC->isNotMinSignedValue()) | ||||
3983 | return new ICmpInst(I.getSwappedPredicate(), X, | ||||
3984 | ConstantExpr::getNeg(RHSC)); | ||||
3985 | } | ||||
3986 | |||||
3987 | { | ||||
3988 | // Try to remove shared constant multiplier from equality comparison: | ||||
3989 | // X * C == Y * C (with no overflowing/aliasing) --> X == Y | ||||
3990 | Value *X, *Y; | ||||
3991 | const APInt *C; | ||||
3992 | if (match(Op0, m_Mul(m_Value(X), m_APInt(C))) && *C != 0 && | ||||
3993 | match(Op1, m_Mul(m_Value(Y), m_SpecificInt(*C))) && I.isEquality()) | ||||
3994 | if (!C->countTrailingZeros() || | ||||
3995 | (BO0->hasNoSignedWrap() && BO1->hasNoSignedWrap()) || | ||||
3996 | (BO0->hasNoUnsignedWrap() && BO1->hasNoUnsignedWrap())) | ||||
3997 | return new ICmpInst(Pred, X, Y); | ||||
3998 | } | ||||
3999 | |||||
4000 | BinaryOperator *SRem = nullptr; | ||||
4001 | // icmp (srem X, Y), Y | ||||
4002 | if (BO0 && BO0->getOpcode() == Instruction::SRem && Op1 == BO0->getOperand(1)) | ||||
4003 | SRem = BO0; | ||||
4004 | // icmp Y, (srem X, Y) | ||||
4005 | else if (BO1 && BO1->getOpcode() == Instruction::SRem && | ||||
4006 | Op0 == BO1->getOperand(1)) | ||||
4007 | SRem = BO1; | ||||
4008 | if (SRem) { | ||||
4009 | // We don't check hasOneUse to avoid increasing register pressure because | ||||
4010 | // the value we use is the same value this instruction was already using. | ||||
4011 | switch (SRem == BO0 ? ICmpInst::getSwappedPredicate(Pred) : Pred) { | ||||
4012 | default: | ||||
4013 | break; | ||||
4014 | case ICmpInst::ICMP_EQ: | ||||
4015 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||
4016 | case ICmpInst::ICMP_NE: | ||||
4017 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||
4018 | case ICmpInst::ICMP_SGT: | ||||
4019 | case ICmpInst::ICMP_SGE: | ||||
4020 | return new ICmpInst(ICmpInst::ICMP_SGT, SRem->getOperand(1), | ||||
4021 | Constant::getAllOnesValue(SRem->getType())); | ||||
4022 | case ICmpInst::ICMP_SLT: | ||||
4023 | case ICmpInst::ICMP_SLE: | ||||
4024 | return new ICmpInst(ICmpInst::ICMP_SLT, SRem->getOperand(1), | ||||
4025 | Constant::getNullValue(SRem->getType())); | ||||
4026 | } | ||||
4027 | } | ||||
4028 | |||||
4029 | if (BO0 && BO1 && BO0->getOpcode() == BO1->getOpcode() && BO0->hasOneUse() && | ||||
4030 | BO1->hasOneUse() && BO0->getOperand(1) == BO1->getOperand(1)) { | ||||
4031 | switch (BO0->getOpcode()) { | ||||
4032 | default: | ||||
4033 | break; | ||||
4034 | case Instruction::Add: | ||||
4035 | case Instruction::Sub: | ||||
4036 | case Instruction::Xor: { | ||||
4037 | if (I.isEquality()) // a+x icmp eq/ne b+x --> a icmp b | ||||
4038 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||
4039 | |||||
4040 | const APInt *C; | ||||
4041 | if (match(BO0->getOperand(1), m_APInt(C))) { | ||||
4042 | // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b | ||||
4043 | if (C->isSignMask()) { | ||||
4044 | ICmpInst::Predicate NewPred = | ||||
4045 | I.isSigned() ? I.getUnsignedPredicate() : I.getSignedPredicate(); | ||||
4046 | return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0)); | ||||
4047 | } | ||||
4048 | |||||
4049 | // icmp u/s (a ^ maxsignval), (b ^ maxsignval) --> icmp s/u' a, b | ||||
4050 | if (BO0->getOpcode() == Instruction::Xor && C->isMaxSignedValue()) { | ||||
4051 | ICmpInst::Predicate NewPred = | ||||
4052 | I.isSigned() ? I.getUnsignedPredicate() : I.getSignedPredicate(); | ||||
4053 | NewPred = I.getSwappedPredicate(NewPred); | ||||
4054 | return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0)); | ||||
4055 | } | ||||
4056 | } | ||||
4057 | break; | ||||
4058 | } | ||||
4059 | case Instruction::Mul: { | ||||
4060 | if (!I.isEquality()) | ||||
4061 | break; | ||||
4062 | |||||
4063 | const APInt *C; | ||||
4064 | if (match(BO0->getOperand(1), m_APInt(C)) && !C->isNullValue() && | ||||
4065 | !C->isOneValue()) { | ||||
4066 | // icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask) | ||||
4067 | // Mask = -1 >> count-trailing-zeros(C). | ||||
4068 | if (unsigned TZs = C->countTrailingZeros()) { | ||||
4069 | Constant *Mask = ConstantInt::get( | ||||
4070 | BO0->getType(), | ||||
4071 | APInt::getLowBitsSet(C->getBitWidth(), C->getBitWidth() - TZs)); | ||||
4072 | Value *And1 = Builder.CreateAnd(BO0->getOperand(0), Mask); | ||||
4073 | Value *And2 = Builder.CreateAnd(BO1->getOperand(0), Mask); | ||||
4074 | return new ICmpInst(Pred, And1, And2); | ||||
4075 | } | ||||
4076 | } | ||||
4077 | break; | ||||
4078 | } | ||||
4079 | case Instruction::UDiv: | ||||
4080 | case Instruction::LShr: | ||||
4081 | if (I.isSigned() || !BO0->isExact() || !BO1->isExact()) | ||||
4082 | break; | ||||
4083 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||
4084 | |||||
4085 | case Instruction::SDiv: | ||||
4086 | if (!I.isEquality() || !BO0->isExact() || !BO1->isExact()) | ||||
4087 | break; | ||||
4088 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||
4089 | |||||
4090 | case Instruction::AShr: | ||||
4091 | if (!BO0->isExact() || !BO1->isExact()) | ||||
4092 | break; | ||||
4093 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||
4094 | |||||
4095 | case Instruction::Shl: { | ||||
4096 | bool NUW = BO0->hasNoUnsignedWrap() && BO1->hasNoUnsignedWrap(); | ||||
4097 | bool NSW = BO0->hasNoSignedWrap() && BO1->hasNoSignedWrap(); | ||||
4098 | if (!NUW && !NSW) | ||||
4099 | break; | ||||
4100 | if (!NSW && I.isSigned()) | ||||
4101 | break; | ||||
4102 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||
4103 | } | ||||
4104 | } | ||||
4105 | } | ||||
4106 | |||||
4107 | if (BO0) { | ||||
4108 | // Transform A & (L - 1) `ult` L --> L != 0 | ||||
4109 | auto LSubOne = m_Add(m_Specific(Op1), m_AllOnes()); | ||||
4110 | auto BitwiseAnd = m_c_And(m_Value(), LSubOne); | ||||
4111 | |||||
4112 | if (match(BO0, BitwiseAnd) && Pred == ICmpInst::ICMP_ULT) { | ||||
4113 | auto *Zero = Constant::getNullValue(BO0->getType()); | ||||
4114 | return new ICmpInst(ICmpInst::ICMP_NE, Op1, Zero); | ||||
4115 | } | ||||
4116 | } | ||||
4117 | |||||
4118 | if (Value *V = foldUnsignedMultiplicationOverflowCheck(I)) | ||||
4119 | return replaceInstUsesWith(I, V); | ||||
4120 | |||||
4121 | if (Value *V = foldICmpWithLowBitMaskedVal(I, Builder)) | ||||
4122 | return replaceInstUsesWith(I, V); | ||||
4123 | |||||
4124 | if (Value *V = foldICmpWithTruncSignExtendedVal(I, Builder)) | ||||
4125 | return replaceInstUsesWith(I, V); | ||||
4126 | |||||
4127 | if (Value *V = foldShiftIntoShiftInAnotherHandOfAndInICmp(I, SQ, Builder)) | ||||
4128 | return replaceInstUsesWith(I, V); | ||||
4129 | |||||
4130 | return nullptr; | ||||
4131 | } | ||||
4132 | |||||
4133 | /// Fold icmp Pred min|max(X, Y), X. | ||||
4134 | static Instruction *foldICmpWithMinMax(ICmpInst &Cmp) { | ||||
4135 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
4136 | Value *Op0 = Cmp.getOperand(0); | ||||
4137 | Value *X = Cmp.getOperand(1); | ||||
4138 | |||||
4139 | // Canonicalize minimum or maximum operand to LHS of the icmp. | ||||
4140 | if (match(X, m_c_SMin(m_Specific(Op0), m_Value())) || | ||||
4141 | match(X, m_c_SMax(m_Specific(Op0), m_Value())) || | ||||
4142 | match(X, m_c_UMin(m_Specific(Op0), m_Value())) || | ||||
4143 | match(X, m_c_UMax(m_Specific(Op0), m_Value()))) { | ||||
4144 | std::swap(Op0, X); | ||||
4145 | Pred = Cmp.getSwappedPredicate(); | ||||
4146 | } | ||||
4147 | |||||
4148 | Value *Y; | ||||
4149 | if (match(Op0, m_c_SMin(m_Specific(X), m_Value(Y)))) { | ||||
4150 | // smin(X, Y) == X --> X s<= Y | ||||
4151 | // smin(X, Y) s>= X --> X s<= Y | ||||
4152 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SGE) | ||||
4153 | return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); | ||||
4154 | |||||
4155 | // smin(X, Y) != X --> X s> Y | ||||
4156 | // smin(X, Y) s< X --> X s> Y | ||||
4157 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SLT) | ||||
4158 | return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); | ||||
4159 | |||||
4160 | // These cases should be handled in InstSimplify: | ||||
4161 | // smin(X, Y) s<= X --> true | ||||
4162 | // smin(X, Y) s> X --> false | ||||
4163 | return nullptr; | ||||
4164 | } | ||||
4165 | |||||
4166 | if (match(Op0, m_c_SMax(m_Specific(X), m_Value(Y)))) { | ||||
4167 | // smax(X, Y) == X --> X s>= Y | ||||
4168 | // smax(X, Y) s<= X --> X s>= Y | ||||
4169 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SLE) | ||||
4170 | return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); | ||||
4171 | |||||
4172 | // smax(X, Y) != X --> X s< Y | ||||
4173 | // smax(X, Y) s> X --> X s< Y | ||||
4174 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SGT) | ||||
4175 | return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); | ||||
4176 | |||||
4177 | // These cases should be handled in InstSimplify: | ||||
4178 | // smax(X, Y) s>= X --> true | ||||
4179 | // smax(X, Y) s< X --> false | ||||
4180 | return nullptr; | ||||
4181 | } | ||||
4182 | |||||
4183 | if (match(Op0, m_c_UMin(m_Specific(X), m_Value(Y)))) { | ||||
4184 | // umin(X, Y) == X --> X u<= Y | ||||
4185 | // umin(X, Y) u>= X --> X u<= Y | ||||
4186 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_UGE) | ||||
4187 | return new ICmpInst(ICmpInst::ICMP_ULE, X, Y); | ||||
4188 | |||||
4189 | // umin(X, Y) != X --> X u> Y | ||||
4190 | // umin(X, Y) u< X --> X u> Y | ||||
4191 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_ULT) | ||||
4192 | return new ICmpInst(ICmpInst::ICMP_UGT, X, Y); | ||||
4193 | |||||
4194 | // These cases should be handled in InstSimplify: | ||||
4195 | // umin(X, Y) u<= X --> true | ||||
4196 | // umin(X, Y) u> X --> false | ||||
4197 | return nullptr; | ||||
4198 | } | ||||
4199 | |||||
4200 | if (match(Op0, m_c_UMax(m_Specific(X), m_Value(Y)))) { | ||||
4201 | // umax(X, Y) == X --> X u>= Y | ||||
4202 | // umax(X, Y) u<= X --> X u>= Y | ||||
4203 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_ULE) | ||||
4204 | return new ICmpInst(ICmpInst::ICMP_UGE, X, Y); | ||||
4205 | |||||
4206 | // umax(X, Y) != X --> X u< Y | ||||
4207 | // umax(X, Y) u> X --> X u< Y | ||||
4208 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_UGT) | ||||
4209 | return new ICmpInst(ICmpInst::ICMP_ULT, X, Y); | ||||
4210 | |||||
4211 | // These cases should be handled in InstSimplify: | ||||
4212 | // umax(X, Y) u>= X --> true | ||||
4213 | // umax(X, Y) u< X --> false | ||||
4214 | return nullptr; | ||||
4215 | } | ||||
4216 | |||||
4217 | return nullptr; | ||||
4218 | } | ||||
4219 | |||||
4220 | Instruction *InstCombinerImpl::foldICmpEquality(ICmpInst &I) { | ||||
4221 | if (!I.isEquality()) | ||||
4222 | return nullptr; | ||||
4223 | |||||
4224 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||
4225 | const CmpInst::Predicate Pred = I.getPredicate(); | ||||
4226 | Value *A, *B, *C, *D; | ||||
4227 | if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) { | ||||
4228 | if (A == Op1 || B == Op1) { // (A^B) == A -> B == 0 | ||||
4229 | Value *OtherVal = A == Op1 ? B : A; | ||||
4230 | return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType())); | ||||
4231 | } | ||||
4232 | |||||
4233 | if (match(Op1, m_Xor(m_Value(C), m_Value(D)))) { | ||||
4234 | // A^c1 == C^c2 --> A == C^(c1^c2) | ||||
4235 | ConstantInt *C1, *C2; | ||||
4236 | if (match(B, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2)) && | ||||
4237 | Op1->hasOneUse()) { | ||||
4238 | Constant *NC = Builder.getInt(C1->getValue() ^ C2->getValue()); | ||||
4239 | Value *Xor = Builder.CreateXor(C, NC); | ||||
4240 | return new ICmpInst(Pred, A, Xor); | ||||
4241 | } | ||||
4242 | |||||
4243 | // A^B == A^D -> B == D | ||||
4244 | if (A == C) | ||||
4245 | return new ICmpInst(Pred, B, D); | ||||
4246 | if (A == D) | ||||
4247 | return new ICmpInst(Pred, B, C); | ||||
4248 | if (B == C) | ||||
4249 | return new ICmpInst(Pred, A, D); | ||||
4250 | if (B == D) | ||||
4251 | return new ICmpInst(Pred, A, C); | ||||
4252 | } | ||||
4253 | } | ||||
4254 | |||||
4255 | if (match(Op1, m_Xor(m_Value(A), m_Value(B))) && (A == Op0 || B == Op0)) { | ||||
4256 | // A == (A^B) -> B == 0 | ||||
4257 | Value *OtherVal = A == Op0 ? B : A; | ||||
4258 | return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType())); | ||||
4259 | } | ||||
4260 | |||||
4261 | // (X&Z) == (Y&Z) -> (X^Y) & Z == 0 | ||||
4262 | if (match(Op0, m_OneUse(m_And(m_Value(A), m_Value(B)))) && | ||||
4263 | match(Op1, m_OneUse(m_And(m_Value(C), m_Value(D))))) { | ||||
4264 | Value *X = nullptr, *Y = nullptr, *Z = nullptr; | ||||
4265 | |||||
4266 | if (A == C) { | ||||
4267 | X = B; | ||||
4268 | Y = D; | ||||
4269 | Z = A; | ||||
4270 | } else if (A == D) { | ||||
4271 | X = B; | ||||
4272 | Y = C; | ||||
4273 | Z = A; | ||||
4274 | } else if (B == C) { | ||||
4275 | X = A; | ||||
4276 | Y = D; | ||||
4277 | Z = B; | ||||
4278 | } else if (B == D) { | ||||
4279 | X = A; | ||||
4280 | Y = C; | ||||
4281 | Z = B; | ||||
4282 | } | ||||
4283 | |||||
4284 | if (X) { // Build (X^Y) & Z | ||||
4285 | Op1 = Builder.CreateXor(X, Y); | ||||
4286 | Op1 = Builder.CreateAnd(Op1, Z); | ||||
4287 | return new ICmpInst(Pred, Op1, Constant::getNullValue(Op1->getType())); | ||||
4288 | } | ||||
4289 | } | ||||
4290 | |||||
4291 | // Transform (zext A) == (B & (1<<X)-1) --> A == (trunc B) | ||||
4292 | // and (B & (1<<X)-1) == (zext A) --> A == (trunc B) | ||||
4293 | ConstantInt *Cst1; | ||||
4294 | if ((Op0->hasOneUse() && match(Op0, m_ZExt(m_Value(A))) && | ||||
4295 | match(Op1, m_And(m_Value(B), m_ConstantInt(Cst1)))) || | ||||
4296 | (Op1->hasOneUse() && match(Op0, m_And(m_Value(B), m_ConstantInt(Cst1))) && | ||||
4297 | match(Op1, m_ZExt(m_Value(A))))) { | ||||
4298 | APInt Pow2 = Cst1->getValue() + 1; | ||||
4299 | if (Pow2.isPowerOf2() && isa<IntegerType>(A->getType()) && | ||||
4300 | Pow2.logBase2() == cast<IntegerType>(A->getType())->getBitWidth()) | ||||
4301 | return new ICmpInst(Pred, A, Builder.CreateTrunc(B, A->getType())); | ||||
4302 | } | ||||
4303 | |||||
4304 | // (A >> C) == (B >> C) --> (A^B) u< (1 << C) | ||||
4305 | // For lshr and ashr pairs. | ||||
4306 | if ((match(Op0, m_OneUse(m_LShr(m_Value(A), m_ConstantInt(Cst1)))) && | ||||
4307 | match(Op1, m_OneUse(m_LShr(m_Value(B), m_Specific(Cst1))))) || | ||||
4308 | (match(Op0, m_OneUse(m_AShr(m_Value(A), m_ConstantInt(Cst1)))) && | ||||
4309 | match(Op1, m_OneUse(m_AShr(m_Value(B), m_Specific(Cst1)))))) { | ||||
4310 | unsigned TypeBits = Cst1->getBitWidth(); | ||||
4311 | unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits); | ||||
4312 | if (ShAmt < TypeBits && ShAmt != 0) { | ||||
4313 | ICmpInst::Predicate NewPred = | ||||
4314 | Pred == ICmpInst::ICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; | ||||
4315 | Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted"); | ||||
4316 | APInt CmpVal = APInt::getOneBitSet(TypeBits, ShAmt); | ||||
4317 | return new ICmpInst(NewPred, Xor, Builder.getInt(CmpVal)); | ||||
4318 | } | ||||
4319 | } | ||||
4320 | |||||
4321 | // (A << C) == (B << C) --> ((A^B) & (~0U >> C)) == 0 | ||||
4322 | if (match(Op0, m_OneUse(m_Shl(m_Value(A), m_ConstantInt(Cst1)))) && | ||||
4323 | match(Op1, m_OneUse(m_Shl(m_Value(B), m_Specific(Cst1))))) { | ||||
4324 | unsigned TypeBits = Cst1->getBitWidth(); | ||||
4325 | unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits); | ||||
4326 | if (ShAmt < TypeBits && ShAmt != 0) { | ||||
4327 | Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted"); | ||||
4328 | APInt AndVal = APInt::getLowBitsSet(TypeBits, TypeBits - ShAmt); | ||||
4329 | Value *And = Builder.CreateAnd(Xor, Builder.getInt(AndVal), | ||||
4330 | I.getName() + ".mask"); | ||||
4331 | return new ICmpInst(Pred, And, Constant::getNullValue(Cst1->getType())); | ||||
4332 | } | ||||
4333 | } | ||||
4334 | |||||
4335 | // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to | ||||
4336 | // "icmp (and X, mask), cst" | ||||
4337 | uint64_t ShAmt = 0; | ||||
4338 | if (Op0->hasOneUse() && | ||||
4339 | match(Op0, m_Trunc(m_OneUse(m_LShr(m_Value(A), m_ConstantInt(ShAmt))))) && | ||||
4340 | match(Op1, m_ConstantInt(Cst1)) && | ||||
4341 | // Only do this when A has multiple uses. This is most important to do | ||||
4342 | // when it exposes other optimizations. | ||||
4343 | !A->hasOneUse()) { | ||||
4344 | unsigned ASize = cast<IntegerType>(A->getType())->getPrimitiveSizeInBits(); | ||||
4345 | |||||
4346 | if (ShAmt < ASize) { | ||||
4347 | APInt MaskV = | ||||
4348 | APInt::getLowBitsSet(ASize, Op0->getType()->getPrimitiveSizeInBits()); | ||||
4349 | MaskV <<= ShAmt; | ||||
4350 | |||||
4351 | APInt CmpV = Cst1->getValue().zext(ASize); | ||||
4352 | CmpV <<= ShAmt; | ||||
4353 | |||||
4354 | Value *Mask = Builder.CreateAnd(A, Builder.getInt(MaskV)); | ||||
4355 | return new ICmpInst(Pred, Mask, Builder.getInt(CmpV)); | ||||
4356 | } | ||||
4357 | } | ||||
4358 | |||||
4359 | // If both operands are byte-swapped or bit-reversed, just compare the | ||||
4360 | // original values. | ||||
4361 | // TODO: Move this to a function similar to foldICmpIntrinsicWithConstant() | ||||
4362 | // and handle more intrinsics. | ||||
4363 | if ((match(Op0, m_BSwap(m_Value(A))) && match(Op1, m_BSwap(m_Value(B)))) || | ||||
4364 | (match(Op0, m_BitReverse(m_Value(A))) && | ||||
4365 | match(Op1, m_BitReverse(m_Value(B))))) | ||||
4366 | return new ICmpInst(Pred, A, B); | ||||
4367 | |||||
4368 | // Canonicalize checking for a power-of-2-or-zero value: | ||||
4369 | // (A & (A-1)) == 0 --> ctpop(A) < 2 (two commuted variants) | ||||
4370 | // ((A-1) & A) != 0 --> ctpop(A) > 1 (two commuted variants) | ||||
4371 | if (!match(Op0, m_OneUse(m_c_And(m_Add(m_Value(A), m_AllOnes()), | ||||
4372 | m_Deferred(A)))) || | ||||
4373 | !match(Op1, m_ZeroInt())) | ||||
4374 | A = nullptr; | ||||
4375 | |||||
4376 | // (A & -A) == A --> ctpop(A) < 2 (four commuted variants) | ||||
4377 | // (-A & A) != A --> ctpop(A) > 1 (four commuted variants) | ||||
4378 | if (match(Op0, m_OneUse(m_c_And(m_Neg(m_Specific(Op1)), m_Specific(Op1))))) | ||||
4379 | A = Op1; | ||||
4380 | else if (match(Op1, | ||||
4381 | m_OneUse(m_c_And(m_Neg(m_Specific(Op0)), m_Specific(Op0))))) | ||||
4382 | A = Op0; | ||||
4383 | |||||
4384 | if (A) { | ||||
4385 | Type *Ty = A->getType(); | ||||
4386 | CallInst *CtPop = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, A); | ||||
4387 | return Pred == ICmpInst::ICMP_EQ | ||||
4388 | ? new ICmpInst(ICmpInst::ICMP_ULT, CtPop, ConstantInt::get(Ty, 2)) | ||||
4389 | : new ICmpInst(ICmpInst::ICMP_UGT, CtPop, ConstantInt::get(Ty, 1)); | ||||
4390 | } | ||||
4391 | |||||
4392 | return nullptr; | ||||
4393 | } | ||||
4394 | |||||
4395 | static Instruction *foldICmpWithZextOrSext(ICmpInst &ICmp, | ||||
4396 | InstCombiner::BuilderTy &Builder) { | ||||
4397 | assert(isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0")((isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0" ) ? static_cast<void> (0) : __assert_fail ("isa<CastInst>(ICmp.getOperand(0)) && \"Expected cast for operand 0\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4397, __PRETTY_FUNCTION__)); | ||||
4398 | auto *CastOp0 = cast<CastInst>(ICmp.getOperand(0)); | ||||
4399 | Value *X; | ||||
4400 | if (!match(CastOp0, m_ZExtOrSExt(m_Value(X)))) | ||||
4401 | return nullptr; | ||||
4402 | |||||
4403 | bool IsSignedExt = CastOp0->getOpcode() == Instruction::SExt; | ||||
4404 | bool IsSignedCmp = ICmp.isSigned(); | ||||
4405 | if (auto *CastOp1 = dyn_cast<CastInst>(ICmp.getOperand(1))) { | ||||
4406 | // If the signedness of the two casts doesn't agree (i.e. one is a sext | ||||
4407 | // and the other is a zext), then we can't handle this. | ||||
4408 | // TODO: This is too strict. We can handle some predicates (equality?). | ||||
4409 | if (CastOp0->getOpcode() != CastOp1->getOpcode()) | ||||
4410 | return nullptr; | ||||
4411 | |||||
4412 | // Not an extension from the same type? | ||||
4413 | Value *Y = CastOp1->getOperand(0); | ||||
4414 | Type *XTy = X->getType(), *YTy = Y->getType(); | ||||
4415 | if (XTy != YTy) { | ||||
4416 | // One of the casts must have one use because we are creating a new cast. | ||||
4417 | if (!CastOp0->hasOneUse() && !CastOp1->hasOneUse()) | ||||
4418 | return nullptr; | ||||
4419 | // Extend the narrower operand to the type of the wider operand. | ||||
4420 | if (XTy->getScalarSizeInBits() < YTy->getScalarSizeInBits()) | ||||
4421 | X = Builder.CreateCast(CastOp0->getOpcode(), X, YTy); | ||||
4422 | else if (YTy->getScalarSizeInBits() < XTy->getScalarSizeInBits()) | ||||
4423 | Y = Builder.CreateCast(CastOp0->getOpcode(), Y, XTy); | ||||
4424 | else | ||||
4425 | return nullptr; | ||||
4426 | } | ||||
4427 | |||||
4428 | // (zext X) == (zext Y) --> X == Y | ||||
4429 | // (sext X) == (sext Y) --> X == Y | ||||
4430 | if (ICmp.isEquality()) | ||||
4431 | return new ICmpInst(ICmp.getPredicate(), X, Y); | ||||
4432 | |||||
4433 | // A signed comparison of sign extended values simplifies into a | ||||
4434 | // signed comparison. | ||||
4435 | if (IsSignedCmp && IsSignedExt) | ||||
4436 | return new ICmpInst(ICmp.getPredicate(), X, Y); | ||||
4437 | |||||
4438 | // The other three cases all fold into an unsigned comparison. | ||||
4439 | return new ICmpInst(ICmp.getUnsignedPredicate(), X, Y); | ||||
4440 | } | ||||
4441 | |||||
4442 | // Below here, we are only folding a compare with constant. | ||||
4443 | auto *C = dyn_cast<Constant>(ICmp.getOperand(1)); | ||||
4444 | if (!C) | ||||
4445 | return nullptr; | ||||
4446 | |||||
4447 | // Compute the constant that would happen if we truncated to SrcTy then | ||||
4448 | // re-extended to DestTy. | ||||
4449 | Type *SrcTy = CastOp0->getSrcTy(); | ||||
4450 | Type *DestTy = CastOp0->getDestTy(); | ||||
4451 | Constant *Res1 = ConstantExpr::getTrunc(C, SrcTy); | ||||
4452 | Constant *Res2 = ConstantExpr::getCast(CastOp0->getOpcode(), Res1, DestTy); | ||||
4453 | |||||
4454 | // If the re-extended constant didn't change... | ||||
4455 | if (Res2 == C) { | ||||
4456 | if (ICmp.isEquality()) | ||||
4457 | return new ICmpInst(ICmp.getPredicate(), X, Res1); | ||||
4458 | |||||
4459 | // A signed comparison of sign extended values simplifies into a | ||||
4460 | // signed comparison. | ||||
4461 | if (IsSignedExt && IsSignedCmp) | ||||
4462 | return new ICmpInst(ICmp.getPredicate(), X, Res1); | ||||
4463 | |||||
4464 | // The other three cases all fold into an unsigned comparison. | ||||
4465 | return new ICmpInst(ICmp.getUnsignedPredicate(), X, Res1); | ||||
4466 | } | ||||
4467 | |||||
4468 | // The re-extended constant changed, partly changed (in the case of a vector), | ||||
4469 | // or could not be determined to be equal (in the case of a constant | ||||
4470 | // expression), so the constant cannot be represented in the shorter type. | ||||
4471 | // All the cases that fold to true or false will have already been handled | ||||
4472 | // by SimplifyICmpInst, so only deal with the tricky case. | ||||
4473 | if (IsSignedCmp || !IsSignedExt || !isa<ConstantInt>(C)) | ||||
4474 | return nullptr; | ||||
4475 | |||||
4476 | // Is source op positive? | ||||
4477 | // icmp ult (sext X), C --> icmp sgt X, -1 | ||||
4478 | if (ICmp.getPredicate() == ICmpInst::ICMP_ULT) | ||||
4479 | return new ICmpInst(CmpInst::ICMP_SGT, X, Constant::getAllOnesValue(SrcTy)); | ||||
4480 | |||||
4481 | // Is source op negative? | ||||
4482 | // icmp ugt (sext X), C --> icmp slt X, 0 | ||||
4483 | assert(ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!")((ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!" ) ? static_cast<void> (0) : __assert_fail ("ICmp.getPredicate() == ICmpInst::ICMP_UGT && \"ICmp should be folded!\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4483, __PRETTY_FUNCTION__)); | ||||
4484 | return new ICmpInst(CmpInst::ICMP_SLT, X, Constant::getNullValue(SrcTy)); | ||||
4485 | } | ||||
4486 | |||||
4487 | /// Handle icmp (cast x), (cast or constant). | ||||
4488 | Instruction *InstCombinerImpl::foldICmpWithCastOp(ICmpInst &ICmp) { | ||||
4489 | auto *CastOp0 = dyn_cast<CastInst>(ICmp.getOperand(0)); | ||||
4490 | if (!CastOp0) | ||||
4491 | return nullptr; | ||||
4492 | if (!isa<Constant>(ICmp.getOperand(1)) && !isa<CastInst>(ICmp.getOperand(1))) | ||||
4493 | return nullptr; | ||||
4494 | |||||
4495 | Value *Op0Src = CastOp0->getOperand(0); | ||||
4496 | Type *SrcTy = CastOp0->getSrcTy(); | ||||
4497 | Type *DestTy = CastOp0->getDestTy(); | ||||
4498 | |||||
4499 | // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the | ||||
4500 | // integer type is the same size as the pointer type. | ||||
4501 | auto CompatibleSizes = [&](Type *SrcTy, Type *DestTy) { | ||||
4502 | if (isa<VectorType>(SrcTy)) { | ||||
4503 | SrcTy = cast<VectorType>(SrcTy)->getElementType(); | ||||
4504 | DestTy = cast<VectorType>(DestTy)->getElementType(); | ||||
4505 | } | ||||
4506 | return DL.getPointerTypeSizeInBits(SrcTy) == DestTy->getIntegerBitWidth(); | ||||
4507 | }; | ||||
4508 | if (CastOp0->getOpcode() == Instruction::PtrToInt && | ||||
4509 | CompatibleSizes(SrcTy, DestTy)) { | ||||
4510 | Value *NewOp1 = nullptr; | ||||
4511 | if (auto *PtrToIntOp1 = dyn_cast<PtrToIntOperator>(ICmp.getOperand(1))) { | ||||
4512 | Value *PtrSrc = PtrToIntOp1->getOperand(0); | ||||
4513 | if (PtrSrc->getType()->getPointerAddressSpace() == | ||||
4514 | Op0Src->getType()->getPointerAddressSpace()) { | ||||
4515 | NewOp1 = PtrToIntOp1->getOperand(0); | ||||
4516 | // If the pointer types don't match, insert a bitcast. | ||||
4517 | if (Op0Src->getType() != NewOp1->getType()) | ||||
4518 | NewOp1 = Builder.CreateBitCast(NewOp1, Op0Src->getType()); | ||||
4519 | } | ||||
4520 | } else if (auto *RHSC = dyn_cast<Constant>(ICmp.getOperand(1))) { | ||||
4521 | NewOp1 = ConstantExpr::getIntToPtr(RHSC, SrcTy); | ||||
4522 | } | ||||
4523 | |||||
4524 | if (NewOp1) | ||||
4525 | return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1); | ||||
4526 | } | ||||
4527 | |||||
4528 | return foldICmpWithZextOrSext(ICmp, Builder); | ||||
4529 | } | ||||
4530 | |||||
4531 | static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS) { | ||||
4532 | switch (BinaryOp) { | ||||
4533 | default: | ||||
4534 | llvm_unreachable("Unsupported binary op")::llvm::llvm_unreachable_internal("Unsupported binary op", "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4534); | ||||
4535 | case Instruction::Add: | ||||
4536 | case Instruction::Sub: | ||||
4537 | return match(RHS, m_Zero()); | ||||
4538 | case Instruction::Mul: | ||||
4539 | return match(RHS, m_One()); | ||||
4540 | } | ||||
4541 | } | ||||
4542 | |||||
4543 | OverflowResult | ||||
4544 | InstCombinerImpl::computeOverflow(Instruction::BinaryOps BinaryOp, | ||||
4545 | bool IsSigned, Value *LHS, Value *RHS, | ||||
4546 | Instruction *CxtI) const { | ||||
4547 | switch (BinaryOp) { | ||||
4548 | default: | ||||
4549 | llvm_unreachable("Unsupported binary op")::llvm::llvm_unreachable_internal("Unsupported binary op", "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4549); | ||||
4550 | case Instruction::Add: | ||||
4551 | if (IsSigned) | ||||
4552 | return computeOverflowForSignedAdd(LHS, RHS, CxtI); | ||||
4553 | else | ||||
4554 | return computeOverflowForUnsignedAdd(LHS, RHS, CxtI); | ||||
4555 | case Instruction::Sub: | ||||
4556 | if (IsSigned) | ||||
4557 | return computeOverflowForSignedSub(LHS, RHS, CxtI); | ||||
4558 | else | ||||
4559 | return computeOverflowForUnsignedSub(LHS, RHS, CxtI); | ||||
4560 | case Instruction::Mul: | ||||
4561 | if (IsSigned) | ||||
4562 | return computeOverflowForSignedMul(LHS, RHS, CxtI); | ||||
4563 | else | ||||
4564 | return computeOverflowForUnsignedMul(LHS, RHS, CxtI); | ||||
4565 | } | ||||
4566 | } | ||||
4567 | |||||
4568 | bool InstCombinerImpl::OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp, | ||||
4569 | bool IsSigned, Value *LHS, | ||||
4570 | Value *RHS, Instruction &OrigI, | ||||
4571 | Value *&Result, | ||||
4572 | Constant *&Overflow) { | ||||
4573 | if (OrigI.isCommutative() && isa<Constant>(LHS) && !isa<Constant>(RHS)) | ||||
4574 | std::swap(LHS, RHS); | ||||
4575 | |||||
4576 | // If the overflow check was an add followed by a compare, the insertion point | ||||
4577 | // may be pointing to the compare. We want to insert the new instructions | ||||
4578 | // before the add in case there are uses of the add between the add and the | ||||
4579 | // compare. | ||||
4580 | Builder.SetInsertPoint(&OrigI); | ||||
4581 | |||||
4582 | if (isNeutralValue(BinaryOp, RHS)) { | ||||
4583 | Result = LHS; | ||||
4584 | Overflow = Builder.getFalse(); | ||||
4585 | return true; | ||||
4586 | } | ||||
4587 | |||||
4588 | switch (computeOverflow(BinaryOp, IsSigned, LHS, RHS, &OrigI)) { | ||||
4589 | case OverflowResult::MayOverflow: | ||||
4590 | return false; | ||||
4591 | case OverflowResult::AlwaysOverflowsLow: | ||||
4592 | case OverflowResult::AlwaysOverflowsHigh: | ||||
4593 | Result = Builder.CreateBinOp(BinaryOp, LHS, RHS); | ||||
4594 | Result->takeName(&OrigI); | ||||
4595 | Overflow = Builder.getTrue(); | ||||
4596 | return true; | ||||
4597 | case OverflowResult::NeverOverflows: | ||||
4598 | Result = Builder.CreateBinOp(BinaryOp, LHS, RHS); | ||||
4599 | Result->takeName(&OrigI); | ||||
4600 | Overflow = Builder.getFalse(); | ||||
4601 | if (auto *Inst = dyn_cast<Instruction>(Result)) { | ||||
4602 | if (IsSigned) | ||||
4603 | Inst->setHasNoSignedWrap(); | ||||
4604 | else | ||||
4605 | Inst->setHasNoUnsignedWrap(); | ||||
4606 | } | ||||
4607 | return true; | ||||
4608 | } | ||||
4609 | |||||
4610 | llvm_unreachable("Unexpected overflow result")::llvm::llvm_unreachable_internal("Unexpected overflow result" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4610); | ||||
4611 | } | ||||
4612 | |||||
4613 | /// Recognize and process idiom involving test for multiplication | ||||
4614 | /// overflow. | ||||
4615 | /// | ||||
4616 | /// The caller has matched a pattern of the form: | ||||
4617 | /// I = cmp u (mul(zext A, zext B), V | ||||
4618 | /// The function checks if this is a test for overflow and if so replaces | ||||
4619 | /// multiplication with call to 'mul.with.overflow' intrinsic. | ||||
4620 | /// | ||||
4621 | /// \param I Compare instruction. | ||||
4622 | /// \param MulVal Result of 'mult' instruction. It is one of the arguments of | ||||
4623 | /// the compare instruction. Must be of integer type. | ||||
4624 | /// \param OtherVal The other argument of compare instruction. | ||||
4625 | /// \returns Instruction which must replace the compare instruction, NULL if no | ||||
4626 | /// replacement required. | ||||
4627 | static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal, | ||||
4628 | Value *OtherVal, | ||||
4629 | InstCombinerImpl &IC) { | ||||
4630 | // Don't bother doing this transformation for pointers, don't do it for | ||||
4631 | // vectors. | ||||
4632 | if (!isa<IntegerType>(MulVal->getType())) | ||||
4633 | return nullptr; | ||||
4634 | |||||
4635 | assert(I.getOperand(0) == MulVal || I.getOperand(1) == MulVal)((I.getOperand(0) == MulVal || I.getOperand(1) == MulVal) ? static_cast <void> (0) : __assert_fail ("I.getOperand(0) == MulVal || I.getOperand(1) == MulVal" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4635, __PRETTY_FUNCTION__)); | ||||
4636 | assert(I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal)((I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal) ? static_cast<void> (0) : __assert_fail ("I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4636, __PRETTY_FUNCTION__)); | ||||
4637 | auto *MulInstr = dyn_cast<Instruction>(MulVal); | ||||
4638 | if (!MulInstr) | ||||
4639 | return nullptr; | ||||
4640 | assert(MulInstr->getOpcode() == Instruction::Mul)((MulInstr->getOpcode() == Instruction::Mul) ? static_cast <void> (0) : __assert_fail ("MulInstr->getOpcode() == Instruction::Mul" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4640, __PRETTY_FUNCTION__)); | ||||
4641 | |||||
4642 | auto *LHS = cast<ZExtOperator>(MulInstr->getOperand(0)), | ||||
4643 | *RHS = cast<ZExtOperator>(MulInstr->getOperand(1)); | ||||
4644 | assert(LHS->getOpcode() == Instruction::ZExt)((LHS->getOpcode() == Instruction::ZExt) ? static_cast< void> (0) : __assert_fail ("LHS->getOpcode() == Instruction::ZExt" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4644, __PRETTY_FUNCTION__)); | ||||
4645 | assert(RHS->getOpcode() == Instruction::ZExt)((RHS->getOpcode() == Instruction::ZExt) ? static_cast< void> (0) : __assert_fail ("RHS->getOpcode() == Instruction::ZExt" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4645, __PRETTY_FUNCTION__)); | ||||
4646 | Value *A = LHS->getOperand(0), *B = RHS->getOperand(0); | ||||
4647 | |||||
4648 | // Calculate type and width of the result produced by mul.with.overflow. | ||||
4649 | Type *TyA = A->getType(), *TyB = B->getType(); | ||||
4650 | unsigned WidthA = TyA->getPrimitiveSizeInBits(), | ||||
4651 | WidthB = TyB->getPrimitiveSizeInBits(); | ||||
4652 | unsigned MulWidth; | ||||
4653 | Type *MulType; | ||||
4654 | if (WidthB > WidthA) { | ||||
4655 | MulWidth = WidthB; | ||||
4656 | MulType = TyB; | ||||
4657 | } else { | ||||
4658 | MulWidth = WidthA; | ||||
4659 | MulType = TyA; | ||||
4660 | } | ||||
4661 | |||||
4662 | // In order to replace the original mul with a narrower mul.with.overflow, | ||||
4663 | // all uses must ignore upper bits of the product. The number of used low | ||||
4664 | // bits must be not greater than the width of mul.with.overflow. | ||||
4665 | if (MulVal->hasNUsesOrMore(2)) | ||||
4666 | for (User *U : MulVal->users()) { | ||||
4667 | if (U == &I) | ||||
4668 | continue; | ||||
4669 | if (TruncInst *TI = dyn_cast<TruncInst>(U)) { | ||||
4670 | // Check if truncation ignores bits above MulWidth. | ||||
4671 | unsigned TruncWidth = TI->getType()->getPrimitiveSizeInBits(); | ||||
4672 | if (TruncWidth > MulWidth) | ||||
4673 | return nullptr; | ||||
4674 | } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) { | ||||
4675 | // Check if AND ignores bits above MulWidth. | ||||
4676 | if (BO->getOpcode() != Instruction::And) | ||||
4677 | return nullptr; | ||||
4678 | if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->getOperand(1))) { | ||||
4679 | const APInt &CVal = CI->getValue(); | ||||
4680 | if (CVal.getBitWidth() - CVal.countLeadingZeros() > MulWidth) | ||||
4681 | return nullptr; | ||||
4682 | } else { | ||||
4683 | // In this case we could have the operand of the binary operation | ||||
4684 | // being defined in another block, and performing the replacement | ||||
4685 | // could break the dominance relation. | ||||
4686 | return nullptr; | ||||
4687 | } | ||||
4688 | } else { | ||||
4689 | // Other uses prohibit this transformation. | ||||
4690 | return nullptr; | ||||
4691 | } | ||||
4692 | } | ||||
4693 | |||||
4694 | // Recognize patterns | ||||
4695 | switch (I.getPredicate()) { | ||||
4696 | case ICmpInst::ICMP_EQ: | ||||
4697 | case ICmpInst::ICMP_NE: | ||||
4698 | // Recognize pattern: | ||||
4699 | // mulval = mul(zext A, zext B) | ||||
4700 | // cmp eq/neq mulval, and(mulval, mask), mask selects low MulWidth bits. | ||||
4701 | ConstantInt *CI; | ||||
4702 | Value *ValToMask; | ||||
4703 | if (match(OtherVal, m_And(m_Value(ValToMask), m_ConstantInt(CI)))) { | ||||
4704 | if (ValToMask != MulVal) | ||||
4705 | return nullptr; | ||||
4706 | const APInt &CVal = CI->getValue() + 1; | ||||
4707 | if (CVal.isPowerOf2()) { | ||||
4708 | unsigned MaskWidth = CVal.logBase2(); | ||||
4709 | if (MaskWidth == MulWidth) | ||||
4710 | break; // Recognized | ||||
4711 | } | ||||
4712 | } | ||||
4713 | return nullptr; | ||||
4714 | |||||
4715 | case ICmpInst::ICMP_UGT: | ||||
4716 | // Recognize pattern: | ||||
4717 | // mulval = mul(zext A, zext B) | ||||
4718 | // cmp ugt mulval, max | ||||
4719 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | ||||
4720 | APInt MaxVal = APInt::getMaxValue(MulWidth); | ||||
4721 | MaxVal = MaxVal.zext(CI->getBitWidth()); | ||||
4722 | if (MaxVal.eq(CI->getValue())) | ||||
4723 | break; // Recognized | ||||
4724 | } | ||||
4725 | return nullptr; | ||||
4726 | |||||
4727 | case ICmpInst::ICMP_UGE: | ||||
4728 | // Recognize pattern: | ||||
4729 | // mulval = mul(zext A, zext B) | ||||
4730 | // cmp uge mulval, max+1 | ||||
4731 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | ||||
4732 | APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth); | ||||
4733 | if (MaxVal.eq(CI->getValue())) | ||||
4734 | break; // Recognized | ||||
4735 | } | ||||
4736 | return nullptr; | ||||
4737 | |||||
4738 | case ICmpInst::ICMP_ULE: | ||||
4739 | // Recognize pattern: | ||||
4740 | // mulval = mul(zext A, zext B) | ||||
4741 | // cmp ule mulval, max | ||||
4742 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | ||||
4743 | APInt MaxVal = APInt::getMaxValue(MulWidth); | ||||
4744 | MaxVal = MaxVal.zext(CI->getBitWidth()); | ||||
4745 | if (MaxVal.eq(CI->getValue())) | ||||
4746 | break; // Recognized | ||||
4747 | } | ||||
4748 | return nullptr; | ||||
4749 | |||||
4750 | case ICmpInst::ICMP_ULT: | ||||
4751 | // Recognize pattern: | ||||
4752 | // mulval = mul(zext A, zext B) | ||||
4753 | // cmp ule mulval, max + 1 | ||||
4754 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | ||||
4755 | APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth); | ||||
4756 | if (MaxVal.eq(CI->getValue())) | ||||
4757 | break; // Recognized | ||||
4758 | } | ||||
4759 | return nullptr; | ||||
4760 | |||||
4761 | default: | ||||
4762 | return nullptr; | ||||
4763 | } | ||||
4764 | |||||
4765 | InstCombiner::BuilderTy &Builder = IC.Builder; | ||||
4766 | Builder.SetInsertPoint(MulInstr); | ||||
4767 | |||||
4768 | // Replace: mul(zext A, zext B) --> mul.with.overflow(A, B) | ||||
4769 | Value *MulA = A, *MulB = B; | ||||
4770 | if (WidthA < MulWidth) | ||||
4771 | MulA = Builder.CreateZExt(A, MulType); | ||||
4772 | if (WidthB < MulWidth) | ||||
4773 | MulB = Builder.CreateZExt(B, MulType); | ||||
4774 | Function *F = Intrinsic::getDeclaration( | ||||
4775 | I.getModule(), Intrinsic::umul_with_overflow, MulType); | ||||
4776 | CallInst *Call = Builder.CreateCall(F, {MulA, MulB}, "umul"); | ||||
4777 | IC.addToWorklist(MulInstr); | ||||
4778 | |||||
4779 | // If there are uses of mul result other than the comparison, we know that | ||||
4780 | // they are truncation or binary AND. Change them to use result of | ||||
4781 | // mul.with.overflow and adjust properly mask/size. | ||||
4782 | if (MulVal->hasNUsesOrMore(2)) { | ||||
4783 | Value *Mul = Builder.CreateExtractValue(Call, 0, "umul.value"); | ||||
4784 | for (auto UI = MulVal->user_begin(), UE = MulVal->user_end(); UI != UE;) { | ||||
4785 | User *U = *UI++; | ||||
4786 | if (U == &I || U == OtherVal) | ||||
4787 | continue; | ||||
4788 | if (TruncInst *TI = dyn_cast<TruncInst>(U)) { | ||||
4789 | if (TI->getType()->getPrimitiveSizeInBits() == MulWidth) | ||||
4790 | IC.replaceInstUsesWith(*TI, Mul); | ||||
4791 | else | ||||
4792 | TI->setOperand(0, Mul); | ||||
4793 | } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) { | ||||
4794 | assert(BO->getOpcode() == Instruction::And)((BO->getOpcode() == Instruction::And) ? static_cast<void > (0) : __assert_fail ("BO->getOpcode() == Instruction::And" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4794, __PRETTY_FUNCTION__)); | ||||
4795 | // Replace (mul & mask) --> zext (mul.with.overflow & short_mask) | ||||
4796 | ConstantInt *CI = cast<ConstantInt>(BO->getOperand(1)); | ||||
4797 | APInt ShortMask = CI->getValue().trunc(MulWidth); | ||||
4798 | Value *ShortAnd = Builder.CreateAnd(Mul, ShortMask); | ||||
4799 | Value *Zext = Builder.CreateZExt(ShortAnd, BO->getType()); | ||||
4800 | IC.replaceInstUsesWith(*BO, Zext); | ||||
4801 | } else { | ||||
4802 | llvm_unreachable("Unexpected Binary operation")::llvm::llvm_unreachable_internal("Unexpected Binary operation" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4802); | ||||
4803 | } | ||||
4804 | IC.addToWorklist(cast<Instruction>(U)); | ||||
4805 | } | ||||
4806 | } | ||||
4807 | if (isa<Instruction>(OtherVal)) | ||||
4808 | IC.addToWorklist(cast<Instruction>(OtherVal)); | ||||
4809 | |||||
4810 | // The original icmp gets replaced with the overflow value, maybe inverted | ||||
4811 | // depending on predicate. | ||||
4812 | bool Inverse = false; | ||||
4813 | switch (I.getPredicate()) { | ||||
4814 | case ICmpInst::ICMP_NE: | ||||
4815 | break; | ||||
4816 | case ICmpInst::ICMP_EQ: | ||||
4817 | Inverse = true; | ||||
4818 | break; | ||||
4819 | case ICmpInst::ICMP_UGT: | ||||
4820 | case ICmpInst::ICMP_UGE: | ||||
4821 | if (I.getOperand(0) == MulVal) | ||||
4822 | break; | ||||
4823 | Inverse = true; | ||||
4824 | break; | ||||
4825 | case ICmpInst::ICMP_ULT: | ||||
4826 | case ICmpInst::ICMP_ULE: | ||||
4827 | if (I.getOperand(1) == MulVal) | ||||
4828 | break; | ||||
4829 | Inverse = true; | ||||
4830 | break; | ||||
4831 | default: | ||||
4832 | llvm_unreachable("Unexpected predicate")::llvm::llvm_unreachable_internal("Unexpected predicate", "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4832); | ||||
4833 | } | ||||
4834 | if (Inverse) { | ||||
4835 | Value *Res = Builder.CreateExtractValue(Call, 1); | ||||
4836 | return BinaryOperator::CreateNot(Res); | ||||
4837 | } | ||||
4838 | |||||
4839 | return ExtractValueInst::Create(Call, 1); | ||||
4840 | } | ||||
4841 | |||||
4842 | /// When performing a comparison against a constant, it is possible that not all | ||||
4843 | /// the bits in the LHS are demanded. This helper method computes the mask that | ||||
4844 | /// IS demanded. | ||||
4845 | static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth) { | ||||
4846 | const APInt *RHS; | ||||
4847 | if (!match(I.getOperand(1), m_APInt(RHS))) | ||||
4848 | return APInt::getAllOnesValue(BitWidth); | ||||
4849 | |||||
4850 | // If this is a normal comparison, it demands all bits. If it is a sign bit | ||||
4851 | // comparison, it only demands the sign bit. | ||||
4852 | bool UnusedBit; | ||||
4853 | if (InstCombiner::isSignBitCheck(I.getPredicate(), *RHS, UnusedBit)) | ||||
4854 | return APInt::getSignMask(BitWidth); | ||||
4855 | |||||
4856 | switch (I.getPredicate()) { | ||||
4857 | // For a UGT comparison, we don't care about any bits that | ||||
4858 | // correspond to the trailing ones of the comparand. The value of these | ||||
4859 | // bits doesn't impact the outcome of the comparison, because any value | ||||
4860 | // greater than the RHS must differ in a bit higher than these due to carry. | ||||
4861 | case ICmpInst::ICMP_UGT: | ||||
4862 | return APInt::getBitsSetFrom(BitWidth, RHS->countTrailingOnes()); | ||||
4863 | |||||
4864 | // Similarly, for a ULT comparison, we don't care about the trailing zeros. | ||||
4865 | // Any value less than the RHS must differ in a higher bit because of carries. | ||||
4866 | case ICmpInst::ICMP_ULT: | ||||
4867 | return APInt::getBitsSetFrom(BitWidth, RHS->countTrailingZeros()); | ||||
4868 | |||||
4869 | default: | ||||
4870 | return APInt::getAllOnesValue(BitWidth); | ||||
4871 | } | ||||
4872 | } | ||||
4873 | |||||
4874 | /// Check if the order of \p Op0 and \p Op1 as operands in an ICmpInst | ||||
4875 | /// should be swapped. | ||||
4876 | /// The decision is based on how many times these two operands are reused | ||||
4877 | /// as subtract operands and their positions in those instructions. | ||||
4878 | /// The rationale is that several architectures use the same instruction for | ||||
4879 | /// both subtract and cmp. Thus, it is better if the order of those operands | ||||
4880 | /// match. | ||||
4881 | /// \return true if Op0 and Op1 should be swapped. | ||||
4882 | static bool swapMayExposeCSEOpportunities(const Value *Op0, const Value *Op1) { | ||||
4883 | // Filter out pointer values as those cannot appear directly in subtract. | ||||
4884 | // FIXME: we may want to go through inttoptrs or bitcasts. | ||||
4885 | if (Op0->getType()->isPointerTy()) | ||||
4886 | return false; | ||||
4887 | // If a subtract already has the same operands as a compare, swapping would be | ||||
4888 | // bad. If a subtract has the same operands as a compare but in reverse order, | ||||
4889 | // then swapping is good. | ||||
4890 | int GoodToSwap = 0; | ||||
4891 | for (const User *U : Op0->users()) { | ||||
4892 | if (match(U, m_Sub(m_Specific(Op1), m_Specific(Op0)))) | ||||
4893 | GoodToSwap++; | ||||
4894 | else if (match(U, m_Sub(m_Specific(Op0), m_Specific(Op1)))) | ||||
4895 | GoodToSwap--; | ||||
4896 | } | ||||
4897 | return GoodToSwap > 0; | ||||
4898 | } | ||||
4899 | |||||
4900 | /// Check that one use is in the same block as the definition and all | ||||
4901 | /// other uses are in blocks dominated by a given block. | ||||
4902 | /// | ||||
4903 | /// \param DI Definition | ||||
4904 | /// \param UI Use | ||||
4905 | /// \param DB Block that must dominate all uses of \p DI outside | ||||
4906 | /// the parent block | ||||
4907 | /// \return true when \p UI is the only use of \p DI in the parent block | ||||
4908 | /// and all other uses of \p DI are in blocks dominated by \p DB. | ||||
4909 | /// | ||||
4910 | bool InstCombinerImpl::dominatesAllUses(const Instruction *DI, | ||||
4911 | const Instruction *UI, | ||||
4912 | const BasicBlock *DB) const { | ||||
4913 | assert(DI && UI && "Instruction not defined\n")((DI && UI && "Instruction not defined\n") ? static_cast <void> (0) : __assert_fail ("DI && UI && \"Instruction not defined\\n\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4913, __PRETTY_FUNCTION__)); | ||||
4914 | // Ignore incomplete definitions. | ||||
4915 | if (!DI->getParent()) | ||||
4916 | return false; | ||||
4917 | // DI and UI must be in the same block. | ||||
4918 | if (DI->getParent() != UI->getParent()) | ||||
4919 | return false; | ||||
4920 | // Protect from self-referencing blocks. | ||||
4921 | if (DI->getParent() == DB) | ||||
4922 | return false; | ||||
4923 | for (const User *U : DI->users()) { | ||||
4924 | auto *Usr = cast<Instruction>(U); | ||||
4925 | if (Usr != UI && !DT.dominates(DB, Usr->getParent())) | ||||
4926 | return false; | ||||
4927 | } | ||||
4928 | return true; | ||||
4929 | } | ||||
4930 | |||||
4931 | /// Return true when the instruction sequence within a block is select-cmp-br. | ||||
4932 | static bool isChainSelectCmpBranch(const SelectInst *SI) { | ||||
4933 | const BasicBlock *BB = SI->getParent(); | ||||
4934 | if (!BB) | ||||
4935 | return false; | ||||
4936 | auto *BI = dyn_cast_or_null<BranchInst>(BB->getTerminator()); | ||||
4937 | if (!BI || BI->getNumSuccessors() != 2) | ||||
4938 | return false; | ||||
4939 | auto *IC = dyn_cast<ICmpInst>(BI->getCondition()); | ||||
4940 | if (!IC || (IC->getOperand(0) != SI && IC->getOperand(1) != SI)) | ||||
4941 | return false; | ||||
4942 | return true; | ||||
4943 | } | ||||
4944 | |||||
4945 | /// True when a select result is replaced by one of its operands | ||||
4946 | /// in select-icmp sequence. This will eventually result in the elimination | ||||
4947 | /// of the select. | ||||
4948 | /// | ||||
4949 | /// \param SI Select instruction | ||||
4950 | /// \param Icmp Compare instruction | ||||
4951 | /// \param SIOpd Operand that replaces the select | ||||
4952 | /// | ||||
4953 | /// Notes: | ||||
4954 | /// - The replacement is global and requires dominator information | ||||
4955 | /// - The caller is responsible for the actual replacement | ||||
4956 | /// | ||||
4957 | /// Example: | ||||
4958 | /// | ||||
4959 | /// entry: | ||||
4960 | /// %4 = select i1 %3, %C* %0, %C* null | ||||
4961 | /// %5 = icmp eq %C* %4, null | ||||
4962 | /// br i1 %5, label %9, label %7 | ||||
4963 | /// ... | ||||
4964 | /// ; <label>:7 ; preds = %entry | ||||
4965 | /// %8 = getelementptr inbounds %C* %4, i64 0, i32 0 | ||||
4966 | /// ... | ||||
4967 | /// | ||||
4968 | /// can be transformed to | ||||
4969 | /// | ||||
4970 | /// %5 = icmp eq %C* %0, null | ||||
4971 | /// %6 = select i1 %3, i1 %5, i1 true | ||||
4972 | /// br i1 %6, label %9, label %7 | ||||
4973 | /// ... | ||||
4974 | /// ; <label>:7 ; preds = %entry | ||||
4975 | /// %8 = getelementptr inbounds %C* %0, i64 0, i32 0 // replace by %0! | ||||
4976 | /// | ||||
4977 | /// Similar when the first operand of the select is a constant or/and | ||||
4978 | /// the compare is for not equal rather than equal. | ||||
4979 | /// | ||||
4980 | /// NOTE: The function is only called when the select and compare constants | ||||
4981 | /// are equal, the optimization can work only for EQ predicates. This is not a | ||||
4982 | /// major restriction since a NE compare should be 'normalized' to an equal | ||||
4983 | /// compare, which usually happens in the combiner and test case | ||||
4984 | /// select-cmp-br.ll checks for it. | ||||
4985 | bool InstCombinerImpl::replacedSelectWithOperand(SelectInst *SI, | ||||
4986 | const ICmpInst *Icmp, | ||||
4987 | const unsigned SIOpd) { | ||||
4988 | assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!")(((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!" ) ? static_cast<void> (0) : __assert_fail ("(SIOpd == 1 || SIOpd == 2) && \"Invalid select operand!\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4988, __PRETTY_FUNCTION__)); | ||||
4989 | if (isChainSelectCmpBranch(SI) && Icmp->getPredicate() == ICmpInst::ICMP_EQ) { | ||||
4990 | BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(1); | ||||
4991 | // The check for the single predecessor is not the best that can be | ||||
4992 | // done. But it protects efficiently against cases like when SI's | ||||
4993 | // home block has two successors, Succ and Succ1, and Succ1 predecessor | ||||
4994 | // of Succ. Then SI can't be replaced by SIOpd because the use that gets | ||||
4995 | // replaced can be reached on either path. So the uniqueness check | ||||
4996 | // guarantees that the path all uses of SI (outside SI's parent) are on | ||||
4997 | // is disjoint from all other paths out of SI. But that information | ||||
4998 | // is more expensive to compute, and the trade-off here is in favor | ||||
4999 | // of compile-time. It should also be noticed that we check for a single | ||||
5000 | // predecessor and not only uniqueness. This to handle the situation when | ||||
5001 | // Succ and Succ1 points to the same basic block. | ||||
5002 | if (Succ->getSinglePredecessor() && dominatesAllUses(SI, Icmp, Succ)) { | ||||
5003 | NumSel++; | ||||
5004 | SI->replaceUsesOutsideBlock(SI->getOperand(SIOpd), SI->getParent()); | ||||
5005 | return true; | ||||
5006 | } | ||||
5007 | } | ||||
5008 | return false; | ||||
5009 | } | ||||
5010 | |||||
5011 | /// Try to fold the comparison based on range information we can get by checking | ||||
5012 | /// whether bits are known to be zero or one in the inputs. | ||||
5013 | Instruction *InstCombinerImpl::foldICmpUsingKnownBits(ICmpInst &I) { | ||||
5014 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||
5015 | Type *Ty = Op0->getType(); | ||||
5016 | ICmpInst::Predicate Pred = I.getPredicate(); | ||||
5017 | |||||
5018 | // Get scalar or pointer size. | ||||
5019 | unsigned BitWidth = Ty->isIntOrIntVectorTy() | ||||
5020 | ? Ty->getScalarSizeInBits() | ||||
5021 | : DL.getPointerTypeSizeInBits(Ty->getScalarType()); | ||||
5022 | |||||
5023 | if (!BitWidth) | ||||
5024 | return nullptr; | ||||
5025 | |||||
5026 | KnownBits Op0Known(BitWidth); | ||||
5027 | KnownBits Op1Known(BitWidth); | ||||
5028 | |||||
5029 | if (SimplifyDemandedBits(&I, 0, | ||||
5030 | getDemandedBitsLHSMask(I, BitWidth), | ||||
5031 | Op0Known, 0)) | ||||
5032 | return &I; | ||||
5033 | |||||
5034 | if (SimplifyDemandedBits(&I, 1, APInt::getAllOnesValue(BitWidth), | ||||
5035 | Op1Known, 0)) | ||||
5036 | return &I; | ||||
5037 | |||||
5038 | // Given the known and unknown bits, compute a range that the LHS could be | ||||
5039 | // in. Compute the Min, Max and RHS values based on the known bits. For the | ||||
5040 | // EQ and NE we use unsigned values. | ||||
5041 | APInt Op0Min(BitWidth, 0), Op0Max(BitWidth, 0); | ||||
5042 | APInt Op1Min(BitWidth, 0), Op1Max(BitWidth, 0); | ||||
5043 | if (I.isSigned()) { | ||||
5044 | computeSignedMinMaxValuesFromKnownBits(Op0Known, Op0Min, Op0Max); | ||||
5045 | computeSignedMinMaxValuesFromKnownBits(Op1Known, Op1Min, Op1Max); | ||||
5046 | } else { | ||||
5047 | computeUnsignedMinMaxValuesFromKnownBits(Op0Known, Op0Min, Op0Max); | ||||
5048 | computeUnsignedMinMaxValuesFromKnownBits(Op1Known, Op1Min, Op1Max); | ||||
5049 | } | ||||
5050 | |||||
5051 | // If Min and Max are known to be the same, then SimplifyDemandedBits figured | ||||
5052 | // out that the LHS or RHS is a constant. Constant fold this now, so that | ||||
5053 | // code below can assume that Min != Max. | ||||
5054 | if (!isa<Constant>(Op0) && Op0Min == Op0Max) | ||||
5055 | return new ICmpInst(Pred, ConstantExpr::getIntegerValue(Ty, Op0Min), Op1); | ||||
5056 | if (!isa<Constant>(Op1) && Op1Min == Op1Max) | ||||
5057 | return new ICmpInst(Pred, Op0, ConstantExpr::getIntegerValue(Ty, Op1Min)); | ||||
5058 | |||||
5059 | // Based on the range information we know about the LHS, see if we can | ||||
5060 | // simplify this comparison. For example, (x&4) < 8 is always true. | ||||
5061 | switch (Pred) { | ||||
5062 | default: | ||||
5063 | llvm_unreachable("Unknown icmp opcode!")::llvm::llvm_unreachable_internal("Unknown icmp opcode!", "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5063); | ||||
5064 | case ICmpInst::ICMP_EQ: | ||||
5065 | case ICmpInst::ICMP_NE: { | ||||
5066 | if (Op0Max.ult(Op1Min) || Op0Min.ugt(Op1Max)) { | ||||
5067 | return Pred == CmpInst::ICMP_EQ | ||||
5068 | ? replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())) | ||||
5069 | : replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||
5070 | } | ||||
5071 | |||||
5072 | // If all bits are known zero except for one, then we know at most one bit | ||||
5073 | // is set. If the comparison is against zero, then this is a check to see if | ||||
5074 | // *that* bit is set. | ||||
5075 | APInt Op0KnownZeroInverted = ~Op0Known.Zero; | ||||
5076 | if (Op1Known.isZero()) { | ||||
5077 | // If the LHS is an AND with the same constant, look through it. | ||||
5078 | Value *LHS = nullptr; | ||||
5079 | const APInt *LHSC; | ||||
5080 | if (!match(Op0, m_And(m_Value(LHS), m_APInt(LHSC))) || | ||||
5081 | *LHSC != Op0KnownZeroInverted) | ||||
5082 | LHS = Op0; | ||||
5083 | |||||
5084 | Value *X; | ||||
5085 | if (match(LHS, m_Shl(m_One(), m_Value(X)))) { | ||||
5086 | APInt ValToCheck = Op0KnownZeroInverted; | ||||
5087 | Type *XTy = X->getType(); | ||||
5088 | if (ValToCheck.isPowerOf2()) { | ||||
5089 | // ((1 << X) & 8) == 0 -> X != 3 | ||||
5090 | // ((1 << X) & 8) != 0 -> X == 3 | ||||
5091 | auto *CmpC = ConstantInt::get(XTy, ValToCheck.countTrailingZeros()); | ||||
5092 | auto NewPred = ICmpInst::getInversePredicate(Pred); | ||||
5093 | return new ICmpInst(NewPred, X, CmpC); | ||||
5094 | } else if ((++ValToCheck).isPowerOf2()) { | ||||
5095 | // ((1 << X) & 7) == 0 -> X >= 3 | ||||
5096 | // ((1 << X) & 7) != 0 -> X < 3 | ||||
5097 | auto *CmpC = ConstantInt::get(XTy, ValToCheck.countTrailingZeros()); | ||||
5098 | auto NewPred = | ||||
5099 | Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGE : CmpInst::ICMP_ULT; | ||||
5100 | return new ICmpInst(NewPred, X, CmpC); | ||||
5101 | } | ||||
5102 | } | ||||
5103 | |||||
5104 | // Check if the LHS is 8 >>u x and the result is a power of 2 like 1. | ||||
5105 | const APInt *CI; | ||||
5106 | if (Op0KnownZeroInverted.isOneValue() && | ||||
5107 | match(LHS, m_LShr(m_Power2(CI), m_Value(X)))) { | ||||
5108 | // ((8 >>u X) & 1) == 0 -> X != 3 | ||||
5109 | // ((8 >>u X) & 1) != 0 -> X == 3 | ||||
5110 | unsigned CmpVal = CI->countTrailingZeros(); | ||||
5111 | auto NewPred = ICmpInst::getInversePredicate(Pred); | ||||
5112 | return new ICmpInst(NewPred, X, ConstantInt::get(X->getType(), CmpVal)); | ||||
5113 | } | ||||
5114 | } | ||||
5115 | break; | ||||
5116 | } | ||||
5117 | case ICmpInst::ICMP_ULT: { | ||||
5118 | if (Op0Max.ult(Op1Min)) // A <u B -> true if max(A) < min(B) | ||||
5119 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||
5120 | if (Op0Min.uge(Op1Max)) // A <u B -> false if min(A) >= max(B) | ||||
5121 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||
5122 | if (Op1Min == Op0Max) // A <u B -> A != B if max(A) == min(B) | ||||
5123 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | ||||
5124 | |||||
5125 | const APInt *CmpC; | ||||
5126 | if (match(Op1, m_APInt(CmpC))) { | ||||
5127 | // A <u C -> A == C-1 if min(A)+1 == C | ||||
5128 | if (*CmpC == Op0Min + 1) | ||||
5129 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||
5130 | ConstantInt::get(Op1->getType(), *CmpC - 1)); | ||||
5131 | // X <u C --> X == 0, if the number of zero bits in the bottom of X | ||||
5132 | // exceeds the log2 of C. | ||||
5133 | if (Op0Known.countMinTrailingZeros() >= CmpC->ceilLogBase2()) | ||||
5134 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||
5135 | Constant::getNullValue(Op1->getType())); | ||||
5136 | } | ||||
5137 | break; | ||||
5138 | } | ||||
5139 | case ICmpInst::ICMP_UGT: { | ||||
5140 | if (Op0Min.ugt(Op1Max)) // A >u B -> true if min(A) > max(B) | ||||
5141 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||
5142 | if (Op0Max.ule(Op1Min)) // A >u B -> false if max(A) <= max(B) | ||||
5143 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||
5144 | if (Op1Max == Op0Min) // A >u B -> A != B if min(A) == max(B) | ||||
5145 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | ||||
5146 | |||||
5147 | const APInt *CmpC; | ||||
5148 | if (match(Op1, m_APInt(CmpC))) { | ||||
5149 | // A >u C -> A == C+1 if max(a)-1 == C | ||||
5150 | if (*CmpC == Op0Max - 1) | ||||
5151 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||
5152 | ConstantInt::get(Op1->getType(), *CmpC + 1)); | ||||
5153 | // X >u C --> X != 0, if the number of zero bits in the bottom of X | ||||
5154 | // exceeds the log2 of C. | ||||
5155 | if (Op0Known.countMinTrailingZeros() >= CmpC->getActiveBits()) | ||||
5156 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, | ||||
5157 | Constant::getNullValue(Op1->getType())); | ||||
5158 | } | ||||
5159 | break; | ||||
5160 | } | ||||
5161 | case ICmpInst::ICMP_SLT: { | ||||
5162 | if (Op0Max.slt(Op1Min)) // A <s B -> true if max(A) < min(C) | ||||
5163 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||
5164 | if (Op0Min.sge(Op1Max)) // A <s B -> false if min(A) >= max(C) | ||||
5165 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||
5166 | if (Op1Min == Op0Max) // A <s B -> A != B if max(A) == min(B) | ||||
5167 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | ||||
5168 | const APInt *CmpC; | ||||
5169 | if (match(Op1, m_APInt(CmpC))) { | ||||
5170 | if (*CmpC == Op0Min + 1) // A <s C -> A == C-1 if min(A)+1 == C | ||||
5171 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||
5172 | ConstantInt::get(Op1->getType(), *CmpC - 1)); | ||||
5173 | } | ||||
5174 | break; | ||||
5175 | } | ||||
5176 | case ICmpInst::ICMP_SGT: { | ||||
5177 | if (Op0Min.sgt(Op1Max)) // A >s B -> true if min(A) > max(B) | ||||
5178 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||
5179 | if (Op0Max.sle(Op1Min)) // A >s B -> false if max(A) <= min(B) | ||||
5180 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||
5181 | if (Op1Max == Op0Min) // A >s B -> A != B if min(A) == max(B) | ||||
5182 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | ||||
5183 | const APInt *CmpC; | ||||
5184 | if (match(Op1, m_APInt(CmpC))) { | ||||
5185 | if (*CmpC == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C | ||||
5186 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||
5187 | ConstantInt::get(Op1->getType(), *CmpC + 1)); | ||||
5188 | } | ||||
5189 | break; | ||||
5190 | } | ||||
5191 | case ICmpInst::ICMP_SGE: | ||||
5192 | assert(!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!")((!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!" ) ? static_cast<void> (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_SGE with ConstantInt not folded!\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5192, __PRETTY_FUNCTION__)); | ||||
5193 | if (Op0Min.sge(Op1Max)) // A >=s B -> true if min(A) >= max(B) | ||||
5194 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||
5195 | if (Op0Max.slt(Op1Min)) // A >=s B -> false if max(A) < min(B) | ||||
5196 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||
5197 | if (Op1Min == Op0Max) // A >=s B -> A == B if max(A) == min(B) | ||||
5198 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); | ||||
5199 | break; | ||||
5200 | case ICmpInst::ICMP_SLE: | ||||
5201 | assert(!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!")((!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!" ) ? static_cast<void> (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_SLE with ConstantInt not folded!\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5201, __PRETTY_FUNCTION__)); | ||||
5202 | if (Op0Max.sle(Op1Min)) // A <=s B -> true if max(A) <= min(B) | ||||
5203 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||
5204 | if (Op0Min.sgt(Op1Max)) // A <=s B -> false if min(A) > max(B) | ||||
5205 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||
5206 | if (Op1Max == Op0Min) // A <=s B -> A == B if min(A) == max(B) | ||||
5207 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); | ||||
5208 | break; | ||||
5209 | case ICmpInst::ICMP_UGE: | ||||
5210 | assert(!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!")((!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!" ) ? static_cast<void> (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_UGE with ConstantInt not folded!\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5210, __PRETTY_FUNCTION__)); | ||||
5211 | if (Op0Min.uge(Op1Max)) // A >=u B -> true if min(A) >= max(B) | ||||
5212 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||
5213 | if (Op0Max.ult(Op1Min)) // A >=u B -> false if max(A) < min(B) | ||||
5214 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||
5215 | if (Op1Min == Op0Max) // A >=u B -> A == B if max(A) == min(B) | ||||
5216 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); | ||||
5217 | break; | ||||
5218 | case ICmpInst::ICMP_ULE: | ||||
5219 | assert(!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!")((!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!" ) ? static_cast<void> (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_ULE with ConstantInt not folded!\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5219, __PRETTY_FUNCTION__)); | ||||
5220 | if (Op0Max.ule(Op1Min)) // A <=u B -> true if max(A) <= min(B) | ||||
5221 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||
5222 | if (Op0Min.ugt(Op1Max)) // A <=u B -> false if min(A) > max(B) | ||||
5223 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||
5224 | if (Op1Max == Op0Min) // A <=u B -> A == B if min(A) == max(B) | ||||
5225 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); | ||||
5226 | break; | ||||
5227 | } | ||||
5228 | |||||
5229 | // Turn a signed comparison into an unsigned one if both operands are known to | ||||
5230 | // have the same sign. | ||||
5231 | if (I.isSigned() && | ||||
5232 | ((Op0Known.Zero.isNegative() && Op1Known.Zero.isNegative()) || | ||||
5233 | (Op0Known.One.isNegative() && Op1Known.One.isNegative()))) | ||||
5234 | return new ICmpInst(I.getUnsignedPredicate(), Op0, Op1); | ||||
5235 | |||||
5236 | return nullptr; | ||||
5237 | } | ||||
5238 | |||||
5239 | llvm::Optional<std::pair<CmpInst::Predicate, Constant *>> | ||||
5240 | InstCombiner::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred, | ||||
5241 | Constant *C) { | ||||
5242 | assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&((ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate (Pred) && "Only for relational integer predicates.") ? static_cast<void> (0) : __assert_fail ("ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) && \"Only for relational integer predicates.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5243, __PRETTY_FUNCTION__)) | ||||
5243 | "Only for relational integer predicates.")((ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate (Pred) && "Only for relational integer predicates.") ? static_cast<void> (0) : __assert_fail ("ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) && \"Only for relational integer predicates.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5243, __PRETTY_FUNCTION__)); | ||||
5244 | |||||
5245 | Type *Type = C->getType(); | ||||
5246 | bool IsSigned = ICmpInst::isSigned(Pred); | ||||
5247 | |||||
5248 | CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(Pred); | ||||
5249 | bool WillIncrement = | ||||
5250 | UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT; | ||||
5251 | |||||
5252 | // Check if the constant operand can be safely incremented/decremented | ||||
5253 | // without overflowing/underflowing. | ||||
5254 | auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) { | ||||
5255 | return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned); | ||||
5256 | }; | ||||
5257 | |||||
5258 | Constant *SafeReplacementConstant = nullptr; | ||||
5259 | if (auto *CI = dyn_cast<ConstantInt>(C)) { | ||||
5260 | // Bail out if the constant can't be safely incremented/decremented. | ||||
5261 | if (!ConstantIsOk(CI)) | ||||
5262 | return llvm::None; | ||||
5263 | } else if (auto *FVTy = dyn_cast<FixedVectorType>(Type)) { | ||||
5264 | unsigned NumElts = FVTy->getNumElements(); | ||||
5265 | for (unsigned i = 0; i != NumElts; ++i) { | ||||
5266 | Constant *Elt = C->getAggregateElement(i); | ||||
5267 | if (!Elt) | ||||
5268 | return llvm::None; | ||||
5269 | |||||
5270 | if (isa<UndefValue>(Elt)) | ||||
5271 | continue; | ||||
5272 | |||||
5273 | // Bail out if we can't determine if this constant is min/max or if we | ||||
5274 | // know that this constant is min/max. | ||||
5275 | auto *CI = dyn_cast<ConstantInt>(Elt); | ||||
5276 | if (!CI || !ConstantIsOk(CI)) | ||||
5277 | return llvm::None; | ||||
5278 | |||||
5279 | if (!SafeReplacementConstant) | ||||
5280 | SafeReplacementConstant = CI; | ||||
5281 | } | ||||
5282 | } else { | ||||
5283 | // ConstantExpr? | ||||
5284 | return llvm::None; | ||||
5285 | } | ||||
5286 | |||||
5287 | // It may not be safe to change a compare predicate in the presence of | ||||
5288 | // undefined elements, so replace those elements with the first safe constant | ||||
5289 | // that we found. | ||||
5290 | if (C->containsUndefElement()) { | ||||
5291 | assert(SafeReplacementConstant && "Replacement constant not set")((SafeReplacementConstant && "Replacement constant not set" ) ? static_cast<void> (0) : __assert_fail ("SafeReplacementConstant && \"Replacement constant not set\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5291, __PRETTY_FUNCTION__)); | ||||
5292 | C = Constant::replaceUndefsWith(C, SafeReplacementConstant); | ||||
5293 | } | ||||
5294 | |||||
5295 | CmpInst::Predicate NewPred = CmpInst::getFlippedStrictnessPredicate(Pred); | ||||
5296 | |||||
5297 | // Increment or decrement the constant. | ||||
5298 | Constant *OneOrNegOne = ConstantInt::get(Type, WillIncrement ? 1 : -1, true); | ||||
5299 | Constant *NewC = ConstantExpr::getAdd(C, OneOrNegOne); | ||||
5300 | |||||
5301 | return std::make_pair(NewPred, NewC); | ||||
5302 | } | ||||
5303 | |||||
5304 | /// If we have an icmp le or icmp ge instruction with a constant operand, turn | ||||
5305 | /// it into the appropriate icmp lt or icmp gt instruction. This transform | ||||
5306 | /// allows them to be folded in visitICmpInst. | ||||
5307 | static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) { | ||||
5308 | ICmpInst::Predicate Pred = I.getPredicate(); | ||||
5309 | if (ICmpInst::isEquality(Pred) || !ICmpInst::isIntPredicate(Pred) || | ||||
5310 | InstCombiner::isCanonicalPredicate(Pred)) | ||||
5311 | return nullptr; | ||||
5312 | |||||
5313 | Value *Op0 = I.getOperand(0); | ||||
5314 | Value *Op1 = I.getOperand(1); | ||||
5315 | auto *Op1C = dyn_cast<Constant>(Op1); | ||||
5316 | if (!Op1C) | ||||
5317 | return nullptr; | ||||
5318 | |||||
5319 | auto FlippedStrictness = | ||||
5320 | InstCombiner::getFlippedStrictnessPredicateAndConstant(Pred, Op1C); | ||||
5321 | if (!FlippedStrictness) | ||||
5322 | return nullptr; | ||||
5323 | |||||
5324 | return new ICmpInst(FlippedStrictness->first, Op0, FlippedStrictness->second); | ||||
5325 | } | ||||
5326 | |||||
5327 | /// If we have a comparison with a non-canonical predicate, if we can update | ||||
5328 | /// all the users, invert the predicate and adjust all the users. | ||||
5329 | CmpInst *InstCombinerImpl::canonicalizeICmpPredicate(CmpInst &I) { | ||||
5330 | // Is the predicate already canonical? | ||||
5331 | CmpInst::Predicate Pred = I.getPredicate(); | ||||
5332 | if (InstCombiner::isCanonicalPredicate(Pred)) | ||||
5333 | return nullptr; | ||||
5334 | |||||
5335 | // Can all users be adjusted to predicate inversion? | ||||
5336 | if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr)) | ||||
5337 | return nullptr; | ||||
5338 | |||||
5339 | // Ok, we can canonicalize comparison! | ||||
5340 | // Let's first invert the comparison's predicate. | ||||
5341 | I.setPredicate(CmpInst::getInversePredicate(Pred)); | ||||
5342 | I.setName(I.getName() + ".not"); | ||||
5343 | |||||
5344 | // And now let's adjust every user. | ||||
5345 | for (User *U : I.users()) { | ||||
5346 | switch (cast<Instruction>(U)->getOpcode()) { | ||||
5347 | case Instruction::Select: { | ||||
5348 | auto *SI = cast<SelectInst>(U); | ||||
5349 | SI->swapValues(); | ||||
5350 | SI->swapProfMetadata(); | ||||
5351 | break; | ||||
5352 | } | ||||
5353 | case Instruction::Br: | ||||
5354 | cast<BranchInst>(U)->swapSuccessors(); // swaps prof metadata too | ||||
5355 | break; | ||||
5356 | case Instruction::Xor: | ||||
5357 | replaceInstUsesWith(cast<Instruction>(*U), &I); | ||||
5358 | break; | ||||
5359 | default: | ||||
5360 | llvm_unreachable("Got unexpected user - out of sync with "::llvm::llvm_unreachable_internal("Got unexpected user - out of sync with " "canFreelyInvertAllUsersOf() ?", "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5361) | ||||
5361 | "canFreelyInvertAllUsersOf() ?")::llvm::llvm_unreachable_internal("Got unexpected user - out of sync with " "canFreelyInvertAllUsersOf() ?", "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5361); | ||||
5362 | } | ||||
5363 | } | ||||
5364 | |||||
5365 | return &I; | ||||
5366 | } | ||||
5367 | |||||
5368 | /// Integer compare with boolean values can always be turned into bitwise ops. | ||||
5369 | static Instruction *canonicalizeICmpBool(ICmpInst &I, | ||||
5370 | InstCombiner::BuilderTy &Builder) { | ||||
5371 | Value *A = I.getOperand(0), *B = I.getOperand(1); | ||||
5372 | assert(A->getType()->isIntOrIntVectorTy(1) && "Bools only")((A->getType()->isIntOrIntVectorTy(1) && "Bools only" ) ? static_cast<void> (0) : __assert_fail ("A->getType()->isIntOrIntVectorTy(1) && \"Bools only\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5372, __PRETTY_FUNCTION__)); | ||||
5373 | |||||
5374 | // A boolean compared to true/false can be simplified to Op0/true/false in | ||||
5375 | // 14 out of the 20 (10 predicates * 2 constants) possible combinations. | ||||
5376 | // Cases not handled by InstSimplify are always 'not' of Op0. | ||||
5377 | if (match(B, m_Zero())) { | ||||
5378 | switch (I.getPredicate()) { | ||||
5379 | case CmpInst::ICMP_EQ: // A == 0 -> !A | ||||
5380 | case CmpInst::ICMP_ULE: // A <=u 0 -> !A | ||||
5381 | case CmpInst::ICMP_SGE: // A >=s 0 -> !A | ||||
5382 | return BinaryOperator::CreateNot(A); | ||||
5383 | default: | ||||
5384 | llvm_unreachable("ICmp i1 X, C not simplified as expected.")::llvm::llvm_unreachable_internal("ICmp i1 X, C not simplified as expected." , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5384); | ||||
5385 | } | ||||
5386 | } else if (match(B, m_One())) { | ||||
5387 | switch (I.getPredicate()) { | ||||
5388 | case CmpInst::ICMP_NE: // A != 1 -> !A | ||||
5389 | case CmpInst::ICMP_ULT: // A <u 1 -> !A | ||||
5390 | case CmpInst::ICMP_SGT: // A >s -1 -> !A | ||||
5391 | return BinaryOperator::CreateNot(A); | ||||
5392 | default: | ||||
5393 | llvm_unreachable("ICmp i1 X, C not simplified as expected.")::llvm::llvm_unreachable_internal("ICmp i1 X, C not simplified as expected." , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5393); | ||||
5394 | } | ||||
5395 | } | ||||
5396 | |||||
5397 | switch (I.getPredicate()) { | ||||
5398 | default: | ||||
5399 | llvm_unreachable("Invalid icmp instruction!")::llvm::llvm_unreachable_internal("Invalid icmp instruction!" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5399); | ||||
5400 | case ICmpInst::ICMP_EQ: | ||||
5401 | // icmp eq i1 A, B -> ~(A ^ B) | ||||
5402 | return BinaryOperator::CreateNot(Builder.CreateXor(A, B)); | ||||
5403 | |||||
5404 | case ICmpInst::ICMP_NE: | ||||
5405 | // icmp ne i1 A, B -> A ^ B | ||||
5406 | return BinaryOperator::CreateXor(A, B); | ||||
5407 | |||||
5408 | case ICmpInst::ICMP_UGT: | ||||
5409 | // icmp ugt -> icmp ult | ||||
5410 | std::swap(A, B); | ||||
5411 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||
5412 | case ICmpInst::ICMP_ULT: | ||||
5413 | // icmp ult i1 A, B -> ~A & B | ||||
5414 | return BinaryOperator::CreateAnd(Builder.CreateNot(A), B); | ||||
5415 | |||||
5416 | case ICmpInst::ICMP_SGT: | ||||
5417 | // icmp sgt -> icmp slt | ||||
5418 | std::swap(A, B); | ||||
5419 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||
5420 | case ICmpInst::ICMP_SLT: | ||||
5421 | // icmp slt i1 A, B -> A & ~B | ||||
5422 | return BinaryOperator::CreateAnd(Builder.CreateNot(B), A); | ||||
5423 | |||||
5424 | case ICmpInst::ICMP_UGE: | ||||
5425 | // icmp uge -> icmp ule | ||||
5426 | std::swap(A, B); | ||||
5427 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||
5428 | case ICmpInst::ICMP_ULE: | ||||
5429 | // icmp ule i1 A, B -> ~A | B | ||||
5430 | return BinaryOperator::CreateOr(Builder.CreateNot(A), B); | ||||
5431 | |||||
5432 | case ICmpInst::ICMP_SGE: | ||||
5433 | // icmp sge -> icmp sle | ||||
5434 | std::swap(A, B); | ||||
5435 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||
5436 | case ICmpInst::ICMP_SLE: | ||||
5437 | // icmp sle i1 A, B -> A | ~B | ||||
5438 | return BinaryOperator::CreateOr(Builder.CreateNot(B), A); | ||||
5439 | } | ||||
5440 | } | ||||
5441 | |||||
5442 | // Transform pattern like: | ||||
5443 | // (1 << Y) u<= X or ~(-1 << Y) u< X or ((1 << Y)+(-1)) u< X | ||||
5444 | // (1 << Y) u> X or ~(-1 << Y) u>= X or ((1 << Y)+(-1)) u>= X | ||||
5445 | // Into: | ||||
5446 | // (X l>> Y) != 0 | ||||
5447 | // (X l>> Y) == 0 | ||||
5448 | static Instruction *foldICmpWithHighBitMask(ICmpInst &Cmp, | ||||
5449 | InstCombiner::BuilderTy &Builder) { | ||||
5450 | ICmpInst::Predicate Pred, NewPred; | ||||
5451 | Value *X, *Y; | ||||
5452 | if (match(&Cmp, | ||||
5453 | m_c_ICmp(Pred, m_OneUse(m_Shl(m_One(), m_Value(Y))), m_Value(X)))) { | ||||
5454 | switch (Pred) { | ||||
5455 | case ICmpInst::ICMP_ULE: | ||||
5456 | NewPred = ICmpInst::ICMP_NE; | ||||
5457 | break; | ||||
5458 | case ICmpInst::ICMP_UGT: | ||||
5459 | NewPred = ICmpInst::ICMP_EQ; | ||||
5460 | break; | ||||
5461 | default: | ||||
5462 | return nullptr; | ||||
5463 | } | ||||
5464 | } else if (match(&Cmp, m_c_ICmp(Pred, | ||||
5465 | m_OneUse(m_CombineOr( | ||||
5466 | m_Not(m_Shl(m_AllOnes(), m_Value(Y))), | ||||
5467 | m_Add(m_Shl(m_One(), m_Value(Y)), | ||||
5468 | m_AllOnes()))), | ||||
5469 | m_Value(X)))) { | ||||
5470 | // The variant with 'add' is not canonical, (the variant with 'not' is) | ||||
5471 | // we only get it because it has extra uses, and can't be canonicalized, | ||||
5472 | |||||
5473 | switch (Pred) { | ||||
5474 | case ICmpInst::ICMP_ULT: | ||||
5475 | NewPred = ICmpInst::ICMP_NE; | ||||
5476 | break; | ||||
5477 | case ICmpInst::ICMP_UGE: | ||||
5478 | NewPred = ICmpInst::ICMP_EQ; | ||||
5479 | break; | ||||
5480 | default: | ||||
5481 | return nullptr; | ||||
5482 | } | ||||
5483 | } else | ||||
5484 | return nullptr; | ||||
5485 | |||||
5486 | Value *NewX = Builder.CreateLShr(X, Y, X->getName() + ".highbits"); | ||||
5487 | Constant *Zero = Constant::getNullValue(NewX->getType()); | ||||
5488 | return CmpInst::Create(Instruction::ICmp, NewPred, NewX, Zero); | ||||
5489 | } | ||||
5490 | |||||
5491 | static Instruction *foldVectorCmp(CmpInst &Cmp, | ||||
5492 | InstCombiner::BuilderTy &Builder) { | ||||
5493 | const CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||
5494 | Value *LHS = Cmp.getOperand(0), *RHS = Cmp.getOperand(1); | ||||
5495 | Value *V1, *V2; | ||||
5496 | ArrayRef<int> M; | ||||
5497 | if (!match(LHS, m_Shuffle(m_Value(V1), m_Undef(), m_Mask(M)))) | ||||
5498 | return nullptr; | ||||
5499 | |||||
5500 | // If both arguments of the cmp are shuffles that use the same mask and | ||||
5501 | // shuffle within a single vector, move the shuffle after the cmp: | ||||
5502 | // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M | ||||
5503 | Type *V1Ty = V1->getType(); | ||||
5504 | if (match(RHS, m_Shuffle(m_Value(V2), m_Undef(), m_SpecificMask(M))) && | ||||
5505 | V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) { | ||||
5506 | Value *NewCmp = Builder.CreateCmp(Pred, V1, V2); | ||||
5507 | return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), M); | ||||
5508 | } | ||||
5509 | |||||
5510 | // Try to canonicalize compare with splatted operand and splat constant. | ||||
5511 | // TODO: We could generalize this for more than splats. See/use the code in | ||||
5512 | // InstCombiner::foldVectorBinop(). | ||||
5513 | Constant *C; | ||||
5514 | if (!LHS->hasOneUse() || !match(RHS, m_Constant(C))) | ||||
5515 | return nullptr; | ||||
5516 | |||||
5517 | // Length-changing splats are ok, so adjust the constants as needed: | ||||
5518 | // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M | ||||
5519 | Constant *ScalarC = C->getSplatValue(/* AllowUndefs */ true); | ||||
5520 | int MaskSplatIndex; | ||||
5521 | if (ScalarC && match(M, m_SplatOrUndefMask(MaskSplatIndex))) { | ||||
5522 | // We allow undefs in matching, but this transform removes those for safety. | ||||
5523 | // Demanded elements analysis should be able to recover some/all of that. | ||||
5524 | C = ConstantVector::getSplat(cast<VectorType>(V1Ty)->getElementCount(), | ||||
5525 | ScalarC); | ||||
5526 | SmallVector<int, 8> NewM(M.size(), MaskSplatIndex); | ||||
5527 | Value *NewCmp = Builder.CreateCmp(Pred, V1, C); | ||||
5528 | return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), | ||||
5529 | NewM); | ||||
5530 | } | ||||
5531 | |||||
5532 | return nullptr; | ||||
5533 | } | ||||
5534 | |||||
5535 | // extract(uadd.with.overflow(A, B), 0) ult A | ||||
5536 | // -> extract(uadd.with.overflow(A, B), 1) | ||||
5537 | static Instruction *foldICmpOfUAddOv(ICmpInst &I) { | ||||
5538 | CmpInst::Predicate Pred = I.getPredicate(); | ||||
5539 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||
5540 | |||||
5541 | Value *UAddOv; | ||||
5542 | Value *A, *B; | ||||
5543 | auto UAddOvResultPat = m_ExtractValue<0>( | ||||
5544 | m_Intrinsic<Intrinsic::uadd_with_overflow>(m_Value(A), m_Value(B))); | ||||
5545 | if (match(Op0, UAddOvResultPat) && | ||||
5546 | ((Pred == ICmpInst::ICMP_ULT && (Op1 == A || Op1 == B)) || | ||||
5547 | (Pred == ICmpInst::ICMP_EQ && match(Op1, m_ZeroInt()) && | ||||
5548 | (match(A, m_One()) || match(B, m_One()))) || | ||||
5549 | (Pred == ICmpInst::ICMP_NE && match(Op1, m_AllOnes()) && | ||||
5550 | (match(A, m_AllOnes()) || match(B, m_AllOnes()))))) | ||||
5551 | // extract(uadd.with.overflow(A, B), 0) < A | ||||
5552 | // extract(uadd.with.overflow(A, 1), 0) == 0 | ||||
5553 | // extract(uadd.with.overflow(A, -1), 0) != -1 | ||||
5554 | UAddOv = cast<ExtractValueInst>(Op0)->getAggregateOperand(); | ||||
5555 | else if (match(Op1, UAddOvResultPat) && | ||||
5556 | Pred == ICmpInst::ICMP_UGT && (Op0 == A || Op0 == B)) | ||||
5557 | // A > extract(uadd.with.overflow(A, B), 0) | ||||
5558 | UAddOv = cast<ExtractValueInst>(Op1)->getAggregateOperand(); | ||||
5559 | else | ||||
5560 | return nullptr; | ||||
5561 | |||||
5562 | return ExtractValueInst::Create(UAddOv, 1); | ||||
5563 | } | ||||
5564 | |||||
5565 | Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) { | ||||
5566 | bool Changed = false; | ||||
5567 | const SimplifyQuery Q = SQ.getWithInstruction(&I); | ||||
5568 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||
5569 | unsigned Op0Cplxity = getComplexity(Op0); | ||||
5570 | unsigned Op1Cplxity = getComplexity(Op1); | ||||
5571 | |||||
5572 | /// Orders the operands of the compare so that they are listed from most | ||||
5573 | /// complex to least complex. This puts constants before unary operators, | ||||
5574 | /// before binary operators. | ||||
5575 | if (Op0Cplxity
| ||||
5576 | (Op0Cplxity == Op1Cplxity && swapMayExposeCSEOpportunities(Op0, Op1))) { | ||||
5577 | I.swapOperands(); | ||||
5578 | std::swap(Op0, Op1); | ||||
5579 | Changed = true; | ||||
5580 | } | ||||
5581 | |||||
5582 | if (Value *V = SimplifyICmpInst(I.getPredicate(), Op0, Op1, Q)) | ||||
| |||||
5583 | return replaceInstUsesWith(I, V); | ||||
5584 | |||||
5585 | // Comparing -val or val with non-zero is the same as just comparing val | ||||
5586 | // ie, abs(val) != 0 -> val != 0 | ||||
5587 | if (I.getPredicate() == ICmpInst::ICMP_NE && match(Op1, m_Zero())) { | ||||
5588 | Value *Cond, *SelectTrue, *SelectFalse; | ||||
5589 | if (match(Op0, m_Select(m_Value(Cond), m_Value(SelectTrue), | ||||
5590 | m_Value(SelectFalse)))) { | ||||
5591 | if (Value *V = dyn_castNegVal(SelectTrue)) { | ||||
5592 | if (V == SelectFalse) | ||||
5593 | return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1); | ||||
5594 | } | ||||
5595 | else if (Value *V = dyn_castNegVal(SelectFalse)) { | ||||
5596 | if (V == SelectTrue) | ||||
5597 | return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1); | ||||
5598 | } | ||||
5599 | } | ||||
5600 | } | ||||
5601 | |||||
5602 | if (Op0->getType()->isIntOrIntVectorTy(1)) | ||||
5603 | if (Instruction *Res = canonicalizeICmpBool(I, Builder)) | ||||
5604 | return Res; | ||||
5605 | |||||
5606 | if (Instruction *Res
| ||||
5607 | return Res; | ||||
5608 | |||||
5609 | if (Instruction *Res
| ||||
5610 | return Res; | ||||
5611 | |||||
5612 | if (Instruction *Res
| ||||
5613 | return Res; | ||||
5614 | |||||
5615 | if (Instruction *Res
| ||||
5616 | return Res; | ||||
5617 | |||||
5618 | if (Instruction *Res = foldICmpBinOp(I, Q)) | ||||
5619 | return Res; | ||||
5620 | |||||
5621 | if (Instruction *Res = foldICmpUsingKnownBits(I)) | ||||
5622 | return Res; | ||||
5623 | |||||
5624 | // Test if the ICmpInst instruction is used exclusively by a select as | ||||
5625 | // part of a minimum or maximum operation. If so, refrain from doing | ||||
5626 | // any other folding. This helps out other analyses which understand | ||||
5627 | // non-obfuscated minimum and maximum idioms, such as ScalarEvolution | ||||
5628 | // and CodeGen. And in this case, at least one of the comparison | ||||
5629 | // operands has at least one user besides the compare (the select), | ||||
5630 | // which would often largely negate the benefit of folding anyway. | ||||
5631 | // | ||||
5632 | // Do the same for the other patterns recognized by matchSelectPattern. | ||||
5633 | if (I.hasOneUse()) | ||||
5634 | if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) { | ||||
5635 | Value *A, *B; | ||||
5636 | SelectPatternResult SPR = matchSelectPattern(SI, A, B); | ||||
5637 | if (SPR.Flavor != SPF_UNKNOWN) | ||||
5638 | return nullptr; | ||||
5639 | } | ||||
5640 | |||||
5641 | // Do this after checking for min/max to prevent infinite looping. | ||||
5642 | if (Instruction *Res
| ||||
5643 | return Res; | ||||
5644 | |||||
5645 | // FIXME: We only do this after checking for min/max to prevent infinite | ||||
5646 | // looping caused by a reverse canonicalization of these patterns for min/max. | ||||
5647 | // FIXME: The organization of folds is a mess. These would naturally go into | ||||
5648 | // canonicalizeCmpWithConstant(), but we can't move all of the above folds | ||||
5649 | // down here after the min/max restriction. | ||||
5650 | ICmpInst::Predicate Pred = I.getPredicate(); | ||||
5651 | const APInt *C; | ||||
5652 | if (match(Op1, m_APInt(C))) { | ||||
5653 | // For i32: x >u 2147483647 -> x <s 0 -> true if sign bit set | ||||
5654 | if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) { | ||||
5655 | Constant *Zero = Constant::getNullValue(Op0->getType()); | ||||
5656 | return new ICmpInst(ICmpInst::ICMP_SLT, Op0, Zero); | ||||
5657 | } | ||||
5658 | |||||
5659 | // For i32: x <u 2147483648 -> x >s -1 -> true if sign bit clear | ||||
5660 | if (Pred == ICmpInst::ICMP_ULT && C->isMinSignedValue()) { | ||||
5661 | Constant *AllOnes = Constant::getAllOnesValue(Op0->getType()); | ||||
5662 | return new ICmpInst(ICmpInst::ICMP_SGT, Op0, AllOnes); | ||||
5663 | } | ||||
5664 | } | ||||
5665 | |||||
5666 | if (Instruction *Res
| ||||
5667 | return Res; | ||||
5668 | |||||
5669 | // Try to match comparison as a sign bit test. Intentionally do this after | ||||
5670 | // foldICmpInstWithConstant() to potentially let other folds to happen first. | ||||
5671 | if (Instruction *New
| ||||
5672 | return New; | ||||
5673 | |||||
5674 | if (Instruction *Res
| ||||
5675 | return Res; | ||||
5676 | |||||
5677 | // If we can optimize a 'icmp GEP, P' or 'icmp P, GEP', do so now. | ||||
5678 | if (GEPOperator *GEP
| ||||
5679 | if (Instruction *NI = foldGEPICmp(GEP, Op1, I.getPredicate(), I)) | ||||
5680 | return NI; | ||||
5681 | if (GEPOperator *GEP = dyn_cast<GEPOperator>(Op1)) | ||||
5682 | if (Instruction *NI = foldGEPICmp(GEP, Op0, | ||||
5683 | ICmpInst::getSwappedPredicate(I.getPredicate()), I)) | ||||
5684 | return NI; | ||||
5685 | |||||
5686 | // Try to optimize equality comparisons against alloca-based pointers. | ||||
5687 | if (Op0->getType()->isPointerTy() && I.isEquality()) { | ||||
5688 | assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?")((Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?" ) ? static_cast<void> (0) : __assert_fail ("Op1->getType()->isPointerTy() && \"Comparing pointer with non-pointer?\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5688, __PRETTY_FUNCTION__)); | ||||
5689 | if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0))) | ||||
5690 | if (Instruction *New = foldAllocaCmp(I, Alloca, Op1)) | ||||
5691 | return New; | ||||
5692 | if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op1))) | ||||
5693 | if (Instruction *New = foldAllocaCmp(I, Alloca, Op0)) | ||||
5694 | return New; | ||||
5695 | } | ||||
5696 | |||||
5697 | if (Instruction *Res = foldICmpBitCast(I, Builder)) | ||||
5698 | return Res; | ||||
5699 | |||||
5700 | // TODO: Hoist this above the min/max bailout. | ||||
5701 | if (Instruction *R = foldICmpWithCastOp(I)) | ||||
5702 | return R; | ||||
5703 | |||||
5704 | if (Instruction *Res = foldICmpWithMinMax(I)) | ||||
5705 | return Res; | ||||
5706 | |||||
5707 | { | ||||
5708 | Value *A, *B; | ||||
5709 | // Transform (A & ~B) == 0 --> (A & B) != 0 | ||||
5710 | // and (A & ~B) != 0 --> (A & B) == 0 | ||||
5711 | // if A is a power of 2. | ||||
5712 | if (match(Op0, m_And(m_Value(A), m_Not(m_Value(B)))) && | ||||
5713 | match(Op1, m_Zero()) && | ||||
5714 | isKnownToBeAPowerOfTwo(A, false, 0, &I) && I.isEquality()) | ||||
5715 | return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(A, B), | ||||
5716 | Op1); | ||||
5717 | |||||
5718 | // ~X < ~Y --> Y < X | ||||
5719 | // ~X < C --> X > ~C | ||||
5720 | if (match(Op0, m_Not(m_Value(A)))) { | ||||
5721 | if (match(Op1, m_Not(m_Value(B)))) | ||||
5722 | return new ICmpInst(I.getPredicate(), B, A); | ||||
5723 | |||||
5724 | const APInt *C; | ||||
5725 | if (match(Op1, m_APInt(C))) | ||||
5726 | return new ICmpInst(I.getSwappedPredicate(), A, | ||||
5727 | ConstantInt::get(Op1->getType(), ~(*C))); | ||||
5728 | } | ||||
5729 | |||||
5730 | Instruction *AddI = nullptr; | ||||
5731 | if (match(&I, m_UAddWithOverflow(m_Value(A), m_Value(B), | ||||
5732 | m_Instruction(AddI))) && | ||||
5733 | isa<IntegerType>(A->getType())) { | ||||
5734 | Value *Result; | ||||
5735 | Constant *Overflow; | ||||
5736 | // m_UAddWithOverflow can match patterns that do not include an explicit | ||||
5737 | // "add" instruction, so check the opcode of the matched op. | ||||
5738 | if (AddI->getOpcode() == Instruction::Add && | ||||
5739 | OptimizeOverflowCheck(Instruction::Add, /*Signed*/ false, A, B, *AddI, | ||||
5740 | Result, Overflow)) { | ||||
5741 | replaceInstUsesWith(*AddI, Result); | ||||
5742 | eraseInstFromFunction(*AddI); | ||||
5743 | return replaceInstUsesWith(I, Overflow); | ||||
5744 | } | ||||
5745 | } | ||||
5746 | |||||
5747 | // (zext a) * (zext b) --> llvm.umul.with.overflow. | ||||
5748 | if (match(Op0, m_Mul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) { | ||||
5749 | if (Instruction *R = processUMulZExtIdiom(I, Op0, Op1, *this)) | ||||
5750 | return R; | ||||
5751 | } | ||||
5752 | if (match(Op1, m_Mul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) { | ||||
5753 | if (Instruction *R = processUMulZExtIdiom(I, Op1, Op0, *this)) | ||||
5754 | return R; | ||||
5755 | } | ||||
5756 | } | ||||
5757 | |||||
5758 | if (Instruction *Res = foldICmpEquality(I)) | ||||
5759 | return Res; | ||||
5760 | |||||
5761 | if (Instruction *Res = foldICmpOfUAddOv(I)) | ||||
5762 | return Res; | ||||
5763 | |||||
5764 | // The 'cmpxchg' instruction returns an aggregate containing the old value and | ||||
5765 | // an i1 which indicates whether or not we successfully did the swap. | ||||
5766 | // | ||||
5767 | // Replace comparisons between the old value and the expected value with the | ||||
5768 | // indicator that 'cmpxchg' returns. | ||||
5769 | // | ||||
5770 | // N.B. This transform is only valid when the 'cmpxchg' is not permitted to | ||||
5771 | // spuriously fail. In those cases, the old value may equal the expected | ||||
5772 | // value but it is possible for the swap to not occur. | ||||
5773 | if (I.getPredicate() == ICmpInst::ICMP_EQ) | ||||
5774 | if (auto *EVI = dyn_cast<ExtractValueInst>(Op0)) | ||||
5775 | if (auto *ACXI = dyn_cast<AtomicCmpXchgInst>(EVI->getAggregateOperand())) | ||||
5776 | if (EVI->getIndices()[0] == 0 && ACXI->getCompareOperand() == Op1 && | ||||
5777 | !ACXI->isWeak()) | ||||
5778 | return ExtractValueInst::Create(ACXI, 1); | ||||
5779 | |||||
5780 | { | ||||
5781 | Value *X; | ||||
5782 | const APInt *C; | ||||
5783 | // icmp X+Cst, X | ||||
5784 | if (match(Op0, m_Add(m_Value(X), m_APInt(C))) && Op1 == X) | ||||
5785 | return foldICmpAddOpConst(X, *C, I.getPredicate()); | ||||
5786 | |||||
5787 | // icmp X, X+Cst | ||||
5788 | if (match(Op1, m_Add(m_Value(X), m_APInt(C))) && Op0 == X) | ||||
5789 | return foldICmpAddOpConst(X, *C, I.getSwappedPredicate()); | ||||
5790 | } | ||||
5791 | |||||
5792 | if (Instruction *Res = foldICmpWithHighBitMask(I, Builder)) | ||||
5793 | return Res; | ||||
5794 | |||||
5795 | if (I.getType()->isVectorTy()) | ||||
5796 | if (Instruction *Res = foldVectorCmp(I, Builder)) | ||||
5797 | return Res; | ||||
5798 | |||||
5799 | return Changed ? &I : nullptr; | ||||
5800 | } | ||||
5801 | |||||
5802 | /// Fold fcmp ([us]itofp x, cst) if possible. | ||||
5803 | Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, | ||||
5804 | Instruction *LHSI, | ||||
5805 | Constant *RHSC) { | ||||
5806 | if (!isa<ConstantFP>(RHSC)) return nullptr; | ||||
5807 | const APFloat &RHS = cast<ConstantFP>(RHSC)->getValueAPF(); | ||||
5808 | |||||
5809 | // Get the width of the mantissa. We don't want to hack on conversions that | ||||
5810 | // might lose information from the integer, e.g. "i64 -> float" | ||||
5811 | int MantissaWidth = LHSI->getType()->getFPMantissaWidth(); | ||||
5812 | if (MantissaWidth == -1) return nullptr; // Unknown. | ||||
5813 | |||||
5814 | IntegerType *IntTy = cast<IntegerType>(LHSI->getOperand(0)->getType()); | ||||
5815 | |||||
5816 | bool LHSUnsigned = isa<UIToFPInst>(LHSI); | ||||
5817 | |||||
5818 | if (I.isEquality()) { | ||||
5819 | FCmpInst::Predicate P = I.getPredicate(); | ||||
5820 | bool IsExact = false; | ||||
5821 | APSInt RHSCvt(IntTy->getBitWidth(), LHSUnsigned); | ||||
5822 | RHS.convertToInteger(RHSCvt, APFloat::rmNearestTiesToEven, &IsExact); | ||||
5823 | |||||
5824 | // If the floating point constant isn't an integer value, we know if we will | ||||
5825 | // ever compare equal / not equal to it. | ||||
5826 | if (!IsExact) { | ||||
5827 | // TODO: Can never be -0.0 and other non-representable values | ||||
5828 | APFloat RHSRoundInt(RHS); | ||||
5829 | RHSRoundInt.roundToIntegral(APFloat::rmNearestTiesToEven); | ||||
5830 | if (RHS != RHSRoundInt) { | ||||
5831 | if (P == FCmpInst::FCMP_OEQ || P == FCmpInst::FCMP_UEQ) | ||||
5832 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||
5833 | |||||
5834 | assert(P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE)((P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE) ? static_cast <void> (0) : __assert_fail ("P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5834, __PRETTY_FUNCTION__)); | ||||
5835 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||
5836 | } | ||||
5837 | } | ||||
5838 | |||||
5839 | // TODO: If the constant is exactly representable, is it always OK to do | ||||
5840 | // equality compares as integer? | ||||
5841 | } | ||||
5842 | |||||
5843 | // Check to see that the input is converted from an integer type that is small | ||||
5844 | // enough that preserves all bits. TODO: check here for "known" sign bits. | ||||
5845 | // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e. | ||||
5846 | unsigned InputSize = IntTy->getScalarSizeInBits(); | ||||
5847 | |||||
5848 | // Following test does NOT adjust InputSize downwards for signed inputs, | ||||
5849 | // because the most negative value still requires all the mantissa bits | ||||
5850 | // to distinguish it from one less than that value. | ||||
5851 | if ((int)InputSize > MantissaWidth) { | ||||
5852 | // Conversion would lose accuracy. Check if loss can impact comparison. | ||||
5853 | int Exp = ilogb(RHS); | ||||
5854 | if (Exp == APFloat::IEK_Inf) { | ||||
5855 | int MaxExponent = ilogb(APFloat::getLargest(RHS.getSemantics())); | ||||
5856 | if (MaxExponent < (int)InputSize - !LHSUnsigned) | ||||
5857 | // Conversion could create infinity. | ||||
5858 | return nullptr; | ||||
5859 | } else { | ||||
5860 | // Note that if RHS is zero or NaN, then Exp is negative | ||||
5861 | // and first condition is trivially false. | ||||
5862 | if (MantissaWidth <= Exp && Exp <= (int)InputSize - !LHSUnsigned) | ||||
5863 | // Conversion could affect comparison. | ||||
5864 | return nullptr; | ||||
5865 | } | ||||
5866 | } | ||||
5867 | |||||
5868 | // Otherwise, we can potentially simplify the comparison. We know that it | ||||
5869 | // will always come through as an integer value and we know the constant is | ||||
5870 | // not a NAN (it would have been previously simplified). | ||||
5871 | assert(!RHS.isNaN() && "NaN comparison not already folded!")((!RHS.isNaN() && "NaN comparison not already folded!" ) ? static_cast<void> (0) : __assert_fail ("!RHS.isNaN() && \"NaN comparison not already folded!\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5871, __PRETTY_FUNCTION__)); | ||||
5872 | |||||
5873 | ICmpInst::Predicate Pred; | ||||
5874 | switch (I.getPredicate()) { | ||||
5875 | default: llvm_unreachable("Unexpected predicate!")::llvm::llvm_unreachable_internal("Unexpected predicate!", "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5875); | ||||
5876 | case FCmpInst::FCMP_UEQ: | ||||
5877 | case FCmpInst::FCMP_OEQ: | ||||
5878 | Pred = ICmpInst::ICMP_EQ; | ||||
5879 | break; | ||||
5880 | case FCmpInst::FCMP_UGT: | ||||
5881 | case FCmpInst::FCMP_OGT: | ||||
5882 | Pred = LHSUnsigned ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_SGT; | ||||
5883 | break; | ||||
5884 | case FCmpInst::FCMP_UGE: | ||||
5885 | case FCmpInst::FCMP_OGE: | ||||
5886 | Pred = LHSUnsigned ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_SGE; | ||||
5887 | break; | ||||
5888 | case FCmpInst::FCMP_ULT: | ||||
5889 | case FCmpInst::FCMP_OLT: | ||||
5890 | Pred = LHSUnsigned ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_SLT; | ||||
5891 | break; | ||||
5892 | case FCmpInst::FCMP_ULE: | ||||
5893 | case FCmpInst::FCMP_OLE: | ||||
5894 | Pred = LHSUnsigned ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_SLE; | ||||
5895 | break; | ||||
5896 | case FCmpInst::FCMP_UNE: | ||||
5897 | case FCmpInst::FCMP_ONE: | ||||
5898 | Pred = ICmpInst::ICMP_NE; | ||||
5899 | break; | ||||
5900 | case FCmpInst::FCMP_ORD: | ||||
5901 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||
5902 | case FCmpInst::FCMP_UNO: | ||||
5903 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||
5904 | } | ||||
5905 | |||||
5906 | // Now we know that the APFloat is a normal number, zero or inf. | ||||
5907 | |||||
5908 | // See if the FP constant is too large for the integer. For example, | ||||
5909 | // comparing an i8 to 300.0. | ||||
5910 | unsigned IntWidth = IntTy->getScalarSizeInBits(); | ||||
5911 | |||||
5912 | if (!LHSUnsigned) { | ||||
5913 | // If the RHS value is > SignedMax, fold the comparison. This handles +INF | ||||
5914 | // and large values. | ||||
5915 | APFloat SMax(RHS.getSemantics()); | ||||
5916 | SMax.convertFromAPInt(APInt::getSignedMaxValue(IntWidth), true, | ||||
5917 | APFloat::rmNearestTiesToEven); | ||||
5918 | if (SMax < RHS) { // smax < 13123.0 | ||||
5919 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT || | ||||
5920 | Pred == ICmpInst::ICMP_SLE) | ||||
5921 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||
5922 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||
5923 | } | ||||
5924 | } else { | ||||
5925 | // If the RHS value is > UnsignedMax, fold the comparison. This handles | ||||
5926 | // +INF and large values. | ||||
5927 | APFloat UMax(RHS.getSemantics()); | ||||
5928 | UMax.convertFromAPInt(APInt::getMaxValue(IntWidth), false, | ||||
5929 | APFloat::rmNearestTiesToEven); | ||||
5930 | if (UMax < RHS) { // umax < 13123.0 | ||||
5931 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT || | ||||
5932 | Pred == ICmpInst::ICMP_ULE) | ||||
5933 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||
5934 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||
5935 | } | ||||
5936 | } | ||||
5937 | |||||
5938 | if (!LHSUnsigned) { | ||||
5939 | // See if the RHS value is < SignedMin. | ||||
5940 | APFloat SMin(RHS.getSemantics()); | ||||
5941 | SMin.convertFromAPInt(APInt::getSignedMinValue(IntWidth), true, | ||||
5942 | APFloat::rmNearestTiesToEven); | ||||
5943 | if (SMin > RHS) { // smin > 12312.0 | ||||
5944 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT || | ||||
5945 | Pred == ICmpInst::ICMP_SGE) | ||||
5946 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||
5947 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||
5948 | } | ||||
5949 | } else { | ||||
5950 | // See if the RHS value is < UnsignedMin. | ||||
5951 | APFloat UMin(RHS.getSemantics()); | ||||
5952 | UMin.convertFromAPInt(APInt::getMinValue(IntWidth), false, | ||||
5953 | APFloat::rmNearestTiesToEven); | ||||
5954 | if (UMin > RHS) { // umin > 12312.0 | ||||
5955 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT || | ||||
5956 | Pred == ICmpInst::ICMP_UGE) | ||||
5957 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||
5958 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||
5959 | } | ||||
5960 | } | ||||
5961 | |||||
5962 | // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or | ||||
5963 | // [0, UMAX], but it may still be fractional. See if it is fractional by | ||||
5964 | // casting the FP value to the integer value and back, checking for equality. | ||||
5965 | // Don't do this for zero, because -0.0 is not fractional. | ||||
5966 | Constant *RHSInt = LHSUnsigned | ||||
5967 | ? ConstantExpr::getFPToUI(RHSC, IntTy) | ||||
5968 | : ConstantExpr::getFPToSI(RHSC, IntTy); | ||||
5969 | if (!RHS.isZero()) { | ||||
5970 | bool Equal = LHSUnsigned | ||||
5971 | ? ConstantExpr::getUIToFP(RHSInt, RHSC->getType()) == RHSC | ||||
5972 | : ConstantExpr::getSIToFP(RHSInt, RHSC->getType()) == RHSC; | ||||
5973 | if (!Equal) { | ||||
5974 | // If we had a comparison against a fractional value, we have to adjust | ||||
5975 | // the compare predicate and sometimes the value. RHSC is rounded towards | ||||
5976 | // zero at this point. | ||||
5977 | switch (Pred) { | ||||
5978 | default: llvm_unreachable("Unexpected integer comparison!")::llvm::llvm_unreachable_internal("Unexpected integer comparison!" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5978); | ||||
5979 | case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true | ||||
5980 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||
5981 | case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false | ||||
5982 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||
5983 | case ICmpInst::ICMP_ULE: | ||||
5984 | // (float)int <= 4.4 --> int <= 4 | ||||
5985 | // (float)int <= -4.4 --> false | ||||
5986 | if (RHS.isNegative()) | ||||
5987 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||
5988 | break; | ||||
5989 | case ICmpInst::ICMP_SLE: | ||||
5990 | // (float)int <= 4.4 --> int <= 4 | ||||
5991 | // (float)int <= -4.4 --> int < -4 | ||||
5992 | if (RHS.isNegative()) | ||||
5993 | Pred = ICmpInst::ICMP_SLT; | ||||
5994 | break; | ||||
5995 | case ICmpInst::ICMP_ULT: | ||||
5996 | // (float)int < -4.4 --> false | ||||
5997 | // (float)int < 4.4 --> int <= 4 | ||||
5998 | if (RHS.isNegative()) | ||||
5999 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||
6000 | Pred = ICmpInst::ICMP_ULE; | ||||
6001 | break; | ||||
6002 | case ICmpInst::ICMP_SLT: | ||||
6003 | // (float)int < -4.4 --> int < -4 | ||||
6004 | // (float)int < 4.4 --> int <= 4 | ||||
6005 | if (!RHS.isNegative()) | ||||
6006 | Pred = ICmpInst::ICMP_SLE; | ||||
6007 | break; | ||||
6008 | case ICmpInst::ICMP_UGT: | ||||
6009 | // (float)int > 4.4 --> int > 4 | ||||
6010 | // (float)int > -4.4 --> true | ||||
6011 | if (RHS.isNegative()) | ||||
6012 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||
6013 | break; | ||||
6014 | case ICmpInst::ICMP_SGT: | ||||
6015 | // (float)int > 4.4 --> int > 4 | ||||
6016 | // (float)int > -4.4 --> int >= -4 | ||||
6017 | if (RHS.isNegative()) | ||||
6018 | Pred = ICmpInst::ICMP_SGE; | ||||
6019 | break; | ||||
6020 | case ICmpInst::ICMP_UGE: | ||||
6021 | // (float)int >= -4.4 --> true | ||||
6022 | // (float)int >= 4.4 --> int > 4 | ||||
6023 | if (RHS.isNegative()) | ||||
6024 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||
6025 | Pred = ICmpInst::ICMP_UGT; | ||||
6026 | break; | ||||
6027 | case ICmpInst::ICMP_SGE: | ||||
6028 | // (float)int >= -4.4 --> int >= -4 | ||||
6029 | // (float)int >= 4.4 --> int > 4 | ||||
6030 | if (!RHS.isNegative()) | ||||
6031 | Pred = ICmpInst::ICMP_SGT; | ||||
6032 | break; | ||||
6033 | } | ||||
6034 | } | ||||
6035 | } | ||||
6036 | |||||
6037 | // Lower this FP comparison into an appropriate integer version of the | ||||
6038 | // comparison. | ||||
6039 | return new ICmpInst(Pred, LHSI->getOperand(0), RHSInt); | ||||
6040 | } | ||||
6041 | |||||
6042 | /// Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary. | ||||
6043 | static Instruction *foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI, | ||||
6044 | Constant *RHSC) { | ||||
6045 | // When C is not 0.0 and infinities are not allowed: | ||||
6046 | // (C / X) < 0.0 is a sign-bit test of X | ||||
6047 | // (C / X) < 0.0 --> X < 0.0 (if C is positive) | ||||
6048 | // (C / X) < 0.0 --> X > 0.0 (if C is negative, swap the predicate) | ||||
6049 | // | ||||
6050 | // Proof: | ||||
6051 | // Multiply (C / X) < 0.0 by X * X / C. | ||||
6052 | // - X is non zero, if it is the flag 'ninf' is violated. | ||||
6053 | // - C defines the sign of X * X * C. Thus it also defines whether to swap | ||||
6054 | // the predicate. C is also non zero by definition. | ||||
6055 | // | ||||
6056 | // Thus X * X / C is non zero and the transformation is valid. [qed] | ||||
6057 | |||||
6058 | FCmpInst::Predicate Pred = I.getPredicate(); | ||||
6059 | |||||
6060 | // Check that predicates are valid. | ||||
6061 | if ((Pred != FCmpInst::FCMP_OGT) && (Pred != FCmpInst::FCMP_OLT) && | ||||
6062 | (Pred != FCmpInst::FCMP_OGE) && (Pred != FCmpInst::FCMP_OLE)) | ||||
6063 | return nullptr; | ||||
6064 | |||||
6065 | // Check that RHS operand is zero. | ||||
6066 | if (!match(RHSC, m_AnyZeroFP())) | ||||
6067 | return nullptr; | ||||
6068 | |||||
6069 | // Check fastmath flags ('ninf'). | ||||
6070 | if (!LHSI->hasNoInfs() || !I.hasNoInfs()) | ||||
6071 | return nullptr; | ||||
6072 | |||||
6073 | // Check the properties of the dividend. It must not be zero to avoid a | ||||
6074 | // division by zero (see Proof). | ||||
6075 | const APFloat *C; | ||||
6076 | if (!match(LHSI->getOperand(0), m_APFloat(C))) | ||||
6077 | return nullptr; | ||||
6078 | |||||
6079 | if (C->isZero()) | ||||
6080 | return nullptr; | ||||
6081 | |||||
6082 | // Get swapped predicate if necessary. | ||||
6083 | if (C->isNegative()) | ||||
6084 | Pred = I.getSwappedPredicate(); | ||||
6085 | |||||
6086 | return new FCmpInst(Pred, LHSI->getOperand(1), RHSC, "", &I); | ||||
6087 | } | ||||
6088 | |||||
6089 | /// Optimize fabs(X) compared with zero. | ||||
6090 | static Instruction *foldFabsWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC) { | ||||
6091 | Value *X; | ||||
6092 | if (!match(I.getOperand(0), m_Intrinsic<Intrinsic::fabs>(m_Value(X))) || | ||||
6093 | !match(I.getOperand(1), m_PosZeroFP())) | ||||
6094 | return nullptr; | ||||
6095 | |||||
6096 | auto replacePredAndOp0 = [&IC](FCmpInst *I, FCmpInst::Predicate P, Value *X) { | ||||
6097 | I->setPredicate(P); | ||||
6098 | return IC.replaceOperand(*I, 0, X); | ||||
6099 | }; | ||||
6100 | |||||
6101 | switch (I.getPredicate()) { | ||||
6102 | case FCmpInst::FCMP_UGE: | ||||
6103 | case FCmpInst::FCMP_OLT: | ||||
6104 | // fabs(X) >= 0.0 --> true | ||||
6105 | // fabs(X) < 0.0 --> false | ||||
6106 | llvm_unreachable("fcmp should have simplified")::llvm::llvm_unreachable_internal("fcmp should have simplified" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 6106); | ||||
6107 | |||||
6108 | case FCmpInst::FCMP_OGT: | ||||
6109 | // fabs(X) > 0.0 --> X != 0.0 | ||||
6110 | return replacePredAndOp0(&I, FCmpInst::FCMP_ONE, X); | ||||
6111 | |||||
6112 | case FCmpInst::FCMP_UGT: | ||||
6113 | // fabs(X) u> 0.0 --> X u!= 0.0 | ||||
6114 | return replacePredAndOp0(&I, FCmpInst::FCMP_UNE, X); | ||||
6115 | |||||
6116 | case FCmpInst::FCMP_OLE: | ||||
6117 | // fabs(X) <= 0.0 --> X == 0.0 | ||||
6118 | return replacePredAndOp0(&I, FCmpInst::FCMP_OEQ, X); | ||||
6119 | |||||
6120 | case FCmpInst::FCMP_ULE: | ||||
6121 | // fabs(X) u<= 0.0 --> X u== 0.0 | ||||
6122 | return replacePredAndOp0(&I, FCmpInst::FCMP_UEQ, X); | ||||
6123 | |||||
6124 | case FCmpInst::FCMP_OGE: | ||||
6125 | // fabs(X) >= 0.0 --> !isnan(X) | ||||
6126 | assert(!I.hasNoNaNs() && "fcmp should have simplified")((!I.hasNoNaNs() && "fcmp should have simplified") ? static_cast <void> (0) : __assert_fail ("!I.hasNoNaNs() && \"fcmp should have simplified\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 6126, __PRETTY_FUNCTION__)); | ||||
6127 | return replacePredAndOp0(&I, FCmpInst::FCMP_ORD, X); | ||||
6128 | |||||
6129 | case FCmpInst::FCMP_ULT: | ||||
6130 | // fabs(X) u< 0.0 --> isnan(X) | ||||
6131 | assert(!I.hasNoNaNs() && "fcmp should have simplified")((!I.hasNoNaNs() && "fcmp should have simplified") ? static_cast <void> (0) : __assert_fail ("!I.hasNoNaNs() && \"fcmp should have simplified\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 6131, __PRETTY_FUNCTION__)); | ||||
6132 | return replacePredAndOp0(&I, FCmpInst::FCMP_UNO, X); | ||||
6133 | |||||
6134 | case FCmpInst::FCMP_OEQ: | ||||
6135 | case FCmpInst::FCMP_UEQ: | ||||
6136 | case FCmpInst::FCMP_ONE: | ||||
6137 | case FCmpInst::FCMP_UNE: | ||||
6138 | case FCmpInst::FCMP_ORD: | ||||
6139 | case FCmpInst::FCMP_UNO: | ||||
6140 | // Look through the fabs() because it doesn't change anything but the sign. | ||||
6141 | // fabs(X) == 0.0 --> X == 0.0, | ||||
6142 | // fabs(X) != 0.0 --> X != 0.0 | ||||
6143 | // isnan(fabs(X)) --> isnan(X) | ||||
6144 | // !isnan(fabs(X) --> !isnan(X) | ||||
6145 | return replacePredAndOp0(&I, I.getPredicate(), X); | ||||
6146 | |||||
6147 | default: | ||||
6148 | return nullptr; | ||||
6149 | } | ||||
6150 | } | ||||
6151 | |||||
6152 | Instruction *InstCombinerImpl::visitFCmpInst(FCmpInst &I) { | ||||
6153 | bool Changed = false; | ||||
6154 | |||||
6155 | /// Orders the operands of the compare so that they are listed from most | ||||
6156 | /// complex to least complex. This puts constants before unary operators, | ||||
6157 | /// before binary operators. | ||||
6158 | if (getComplexity(I.getOperand(0)) < getComplexity(I.getOperand(1))) { | ||||
6159 | I.swapOperands(); | ||||
6160 | Changed = true; | ||||
6161 | } | ||||
6162 | |||||
6163 | const CmpInst::Predicate Pred = I.getPredicate(); | ||||
6164 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||
6165 | if (Value *V = SimplifyFCmpInst(Pred, Op0, Op1, I.getFastMathFlags(), | ||||
6166 | SQ.getWithInstruction(&I))) | ||||
6167 | return replaceInstUsesWith(I, V); | ||||
6168 | |||||
6169 | // Simplify 'fcmp pred X, X' | ||||
6170 | Type *OpType = Op0->getType(); | ||||
6171 | assert(OpType == Op1->getType() && "fcmp with different-typed operands?")((OpType == Op1->getType() && "fcmp with different-typed operands?" ) ? static_cast<void> (0) : __assert_fail ("OpType == Op1->getType() && \"fcmp with different-typed operands?\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 6171, __PRETTY_FUNCTION__)); | ||||
6172 | if (Op0 == Op1) { | ||||
6173 | switch (Pred) { | ||||
6174 | default: break; | ||||
6175 | case FCmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y) | ||||
6176 | case FCmpInst::FCMP_ULT: // True if unordered or less than | ||||
6177 | case FCmpInst::FCMP_UGT: // True if unordered or greater than | ||||
6178 | case FCmpInst::FCMP_UNE: // True if unordered or not equal | ||||
6179 | // Canonicalize these to be 'fcmp uno %X, 0.0'. | ||||
6180 | I.setPredicate(FCmpInst::FCMP_UNO); | ||||
6181 | I.setOperand(1, Constant::getNullValue(OpType)); | ||||
6182 | return &I; | ||||
6183 | |||||
6184 | case FCmpInst::FCMP_ORD: // True if ordered (no nans) | ||||
6185 | case FCmpInst::FCMP_OEQ: // True if ordered and equal | ||||
6186 | case FCmpInst::FCMP_OGE: // True if ordered and greater than or equal | ||||
6187 | case FCmpInst::FCMP_OLE: // True if ordered and less than or equal | ||||
6188 | // Canonicalize these to be 'fcmp ord %X, 0.0'. | ||||
6189 | I.setPredicate(FCmpInst::FCMP_ORD); | ||||
6190 | I.setOperand(1, Constant::getNullValue(OpType)); | ||||
6191 | return &I; | ||||
6192 | } | ||||
6193 | } | ||||
6194 | |||||
6195 | // If we're just checking for a NaN (ORD/UNO) and have a non-NaN operand, | ||||
6196 | // then canonicalize the operand to 0.0. | ||||
6197 | if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) { | ||||
6198 | if (!match(Op0, m_PosZeroFP()) && isKnownNeverNaN(Op0, &TLI)) | ||||
6199 | return replaceOperand(I, 0, ConstantFP::getNullValue(OpType)); | ||||
6200 | |||||
6201 | if (!match(Op1, m_PosZeroFP()) && isKnownNeverNaN(Op1, &TLI)) | ||||
6202 | return replaceOperand(I, 1, ConstantFP::getNullValue(OpType)); | ||||
6203 | } | ||||
6204 | |||||
6205 | // fcmp pred (fneg X), (fneg Y) -> fcmp swap(pred) X, Y | ||||
6206 | Value *X, *Y; | ||||
6207 | if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_FNeg(m_Value(Y)))) | ||||
6208 | return new FCmpInst(I.getSwappedPredicate(), X, Y, "", &I); | ||||
6209 | |||||
6210 | // Test if the FCmpInst instruction is used exclusively by a select as | ||||
6211 | // part of a minimum or maximum operation. If so, refrain from doing | ||||
6212 | // any other folding. This helps out other analyses which understand | ||||
6213 | // non-obfuscated minimum and maximum idioms, such as ScalarEvolution | ||||
6214 | // and CodeGen. And in this case, at least one of the comparison | ||||
6215 | // operands has at least one user besides the compare (the select), | ||||
6216 | // which would often largely negate the benefit of folding anyway. | ||||
6217 | if (I.hasOneUse()) | ||||
6218 | if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) { | ||||
6219 | Value *A, *B; | ||||
6220 | SelectPatternResult SPR = matchSelectPattern(SI, A, B); | ||||
6221 | if (SPR.Flavor != SPF_UNKNOWN) | ||||
6222 | return nullptr; | ||||
6223 | } | ||||
6224 | |||||
6225 | // The sign of 0.0 is ignored by fcmp, so canonicalize to +0.0: | ||||
6226 | // fcmp Pred X, -0.0 --> fcmp Pred X, 0.0 | ||||
6227 | if (match(Op1, m_AnyZeroFP()) && !match(Op1, m_PosZeroFP())) | ||||
6228 | return replaceOperand(I, 1, ConstantFP::getNullValue(OpType)); | ||||
6229 | |||||
6230 | // Handle fcmp with instruction LHS and constant RHS. | ||||
6231 | Instruction *LHSI; | ||||
6232 | Constant *RHSC; | ||||
6233 | if (match(Op0, m_Instruction(LHSI)) && match(Op1, m_Constant(RHSC))) { | ||||
6234 | switch (LHSI->getOpcode()) { | ||||
6235 | case Instruction::PHI: | ||||
6236 | // Only fold fcmp into the PHI if the phi and fcmp are in the same | ||||
6237 | // block. If in the same block, we're encouraging jump threading. If | ||||
6238 | // not, we are just pessimizing the code by making an i1 phi. | ||||
6239 | if (LHSI->getParent() == I.getParent()) | ||||
6240 | if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI))) | ||||
6241 | return NV; | ||||
6242 | break; | ||||
6243 | case Instruction::SIToFP: | ||||
6244 | case Instruction::UIToFP: | ||||
6245 | if (Instruction *NV = foldFCmpIntToFPConst(I, LHSI, RHSC)) | ||||
6246 | return NV; | ||||
6247 | break; | ||||
6248 | case Instruction::FDiv: | ||||
6249 | if (Instruction *NV = foldFCmpReciprocalAndZero(I, LHSI, RHSC)) | ||||
6250 | return NV; | ||||
6251 | break; | ||||
6252 | case Instruction::Load: | ||||
6253 | if (auto *GEP = dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) | ||||
6254 | if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) | ||||
6255 | if (GV->isConstant() && GV->hasDefinitiveInitializer() && | ||||
6256 | !cast<LoadInst>(LHSI)->isVolatile()) | ||||
6257 | if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, I)) | ||||
6258 | return Res; | ||||
6259 | break; | ||||
6260 | } | ||||
6261 | } | ||||
6262 | |||||
6263 | if (Instruction *R = foldFabsWithFcmpZero(I, *this)) | ||||
6264 | return R; | ||||
6265 | |||||
6266 | if (match(Op0, m_FNeg(m_Value(X)))) { | ||||
6267 | // fcmp pred (fneg X), C --> fcmp swap(pred) X, -C | ||||
6268 | Constant *C; | ||||
6269 | if (match(Op1, m_Constant(C))) { | ||||
6270 | Constant *NegC = ConstantExpr::getFNeg(C); | ||||
6271 | return new FCmpInst(I.getSwappedPredicate(), X, NegC, "", &I); | ||||
6272 | } | ||||
6273 | } | ||||
6274 | |||||
6275 | if (match(Op0, m_FPExt(m_Value(X)))) { | ||||
6276 | // fcmp (fpext X), (fpext Y) -> fcmp X, Y | ||||
6277 | if (match(Op1, m_FPExt(m_Value(Y))) && X->getType() == Y->getType()) | ||||
6278 | return new FCmpInst(Pred, X, Y, "", &I); | ||||
6279 | |||||
6280 | // fcmp (fpext X), C -> fcmp X, (fptrunc C) if fptrunc is lossless | ||||
6281 | const APFloat *C; | ||||
6282 | if (match(Op1, m_APFloat(C))) { | ||||
6283 | const fltSemantics &FPSem = | ||||
6284 | X->getType()->getScalarType()->getFltSemantics(); | ||||
6285 | bool Lossy; | ||||
6286 | APFloat TruncC = *C; | ||||
6287 | TruncC.convert(FPSem, APFloat::rmNearestTiesToEven, &Lossy); | ||||
6288 | |||||
6289 | // Avoid lossy conversions and denormals. | ||||
6290 | // Zero is a special case that's OK to convert. | ||||
6291 | APFloat Fabs = TruncC; | ||||
6292 | Fabs.clearSign(); | ||||
6293 | if (!Lossy && | ||||
6294 | (!(Fabs < APFloat::getSmallestNormalized(FPSem)) || Fabs.isZero())) { | ||||
6295 | Constant *NewC = ConstantFP::get(X->getType(), TruncC); | ||||
6296 | return new FCmpInst(Pred, X, NewC, "", &I); | ||||
6297 | } | ||||
6298 | } | ||||
6299 | } | ||||
6300 | |||||
6301 | if (I.getType()->isVectorTy()) | ||||
6302 | if (Instruction *Res = foldVectorCmp(I, Builder)) | ||||
6303 | return Res; | ||||
6304 | |||||
6305 | return Changed ? &I : nullptr; | ||||
6306 | } |
1 | //===- llvm/ADT/SmallVector.h - 'Normally small' vectors --------*- C++ -*-===// | ||||
2 | // | ||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||
6 | // | ||||
7 | //===----------------------------------------------------------------------===// | ||||
8 | // | ||||
9 | // This file defines the SmallVector class. | ||||
10 | // | ||||
11 | //===----------------------------------------------------------------------===// | ||||
12 | |||||
13 | #ifndef LLVM_ADT_SMALLVECTOR_H | ||||
14 | #define LLVM_ADT_SMALLVECTOR_H | ||||
15 | |||||
16 | #include "llvm/ADT/iterator_range.h" | ||||
17 | #include "llvm/Support/AlignOf.h" | ||||
18 | #include "llvm/Support/Compiler.h" | ||||
19 | #include "llvm/Support/ErrorHandling.h" | ||||
20 | #include "llvm/Support/MathExtras.h" | ||||
21 | #include "llvm/Support/MemAlloc.h" | ||||
22 | #include "llvm/Support/type_traits.h" | ||||
23 | #include <algorithm> | ||||
24 | #include <cassert> | ||||
25 | #include <cstddef> | ||||
26 | #include <cstdlib> | ||||
27 | #include <cstring> | ||||
28 | #include <initializer_list> | ||||
29 | #include <iterator> | ||||
30 | #include <limits> | ||||
31 | #include <memory> | ||||
32 | #include <new> | ||||
33 | #include <type_traits> | ||||
34 | #include <utility> | ||||
35 | |||||
36 | namespace llvm { | ||||
37 | |||||
38 | /// This is all the stuff common to all SmallVectors. | ||||
39 | /// | ||||
40 | /// The template parameter specifies the type which should be used to hold the | ||||
41 | /// Size and Capacity of the SmallVector, so it can be adjusted. | ||||
42 | /// Using 32 bit size is desirable to shrink the size of the SmallVector. | ||||
43 | /// Using 64 bit size is desirable for cases like SmallVector<char>, where a | ||||
44 | /// 32 bit size would limit the vector to ~4GB. SmallVectors are used for | ||||
45 | /// buffering bitcode output - which can exceed 4GB. | ||||
46 | template <class Size_T> class SmallVectorBase { | ||||
47 | protected: | ||||
48 | void *BeginX; | ||||
49 | Size_T Size = 0, Capacity; | ||||
50 | |||||
51 | /// The maximum value of the Size_T used. | ||||
52 | static constexpr size_t SizeTypeMax() { | ||||
53 | return std::numeric_limits<Size_T>::max(); | ||||
54 | } | ||||
55 | |||||
56 | SmallVectorBase() = delete; | ||||
57 | SmallVectorBase(void *FirstEl, size_t TotalCapacity) | ||||
58 | : BeginX(FirstEl), Capacity(TotalCapacity) {} | ||||
59 | |||||
60 | /// This is an implementation of the grow() method which only works | ||||
61 | /// on POD-like data types and is out of line to reduce code duplication. | ||||
62 | /// This function will report a fatal error if it cannot increase capacity. | ||||
63 | void grow_pod(void *FirstEl, size_t MinSize, size_t TSize); | ||||
64 | |||||
65 | /// Report that MinSize doesn't fit into this vector's size type. Throws | ||||
66 | /// std::length_error or calls report_fatal_error. | ||||
67 | LLVM_ATTRIBUTE_NORETURN__attribute__((noreturn)) static void report_size_overflow(size_t MinSize); | ||||
68 | /// Report that this vector is already at maximum capacity. Throws | ||||
69 | /// std::length_error or calls report_fatal_error. | ||||
70 | LLVM_ATTRIBUTE_NORETURN__attribute__((noreturn)) static void report_at_maximum_capacity(); | ||||
71 | |||||
72 | public: | ||||
73 | size_t size() const { return Size; } | ||||
74 | size_t capacity() const { return Capacity; } | ||||
75 | |||||
76 | LLVM_NODISCARD[[clang::warn_unused_result]] bool empty() const { return !Size
| ||||
77 | |||||
78 | /// Set the array size to \p N, which the current array must have enough | ||||
79 | /// capacity for. | ||||
80 | /// | ||||
81 | /// This does not construct or destroy any elements in the vector. | ||||
82 | /// | ||||
83 | /// Clients can use this in conjunction with capacity() to write past the end | ||||
84 | /// of the buffer when they know that more elements are available, and only | ||||
85 | /// update the size later. This avoids the cost of value initializing elements | ||||
86 | /// which will only be overwritten. | ||||
87 | void set_size(size_t N) { | ||||
88 | assert(N <= capacity())((N <= capacity()) ? static_cast<void> (0) : __assert_fail ("N <= capacity()", "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 88, __PRETTY_FUNCTION__)); | ||||
89 | Size = N; | ||||
90 | } | ||||
91 | }; | ||||
92 | |||||
93 | template <class T> | ||||
94 | using SmallVectorSizeType = | ||||
95 | typename std::conditional<sizeof(T) < 4 && sizeof(void *) >= 8, uint64_t, | ||||
96 | uint32_t>::type; | ||||
97 | |||||
98 | /// Figure out the offset of the first element. | ||||
99 | template <class T, typename = void> struct SmallVectorAlignmentAndSize { | ||||
100 | AlignedCharArrayUnion<SmallVectorBase<SmallVectorSizeType<T>>> Base; | ||||
101 | AlignedCharArrayUnion<T> FirstEl; | ||||
102 | }; | ||||
103 | |||||
104 | /// This is the part of SmallVectorTemplateBase which does not depend on whether | ||||
105 | /// the type T is a POD. The extra dummy template argument is used by ArrayRef | ||||
106 | /// to avoid unnecessarily requiring T to be complete. | ||||
107 | template <typename T, typename = void> | ||||
108 | class SmallVectorTemplateCommon | ||||
109 | : public SmallVectorBase<SmallVectorSizeType<T>> { | ||||
110 | using Base = SmallVectorBase<SmallVectorSizeType<T>>; | ||||
111 | |||||
112 | /// Find the address of the first element. For this pointer math to be valid | ||||
113 | /// with small-size of 0 for T with lots of alignment, it's important that | ||||
114 | /// SmallVectorStorage is properly-aligned even for small-size of 0. | ||||
115 | void *getFirstEl() const { | ||||
116 | return const_cast<void *>(reinterpret_cast<const void *>( | ||||
117 | reinterpret_cast<const char *>(this) + | ||||
118 | offsetof(SmallVectorAlignmentAndSize<T>, FirstEl)__builtin_offsetof(SmallVectorAlignmentAndSize<T>, FirstEl ))); | ||||
119 | } | ||||
120 | // Space after 'FirstEl' is clobbered, do not add any instance vars after it. | ||||
121 | |||||
122 | protected: | ||||
123 | SmallVectorTemplateCommon(size_t Size) : Base(getFirstEl(), Size) {} | ||||
124 | |||||
125 | void grow_pod(size_t MinSize, size_t TSize) { | ||||
126 | Base::grow_pod(getFirstEl(), MinSize, TSize); | ||||
127 | } | ||||
128 | |||||
129 | /// Return true if this is a smallvector which has not had dynamic | ||||
130 | /// memory allocated for it. | ||||
131 | bool isSmall() const { return this->BeginX == getFirstEl(); } | ||||
132 | |||||
133 | /// Put this vector in a state of being small. | ||||
134 | void resetToSmall() { | ||||
135 | this->BeginX = getFirstEl(); | ||||
136 | this->Size = this->Capacity = 0; // FIXME: Setting Capacity to 0 is suspect. | ||||
137 | } | ||||
138 | |||||
139 | public: | ||||
140 | using size_type = size_t; | ||||
141 | using difference_type = ptrdiff_t; | ||||
142 | using value_type = T; | ||||
143 | using iterator = T *; | ||||
144 | using const_iterator = const T *; | ||||
145 | |||||
146 | using const_reverse_iterator = std::reverse_iterator<const_iterator>; | ||||
147 | using reverse_iterator = std::reverse_iterator<iterator>; | ||||
148 | |||||
149 | using reference = T &; | ||||
150 | using const_reference = const T &; | ||||
151 | using pointer = T *; | ||||
152 | using const_pointer = const T *; | ||||
153 | |||||
154 | using Base::capacity; | ||||
155 | using Base::empty; | ||||
156 | using Base::size; | ||||
157 | |||||
158 | // forward iterator creation methods. | ||||
159 | iterator begin() { return (iterator)this->BeginX; } | ||||
160 | const_iterator begin() const { return (const_iterator)this->BeginX; } | ||||
161 | iterator end() { return begin() + size(); } | ||||
162 | const_iterator end() const { return begin() + size(); } | ||||
163 | |||||
164 | // reverse iterator creation methods. | ||||
165 | reverse_iterator rbegin() { return reverse_iterator(end()); } | ||||
166 | const_reverse_iterator rbegin() const{ return const_reverse_iterator(end()); } | ||||
167 | reverse_iterator rend() { return reverse_iterator(begin()); } | ||||
168 | const_reverse_iterator rend() const { return const_reverse_iterator(begin());} | ||||
169 | |||||
170 | size_type size_in_bytes() const { return size() * sizeof(T); } | ||||
171 | size_type max_size() const { | ||||
172 | return std::min(this->SizeTypeMax(), size_type(-1) / sizeof(T)); | ||||
173 | } | ||||
174 | |||||
175 | size_t capacity_in_bytes() const { return capacity() * sizeof(T); } | ||||
176 | |||||
177 | /// Return a pointer to the vector's buffer, even if empty(). | ||||
178 | pointer data() { return pointer(begin()); } | ||||
179 | /// Return a pointer to the vector's buffer, even if empty(). | ||||
180 | const_pointer data() const { return const_pointer(begin()); } | ||||
181 | |||||
182 | reference operator[](size_type idx) { | ||||
183 | assert(idx < size())((idx < size()) ? static_cast<void> (0) : __assert_fail ("idx < size()", "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 183, __PRETTY_FUNCTION__)); | ||||
184 | return begin()[idx]; | ||||
185 | } | ||||
186 | const_reference operator[](size_type idx) const { | ||||
187 | assert(idx < size())((idx < size()) ? static_cast<void> (0) : __assert_fail ("idx < size()", "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 187, __PRETTY_FUNCTION__)); | ||||
188 | return begin()[idx]; | ||||
189 | } | ||||
190 | |||||
191 | reference front() { | ||||
192 | assert(!empty())((!empty()) ? static_cast<void> (0) : __assert_fail ("!empty()" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 192, __PRETTY_FUNCTION__)); | ||||
193 | return begin()[0]; | ||||
194 | } | ||||
195 | const_reference front() const { | ||||
196 | assert(!empty())((!empty()) ? static_cast<void> (0) : __assert_fail ("!empty()" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 196, __PRETTY_FUNCTION__)); | ||||
197 | return begin()[0]; | ||||
198 | } | ||||
199 | |||||
200 | reference back() { | ||||
201 | assert(!empty())((!empty()) ? static_cast<void> (0) : __assert_fail ("!empty()" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 201, __PRETTY_FUNCTION__)); | ||||
202 | return end()[-1]; | ||||
203 | } | ||||
204 | const_reference back() const { | ||||
205 | assert(!empty())((!empty()) ? static_cast<void> (0) : __assert_fail ("!empty()" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 205, __PRETTY_FUNCTION__)); | ||||
206 | return end()[-1]; | ||||
207 | } | ||||
208 | }; | ||||
209 | |||||
210 | /// SmallVectorTemplateBase<TriviallyCopyable = false> - This is where we put | ||||
211 | /// method implementations that are designed to work with non-trivial T's. | ||||
212 | /// | ||||
213 | /// We approximate is_trivially_copyable with trivial move/copy construction and | ||||
214 | /// trivial destruction. While the standard doesn't specify that you're allowed | ||||
215 | /// copy these types with memcpy, there is no way for the type to observe this. | ||||
216 | /// This catches the important case of std::pair<POD, POD>, which is not | ||||
217 | /// trivially assignable. | ||||
218 | template <typename T, bool = (is_trivially_copy_constructible<T>::value) && | ||||
219 | (is_trivially_move_constructible<T>::value) && | ||||
220 | std::is_trivially_destructible<T>::value> | ||||
221 | class SmallVectorTemplateBase : public SmallVectorTemplateCommon<T> { | ||||
222 | protected: | ||||
223 | SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon<T>(Size) {} | ||||
224 | |||||
225 | static void destroy_range(T *S, T *E) { | ||||
226 | while (S != E) { | ||||
227 | --E; | ||||
228 | E->~T(); | ||||
229 | } | ||||
230 | } | ||||
231 | |||||
232 | /// Move the range [I, E) into the uninitialized memory starting with "Dest", | ||||
233 | /// constructing elements as needed. | ||||
234 | template<typename It1, typename It2> | ||||
235 | static void uninitialized_move(It1 I, It1 E, It2 Dest) { | ||||
236 | std::uninitialized_copy(std::make_move_iterator(I), | ||||
237 | std::make_move_iterator(E), Dest); | ||||
238 | } | ||||
239 | |||||
240 | /// Copy the range [I, E) onto the uninitialized memory starting with "Dest", | ||||
241 | /// constructing elements as needed. | ||||
242 | template<typename It1, typename It2> | ||||
243 | static void uninitialized_copy(It1 I, It1 E, It2 Dest) { | ||||
244 | std::uninitialized_copy(I, E, Dest); | ||||
245 | } | ||||
246 | |||||
247 | /// Grow the allocated memory (without initializing new elements), doubling | ||||
248 | /// the size of the allocated memory. Guarantees space for at least one more | ||||
249 | /// element, or MinSize more elements if specified. | ||||
250 | void grow(size_t MinSize = 0); | ||||
251 | |||||
252 | public: | ||||
253 | void push_back(const T &Elt) { | ||||
254 | if (LLVM_UNLIKELY(this->size() >= this->capacity())__builtin_expect((bool)(this->size() >= this->capacity ()), false)) | ||||
255 | this->grow(); | ||||
256 | ::new ((void*) this->end()) T(Elt); | ||||
257 | this->set_size(this->size() + 1); | ||||
258 | } | ||||
259 | |||||
260 | void push_back(T &&Elt) { | ||||
261 | if (LLVM_UNLIKELY(this->size() >= this->capacity())__builtin_expect((bool)(this->size() >= this->capacity ()), false)) | ||||
262 | this->grow(); | ||||
263 | ::new ((void*) this->end()) T(::std::move(Elt)); | ||||
264 | this->set_size(this->size() + 1); | ||||
265 | } | ||||
266 | |||||
267 | void pop_back() { | ||||
268 | this->set_size(this->size() - 1); | ||||
269 | this->end()->~T(); | ||||
270 | } | ||||
271 | }; | ||||
272 | |||||
273 | // Define this out-of-line to dissuade the C++ compiler from inlining it. | ||||
274 | template <typename T, bool TriviallyCopyable> | ||||
275 | void SmallVectorTemplateBase<T, TriviallyCopyable>::grow(size_t MinSize) { | ||||
276 | // Ensure we can fit the new capacity. | ||||
277 | // This is only going to be applicable when the capacity is 32 bit. | ||||
278 | if (MinSize > this->SizeTypeMax()) | ||||
279 | this->report_size_overflow(MinSize); | ||||
280 | |||||
281 | // Ensure we can meet the guarantee of space for at least one more element. | ||||
282 | // The above check alone will not catch the case where grow is called with a | ||||
283 | // default MinSize of 0, but the current capacity cannot be increased. | ||||
284 | // This is only going to be applicable when the capacity is 32 bit. | ||||
285 | if (this->capacity() == this->SizeTypeMax()) | ||||
286 | this->report_at_maximum_capacity(); | ||||
287 | |||||
288 | // Always grow, even from zero. | ||||
289 | size_t NewCapacity = size_t(NextPowerOf2(this->capacity() + 2)); | ||||
290 | NewCapacity = std::min(std::max(NewCapacity, MinSize), this->SizeTypeMax()); | ||||
291 | T *NewElts = static_cast<T*>(llvm::safe_malloc(NewCapacity*sizeof(T))); | ||||
292 | |||||
293 | // Move the elements over. | ||||
294 | this->uninitialized_move(this->begin(), this->end(), NewElts); | ||||
295 | |||||
296 | // Destroy the original elements. | ||||
297 | destroy_range(this->begin(), this->end()); | ||||
298 | |||||
299 | // If this wasn't grown from the inline copy, deallocate the old space. | ||||
300 | if (!this->isSmall()) | ||||
301 | free(this->begin()); | ||||
302 | |||||
303 | this->BeginX = NewElts; | ||||
304 | this->Capacity = NewCapacity; | ||||
305 | } | ||||
306 | |||||
307 | /// SmallVectorTemplateBase<TriviallyCopyable = true> - This is where we put | ||||
308 | /// method implementations that are designed to work with trivially copyable | ||||
309 | /// T's. This allows using memcpy in place of copy/move construction and | ||||
310 | /// skipping destruction. | ||||
311 | template <typename T> | ||||
312 | class SmallVectorTemplateBase<T, true> : public SmallVectorTemplateCommon<T> { | ||||
313 | protected: | ||||
314 | SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon<T>(Size) {} | ||||
315 | |||||
316 | // No need to do a destroy loop for POD's. | ||||
317 | static void destroy_range(T *, T *) {} | ||||
318 | |||||
319 | /// Move the range [I, E) onto the uninitialized memory | ||||
320 | /// starting with "Dest", constructing elements into it as needed. | ||||
321 | template<typename It1, typename It2> | ||||
322 | static void uninitialized_move(It1 I, It1 E, It2 Dest) { | ||||
323 | // Just do a copy. | ||||
324 | uninitialized_copy(I, E, Dest); | ||||
325 | } | ||||
326 | |||||
327 | /// Copy the range [I, E) onto the uninitialized memory | ||||
328 | /// starting with "Dest", constructing elements into it as needed. | ||||
329 | template<typename It1, typename It2> | ||||
330 | static void uninitialized_copy(It1 I, It1 E, It2 Dest) { | ||||
331 | // Arbitrary iterator types; just use the basic implementation. | ||||
332 | std::uninitialized_copy(I, E, Dest); | ||||
333 | } | ||||
334 | |||||
335 | /// Copy the range [I, E) onto the uninitialized memory | ||||
336 | /// starting with "Dest", constructing elements into it as needed. | ||||
337 | template <typename T1, typename T2> | ||||
338 | static void uninitialized_copy( | ||||
339 | T1 *I, T1 *E, T2 *Dest, | ||||
340 | std::enable_if_t<std::is_same<typename std::remove_const<T1>::type, | ||||
341 | T2>::value> * = nullptr) { | ||||
342 | // Use memcpy for PODs iterated by pointers (which includes SmallVector | ||||
343 | // iterators): std::uninitialized_copy optimizes to memmove, but we can | ||||
344 | // use memcpy here. Note that I and E are iterators and thus might be | ||||
345 | // invalid for memcpy if they are equal. | ||||
346 | if (I != E) | ||||
347 | memcpy(reinterpret_cast<void *>(Dest), I, (E - I) * sizeof(T)); | ||||
348 | } | ||||
349 | |||||
350 | /// Double the size of the allocated memory, guaranteeing space for at | ||||
351 | /// least one more element or MinSize if specified. | ||||
352 | void grow(size_t MinSize = 0) { this->grow_pod(MinSize, sizeof(T)); } | ||||
353 | |||||
354 | public: | ||||
355 | void push_back(const T &Elt) { | ||||
356 | if (LLVM_UNLIKELY(this->size() >= this->capacity())__builtin_expect((bool)(this->size() >= this->capacity ()), false)) | ||||
357 | this->grow(); | ||||
358 | memcpy(reinterpret_cast<void *>(this->end()), &Elt, sizeof(T)); | ||||
359 | this->set_size(this->size() + 1); | ||||
360 | } | ||||
361 | |||||
362 | void pop_back() { this->set_size(this->size() - 1); } | ||||
363 | }; | ||||
364 | |||||
365 | /// This class consists of common code factored out of the SmallVector class to | ||||
366 | /// reduce code duplication based on the SmallVector 'N' template parameter. | ||||
367 | template <typename T> | ||||
368 | class SmallVectorImpl : public SmallVectorTemplateBase<T> { | ||||
369 | using SuperClass = SmallVectorTemplateBase<T>; | ||||
370 | |||||
371 | public: | ||||
372 | using iterator = typename SuperClass::iterator; | ||||
373 | using const_iterator = typename SuperClass::const_iterator; | ||||
374 | using reference = typename SuperClass::reference; | ||||
375 | using size_type = typename SuperClass::size_type; | ||||
376 | |||||
377 | protected: | ||||
378 | // Default ctor - Initialize to empty. | ||||
379 | explicit SmallVectorImpl(unsigned N) | ||||
380 | : SmallVectorTemplateBase<T>(N) {} | ||||
381 | |||||
382 | public: | ||||
383 | SmallVectorImpl(const SmallVectorImpl &) = delete; | ||||
384 | |||||
385 | ~SmallVectorImpl() { | ||||
386 | // Subclass has already destructed this vector's elements. | ||||
387 | // If this wasn't grown from the inline copy, deallocate the old space. | ||||
388 | if (!this->isSmall()) | ||||
389 | free(this->begin()); | ||||
390 | } | ||||
391 | |||||
392 | void clear() { | ||||
393 | this->destroy_range(this->begin(), this->end()); | ||||
394 | this->Size = 0; | ||||
395 | } | ||||
396 | |||||
397 | void resize(size_type N) { | ||||
398 | if (N < this->size()) { | ||||
399 | this->destroy_range(this->begin()+N, this->end()); | ||||
400 | this->set_size(N); | ||||
401 | } else if (N > this->size()) { | ||||
402 | if (this->capacity() < N) | ||||
403 | this->grow(N); | ||||
404 | for (auto I = this->end(), E = this->begin() + N; I != E; ++I) | ||||
405 | new (&*I) T(); | ||||
406 | this->set_size(N); | ||||
407 | } | ||||
408 | } | ||||
409 | |||||
410 | void resize(size_type N, const T &NV) { | ||||
411 | if (N < this->size()) { | ||||
412 | this->destroy_range(this->begin()+N, this->end()); | ||||
413 | this->set_size(N); | ||||
414 | } else if (N > this->size()) { | ||||
415 | if (this->capacity() < N) | ||||
416 | this->grow(N); | ||||
417 | std::uninitialized_fill(this->end(), this->begin()+N, NV); | ||||
418 | this->set_size(N); | ||||
419 | } | ||||
420 | } | ||||
421 | |||||
422 | void reserve(size_type N) { | ||||
423 | if (this->capacity() < N) | ||||
424 | this->grow(N); | ||||
425 | } | ||||
426 | |||||
427 | LLVM_NODISCARD[[clang::warn_unused_result]] T pop_back_val() { | ||||
428 | T Result = ::std::move(this->back()); | ||||
429 | this->pop_back(); | ||||
430 | return Result; | ||||
431 | } | ||||
432 | |||||
433 | void swap(SmallVectorImpl &RHS); | ||||
434 | |||||
435 | /// Add the specified range to the end of the SmallVector. | ||||
436 | template <typename in_iter, | ||||
437 | typename = std::enable_if_t<std::is_convertible< | ||||
438 | typename std::iterator_traits<in_iter>::iterator_category, | ||||
439 | std::input_iterator_tag>::value>> | ||||
440 | void append(in_iter in_start, in_iter in_end) { | ||||
441 | size_type NumInputs = std::distance(in_start, in_end); | ||||
442 | if (NumInputs > this->capacity() - this->size()) | ||||
443 | this->grow(this->size()+NumInputs); | ||||
444 | |||||
445 | this->uninitialized_copy(in_start, in_end, this->end()); | ||||
446 | this->set_size(this->size() + NumInputs); | ||||
447 | } | ||||
448 | |||||
449 | /// Append \p NumInputs copies of \p Elt to the end. | ||||
450 | void append(size_type NumInputs, const T &Elt) { | ||||
451 | if (NumInputs > this->capacity() - this->size()) | ||||
452 | this->grow(this->size()+NumInputs); | ||||
453 | |||||
454 | std::uninitialized_fill_n(this->end(), NumInputs, Elt); | ||||
455 | this->set_size(this->size() + NumInputs); | ||||
456 | } | ||||
457 | |||||
458 | void append(std::initializer_list<T> IL) { | ||||
459 | append(IL.begin(), IL.end()); | ||||
460 | } | ||||
461 | |||||
462 | // FIXME: Consider assigning over existing elements, rather than clearing & | ||||
463 | // re-initializing them - for all assign(...) variants. | ||||
464 | |||||
465 | void assign(size_type NumElts, const T &Elt) { | ||||
466 | clear(); | ||||
467 | if (this->capacity() < NumElts) | ||||
468 | this->grow(NumElts); | ||||
469 | this->set_size(NumElts); | ||||
470 | std::uninitialized_fill(this->begin(), this->end(), Elt); | ||||
471 | } | ||||
472 | |||||
473 | template <typename in_iter, | ||||
474 | typename = std::enable_if_t<std::is_convertible< | ||||
475 | typename std::iterator_traits<in_iter>::iterator_category, | ||||
476 | std::input_iterator_tag>::value>> | ||||
477 | void assign(in_iter in_start, in_iter in_end) { | ||||
478 | clear(); | ||||
479 | append(in_start, in_end); | ||||
480 | } | ||||
481 | |||||
482 | void assign(std::initializer_list<T> IL) { | ||||
483 | clear(); | ||||
484 | append(IL); | ||||
485 | } | ||||
486 | |||||
487 | iterator erase(const_iterator CI) { | ||||
488 | // Just cast away constness because this is a non-const member function. | ||||
489 | iterator I = const_cast<iterator>(CI); | ||||
490 | |||||
491 | assert(I >= this->begin() && "Iterator to erase is out of bounds.")((I >= this->begin() && "Iterator to erase is out of bounds." ) ? static_cast<void> (0) : __assert_fail ("I >= this->begin() && \"Iterator to erase is out of bounds.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 491, __PRETTY_FUNCTION__)); | ||||
492 | assert(I < this->end() && "Erasing at past-the-end iterator.")((I < this->end() && "Erasing at past-the-end iterator." ) ? static_cast<void> (0) : __assert_fail ("I < this->end() && \"Erasing at past-the-end iterator.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 492, __PRETTY_FUNCTION__)); | ||||
493 | |||||
494 | iterator N = I; | ||||
495 | // Shift all elts down one. | ||||
496 | std::move(I+1, this->end(), I); | ||||
497 | // Drop the last elt. | ||||
498 | this->pop_back(); | ||||
499 | return(N); | ||||
500 | } | ||||
501 | |||||
502 | iterator erase(const_iterator CS, const_iterator CE) { | ||||
503 | // Just cast away constness because this is a non-const member function. | ||||
504 | iterator S = const_cast<iterator>(CS); | ||||
505 | iterator E = const_cast<iterator>(CE); | ||||
506 | |||||
507 | assert(S >= this->begin() && "Range to erase is out of bounds.")((S >= this->begin() && "Range to erase is out of bounds." ) ? static_cast<void> (0) : __assert_fail ("S >= this->begin() && \"Range to erase is out of bounds.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 507, __PRETTY_FUNCTION__)); | ||||
508 | assert(S <= E && "Trying to erase invalid range.")((S <= E && "Trying to erase invalid range.") ? static_cast <void> (0) : __assert_fail ("S <= E && \"Trying to erase invalid range.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 508, __PRETTY_FUNCTION__)); | ||||
509 | assert(E <= this->end() && "Trying to erase past the end.")((E <= this->end() && "Trying to erase past the end." ) ? static_cast<void> (0) : __assert_fail ("E <= this->end() && \"Trying to erase past the end.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 509, __PRETTY_FUNCTION__)); | ||||
510 | |||||
511 | iterator N = S; | ||||
512 | // Shift all elts down. | ||||
513 | iterator I = std::move(E, this->end(), S); | ||||
514 | // Drop the last elts. | ||||
515 | this->destroy_range(I, this->end()); | ||||
516 | this->set_size(I - this->begin()); | ||||
517 | return(N); | ||||
518 | } | ||||
519 | |||||
520 | iterator insert(iterator I, T &&Elt) { | ||||
521 | if (I == this->end()) { // Important special case for empty vector. | ||||
522 | this->push_back(::std::move(Elt)); | ||||
523 | return this->end()-1; | ||||
524 | } | ||||
525 | |||||
526 | assert(I >= this->begin() && "Insertion iterator is out of bounds.")((I >= this->begin() && "Insertion iterator is out of bounds." ) ? static_cast<void> (0) : __assert_fail ("I >= this->begin() && \"Insertion iterator is out of bounds.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 526, __PRETTY_FUNCTION__)); | ||||
527 | assert(I <= this->end() && "Inserting past the end of the vector.")((I <= this->end() && "Inserting past the end of the vector." ) ? static_cast<void> (0) : __assert_fail ("I <= this->end() && \"Inserting past the end of the vector.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 527, __PRETTY_FUNCTION__)); | ||||
528 | |||||
529 | if (this->size() >= this->capacity()) { | ||||
530 | size_t EltNo = I-this->begin(); | ||||
531 | this->grow(); | ||||
532 | I = this->begin()+EltNo; | ||||
533 | } | ||||
534 | |||||
535 | ::new ((void*) this->end()) T(::std::move(this->back())); | ||||
536 | // Push everything else over. | ||||
537 | std::move_backward(I, this->end()-1, this->end()); | ||||
538 | this->set_size(this->size() + 1); | ||||
539 | |||||
540 | // If we just moved the element we're inserting, be sure to update | ||||
541 | // the reference. | ||||
542 | T *EltPtr = &Elt; | ||||
543 | if (I <= EltPtr && EltPtr < this->end()) | ||||
544 | ++EltPtr; | ||||
545 | |||||
546 | *I = ::std::move(*EltPtr); | ||||
547 | return I; | ||||
548 | } | ||||
549 | |||||
550 | iterator insert(iterator I, const T &Elt) { | ||||
551 | if (I == this->end()) { // Important special case for empty vector. | ||||
552 | this->push_back(Elt); | ||||
553 | return this->end()-1; | ||||
554 | } | ||||
555 | |||||
556 | assert(I >= this->begin() && "Insertion iterator is out of bounds.")((I >= this->begin() && "Insertion iterator is out of bounds." ) ? static_cast<void> (0) : __assert_fail ("I >= this->begin() && \"Insertion iterator is out of bounds.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 556, __PRETTY_FUNCTION__)); | ||||
557 | assert(I <= this->end() && "Inserting past the end of the vector.")((I <= this->end() && "Inserting past the end of the vector." ) ? static_cast<void> (0) : __assert_fail ("I <= this->end() && \"Inserting past the end of the vector.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 557, __PRETTY_FUNCTION__)); | ||||
558 | |||||
559 | if (this->size() >= this->capacity()) { | ||||
560 | size_t EltNo = I-this->begin(); | ||||
561 | this->grow(); | ||||
562 | I = this->begin()+EltNo; | ||||
563 | } | ||||
564 | ::new ((void*) this->end()) T(std::move(this->back())); | ||||
565 | // Push everything else over. | ||||
566 | std::move_backward(I, this->end()-1, this->end()); | ||||
567 | this->set_size(this->size() + 1); | ||||
568 | |||||
569 | // If we just moved the element we're inserting, be sure to update | ||||
570 | // the reference. | ||||
571 | const T *EltPtr = &Elt; | ||||
572 | if (I <= EltPtr && EltPtr < this->end()) | ||||
573 | ++EltPtr; | ||||
574 | |||||
575 | *I = *EltPtr; | ||||
576 | return I; | ||||
577 | } | ||||
578 | |||||
579 | iterator insert(iterator I, size_type NumToInsert, const T &Elt) { | ||||
580 | // Convert iterator to elt# to avoid invalidating iterator when we reserve() | ||||
581 | size_t InsertElt = I - this->begin(); | ||||
582 | |||||
583 | if (I == this->end()) { // Important special case for empty vector. | ||||
584 | append(NumToInsert, Elt); | ||||
585 | return this->begin()+InsertElt; | ||||
586 | } | ||||
587 | |||||
588 | assert(I >= this->begin() && "Insertion iterator is out of bounds.")((I >= this->begin() && "Insertion iterator is out of bounds." ) ? static_cast<void> (0) : __assert_fail ("I >= this->begin() && \"Insertion iterator is out of bounds.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 588, __PRETTY_FUNCTION__)); | ||||
589 | assert(I <= this->end() && "Inserting past the end of the vector.")((I <= this->end() && "Inserting past the end of the vector." ) ? static_cast<void> (0) : __assert_fail ("I <= this->end() && \"Inserting past the end of the vector.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 589, __PRETTY_FUNCTION__)); | ||||
590 | |||||
591 | // Ensure there is enough space. | ||||
592 | reserve(this->size() + NumToInsert); | ||||
593 | |||||
594 | // Uninvalidate the iterator. | ||||
595 | I = this->begin()+InsertElt; | ||||
596 | |||||
597 | // If there are more elements between the insertion point and the end of the | ||||
598 | // range than there are being inserted, we can use a simple approach to | ||||
599 | // insertion. Since we already reserved space, we know that this won't | ||||
600 | // reallocate the vector. | ||||
601 | if (size_t(this->end()-I) >= NumToInsert) { | ||||
602 | T *OldEnd = this->end(); | ||||
603 | append(std::move_iterator<iterator>(this->end() - NumToInsert), | ||||
604 | std::move_iterator<iterator>(this->end())); | ||||
605 | |||||
606 | // Copy the existing elements that get replaced. | ||||
607 | std::move_backward(I, OldEnd-NumToInsert, OldEnd); | ||||
608 | |||||
609 | std::fill_n(I, NumToInsert, Elt); | ||||
610 | return I; | ||||
611 | } | ||||
612 | |||||
613 | // Otherwise, we're inserting more elements than exist already, and we're | ||||
614 | // not inserting at the end. | ||||
615 | |||||
616 | // Move over the elements that we're about to overwrite. | ||||
617 | T *OldEnd = this->end(); | ||||
618 | this->set_size(this->size() + NumToInsert); | ||||
619 | size_t NumOverwritten = OldEnd-I; | ||||
620 | this->uninitialized_move(I, OldEnd, this->end()-NumOverwritten); | ||||
621 | |||||
622 | // Replace the overwritten part. | ||||
623 | std::fill_n(I, NumOverwritten, Elt); | ||||
624 | |||||
625 | // Insert the non-overwritten middle part. | ||||
626 | std::uninitialized_fill_n(OldEnd, NumToInsert-NumOverwritten, Elt); | ||||
627 | return I; | ||||
628 | } | ||||
629 | |||||
630 | template <typename ItTy, | ||||
631 | typename = std::enable_if_t<std::is_convertible< | ||||
632 | typename std::iterator_traits<ItTy>::iterator_category, | ||||
633 | std::input_iterator_tag>::value>> | ||||
634 | iterator insert(iterator I, ItTy From, ItTy To) { | ||||
635 | // Convert iterator to elt# to avoid invalidating iterator when we reserve() | ||||
636 | size_t InsertElt = I - this->begin(); | ||||
637 | |||||
638 | if (I == this->end()) { // Important special case for empty vector. | ||||
639 | append(From, To); | ||||
640 | return this->begin()+InsertElt; | ||||
641 | } | ||||
642 | |||||
643 | assert(I >= this->begin() && "Insertion iterator is out of bounds.")((I >= this->begin() && "Insertion iterator is out of bounds." ) ? static_cast<void> (0) : __assert_fail ("I >= this->begin() && \"Insertion iterator is out of bounds.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 643, __PRETTY_FUNCTION__)); | ||||
644 | assert(I <= this->end() && "Inserting past the end of the vector.")((I <= this->end() && "Inserting past the end of the vector." ) ? static_cast<void> (0) : __assert_fail ("I <= this->end() && \"Inserting past the end of the vector.\"" , "/build/llvm-toolchain-snapshot-12~++20200927111121+5811d723998/llvm/include/llvm/ADT/SmallVector.h" , 644, __PRETTY_FUNCTION__)); | ||||
645 | |||||
646 | size_t NumToInsert = std::distance(From, To); | ||||
647 | |||||
648 | // Ensure there is enough space. | ||||
649 | reserve(this->size() + NumToInsert); | ||||
650 | |||||
651 | // Uninvalidate the iterator. | ||||
652 | I = this->begin()+InsertElt; | ||||
653 | |||||
654 | // If there are more elements between the insertion point and the end of the | ||||
655 | // range than there are being inserted, we can use a simple approach to | ||||
656 | // insertion. Since we already reserved space, we know that this won't | ||||
657 | // reallocate the vector. | ||||
658 | if (size_t(this->end()-I) >= NumToInsert) { | ||||
659 | T *OldEnd = this->end(); | ||||
660 | append(std::move_iterator<iterator>(this->end() - NumToInsert), | ||||
661 | std::move_iterator<iterator>(this->end())); | ||||
662 | |||||
663 | // Copy the existing elements that get replaced. | ||||
664 | std::move_backward(I, OldEnd-NumToInsert, OldEnd); | ||||
665 | |||||
666 | std::copy(From, To, I); | ||||
667 | return I; | ||||
668 | } | ||||
669 | |||||
670 | // Otherwise, we're inserting more elements than exist already, and we're | ||||
671 | // not inserting at the end. | ||||
672 | |||||
673 | // Move over the elements that we're about to overwrite. | ||||
674 | T *OldEnd = this->end(); | ||||
675 | this->set_size(this->size() + NumToInsert); | ||||
676 | size_t NumOverwritten = OldEnd-I; | ||||
677 | this->uninitialized_move(I, OldEnd, this->end()-NumOverwritten); | ||||
678 | |||||
679 | // Replace the overwritten part. | ||||
680 | for (T *J = I; NumOverwritten > 0; --NumOverwritten) { | ||||
681 | *J = *From; | ||||
682 | ++J; ++From; | ||||
683 | } | ||||
684 | |||||
685 | // Insert the non-overwritten middle part. | ||||
686 | this->uninitialized_copy(From, To, OldEnd); | ||||
687 | return I; | ||||
688 | } | ||||
689 | |||||
690 | void insert(iterator I, std::initializer_list<T> IL) { | ||||
691 | insert(I, IL.begin(), IL.end()); | ||||
692 | } | ||||
693 | |||||
694 | template <typename... ArgTypes> reference emplace_back(ArgTypes &&... Args) { | ||||
695 | if (LLVM_UNLIKELY(this->size() >= this->capacity())__builtin_expect((bool)(this->size() >= this->capacity ()), false)) | ||||
696 | this->grow(); | ||||
697 | ::new ((void *)this->end()) T(std::forward<ArgTypes>(Args)...); | ||||
698 | this->set_size(this->size() + 1); | ||||
699 | return this->back(); | ||||
700 | } | ||||
701 | |||||
702 | SmallVectorImpl &operator=(const SmallVectorImpl &RHS); | ||||
703 | |||||
704 | SmallVectorImpl &operator=(SmallVectorImpl &&RHS); | ||||
705 | |||||
706 | bool operator==(const SmallVectorImpl &RHS) const { | ||||
707 | if (this->size() != RHS.size()) return false; | ||||
708 | return std::equal(this->begin(), this->end(), RHS.begin()); | ||||
709 | } | ||||
710 | bool operator!=(const SmallVectorImpl &RHS) const { | ||||
711 | return !(*this == RHS); | ||||
712 | } | ||||
713 | |||||
714 | bool operator<(const SmallVectorImpl &RHS) const { | ||||
715 | return std::lexicographical_compare(this->begin(), this->end(), | ||||
716 | RHS.begin(), RHS.end()); | ||||
717 | } | ||||
718 | }; | ||||
719 | |||||
720 | template <typename T> | ||||
721 | void SmallVectorImpl<T>::swap(SmallVectorImpl<T> &RHS) { | ||||
722 | if (this == &RHS) return; | ||||
723 | |||||
724 | // We can only avoid copying elements if neither vector is small. | ||||
725 | if (!this->isSmall() && !RHS.isSmall()) { | ||||
726 | std::swap(this->BeginX, RHS.BeginX); | ||||
727 | std::swap(this->Size, RHS.Size); | ||||
728 | std::swap(this->Capacity, RHS.Capacity); | ||||
729 | return; | ||||
730 | } | ||||
731 | if (RHS.size() > this->capacity()) | ||||
732 | this->grow(RHS.size()); | ||||
733 | if (this->size() > RHS.capacity()) | ||||
734 | RHS.grow(this->size()); | ||||
735 | |||||
736 | // Swap the shared elements. | ||||
737 | size_t NumShared = this->size(); | ||||
738 | if (NumShared > RHS.size()) NumShared = RHS.size(); | ||||
739 | for (size_type i = 0; i != NumShared; ++i) | ||||
740 | std::swap((*this)[i], RHS[i]); | ||||
741 | |||||
742 | // Copy over the extra elts. | ||||
743 | if (this->size() > RHS.size()) { | ||||
744 | size_t EltDiff = this->size() - RHS.size(); | ||||
745 | this->uninitialized_copy(this->begin()+NumShared, this->end(), RHS.end()); | ||||
746 | RHS.set_size(RHS.size() + EltDiff); | ||||
747 | this->destroy_range(this->begin()+NumShared, this->end()); | ||||
748 | this->set_size(NumShared); | ||||
749 | } else if (RHS.size() > this->size()) { | ||||
750 | size_t EltDiff = RHS.size() - this->size(); | ||||
751 | this->uninitialized_copy(RHS.begin()+NumShared, RHS.end(), this->end()); | ||||
752 | this->set_size(this->size() + EltDiff); | ||||
753 | this->destroy_range(RHS.begin()+NumShared, RHS.end()); | ||||
754 | RHS.set_size(NumShared); | ||||
755 | } | ||||
756 | } | ||||
757 | |||||
758 | template <typename T> | ||||
759 | SmallVectorImpl<T> &SmallVectorImpl<T>:: | ||||
760 | operator=(const SmallVectorImpl<T> &RHS) { | ||||
761 | // Avoid self-assignment. | ||||
762 | if (this == &RHS) return *this; | ||||
763 | |||||
764 | // If we already have sufficient space, assign the common elements, then | ||||
765 | // destroy any excess. | ||||
766 | size_t RHSSize = RHS.size(); | ||||
767 | size_t CurSize = this->size(); | ||||
768 | if (CurSize >= RHSSize) { | ||||
769 | // Assign common elements. | ||||
770 | iterator NewEnd; | ||||
771 | if (RHSSize) | ||||
772 | NewEnd = std::copy(RHS.begin(), RHS.begin()+RHSSize, this->begin()); | ||||
773 | else | ||||
774 | NewEnd = this->begin(); | ||||
775 | |||||
776 | // Destroy excess elements. | ||||
777 | this->destroy_range(NewEnd, this->end()); | ||||
778 | |||||
779 | // Trim. | ||||
780 | this->set_size(RHSSize); | ||||
781 | return *this; | ||||
782 | } | ||||
783 | |||||
784 | // If we have to grow to have enough elements, destroy the current elements. | ||||
785 | // This allows us to avoid copying them during the grow. | ||||
786 | // FIXME: don't do this if they're efficiently moveable. | ||||
787 | if (this->capacity() < RHSSize) { | ||||
788 | // Destroy current elements. | ||||
789 | this->destroy_range(this->begin(), this->end()); | ||||
790 | this->set_size(0); | ||||
791 | CurSize = 0; | ||||
792 | this->grow(RHSSize); | ||||
793 | } else if (CurSize) { | ||||
794 | // Otherwise, use assignment for the already-constructed elements. | ||||
795 | std::copy(RHS.begin(), RHS.begin()+CurSize, this->begin()); | ||||
796 | } | ||||
797 | |||||
798 | // Copy construct the new elements in place. | ||||
799 | this->uninitialized_copy(RHS.begin()+CurSize, RHS.end(), | ||||
800 | this->begin()+CurSize); | ||||
801 | |||||
802 | // Set end. | ||||
803 | this->set_size(RHSSize); | ||||
804 | return *this; | ||||
805 | } | ||||
806 | |||||
807 | template <typename T> | ||||
808 | SmallVectorImpl<T> &SmallVectorImpl<T>::operator=(SmallVectorImpl<T> &&RHS) { | ||||
809 | // Avoid self-assignment. | ||||
810 | if (this == &RHS) return *this; | ||||
811 | |||||
812 | // If the RHS isn't small, clear this vector and then steal its buffer. | ||||
813 | if (!RHS.isSmall()) { | ||||
814 | this->destroy_range(this->begin(), this->end()); | ||||
815 | if (!this->isSmall()) free(this->begin()); | ||||
816 | this->BeginX = RHS.BeginX; | ||||
817 | this->Size = RHS.Size; | ||||
818 | this->Capacity = RHS.Capacity; | ||||
819 | RHS.resetToSmall(); | ||||
820 | return *this; | ||||
821 | } | ||||
822 | |||||
823 | // If we already have sufficient space, assign the common elements, then | ||||
824 | // destroy any excess. | ||||
825 | size_t RHSSize = RHS.size(); | ||||
826 | size_t CurSize = this->size(); | ||||
827 | if (CurSize >= RHSSize) { | ||||
828 | // Assign common elements. | ||||
829 | iterator NewEnd = this->begin(); | ||||
830 | if (RHSSize) | ||||
831 | NewEnd = std::move(RHS.begin(), RHS.end(), NewEnd); | ||||
832 | |||||
833 | // Destroy excess elements and trim the bounds. | ||||
834 | this->destroy_range(NewEnd, this->end()); | ||||
835 | this->set_size(RHSSize); | ||||
836 | |||||
837 | // Clear the RHS. | ||||
838 | RHS.clear(); | ||||
839 | |||||
840 | return *this; | ||||
841 | } | ||||
842 | |||||
843 | // If we have to grow to have enough elements, destroy the current elements. | ||||
844 | // This allows us to avoid copying them during the grow. | ||||
845 | // FIXME: this may not actually make any sense if we can efficiently move | ||||
846 | // elements. | ||||
847 | if (this->capacity() < RHSSize) { | ||||
848 | // Destroy current elements. | ||||
849 | this->destroy_range(this->begin(), this->end()); | ||||
850 | this->set_size(0); | ||||
851 | CurSize = 0; | ||||
852 | this->grow(RHSSize); | ||||
853 | } else if (CurSize) { | ||||
854 | // Otherwise, use assignment for the already-constructed elements. | ||||
855 | std::move(RHS.begin(), RHS.begin()+CurSize, this->begin()); | ||||
856 | } | ||||
857 | |||||
858 | // Move-construct the new elements in place. | ||||
859 | this->uninitialized_move(RHS.begin()+CurSize, RHS.end(), | ||||
860 | this->begin()+CurSize); | ||||
861 | |||||
862 | // Set end. | ||||
863 | this->set_size(RHSSize); | ||||
864 | |||||
865 | RHS.clear(); | ||||
866 | return *this; | ||||
867 | } | ||||
868 | |||||
869 | /// Storage for the SmallVector elements. This is specialized for the N=0 case | ||||
870 | /// to avoid allocating unnecessary storage. | ||||
871 | template <typename T, unsigned N> | ||||
872 | struct SmallVectorStorage { | ||||
873 | AlignedCharArrayUnion<T> InlineElts[N]; | ||||
874 | }; | ||||
875 | |||||
876 | /// We need the storage to be properly aligned even for small-size of 0 so that | ||||
877 | /// the pointer math in \a SmallVectorTemplateCommon::getFirstEl() is | ||||
878 | /// well-defined. | ||||
879 | template <typename T> struct alignas(alignof(T)) SmallVectorStorage<T, 0> {}; | ||||
880 | |||||
881 | /// This is a 'vector' (really, a variable-sized array), optimized | ||||
882 | /// for the case when the array is small. It contains some number of elements | ||||
883 | /// in-place, which allows it to avoid heap allocation when the actual number of | ||||
884 | /// elements is below that threshold. This allows normal "small" cases to be | ||||
885 | /// fast without losing generality for large inputs. | ||||
886 | /// | ||||
887 | /// Note that this does not attempt to be exception safe. | ||||
888 | /// | ||||
889 | template <typename T, unsigned N> | ||||
890 | class LLVM_GSL_OWNER[[gsl::Owner]] SmallVector : public SmallVectorImpl<T>, | ||||
891 | SmallVectorStorage<T, N> { | ||||
892 | public: | ||||
893 | SmallVector() : SmallVectorImpl<T>(N) {} | ||||
894 | |||||
895 | ~SmallVector() { | ||||
896 | // Destroy the constructed elements in the vector. | ||||
897 | this->destroy_range(this->begin(), this->end()); | ||||
898 | } | ||||
899 | |||||
900 | explicit SmallVector(size_t Size, const T &Value = T()) | ||||
901 | : SmallVectorImpl<T>(N) { | ||||
902 | this->assign(Size, Value); | ||||
903 | } | ||||
904 | |||||
905 | template <typename ItTy, | ||||
906 | typename = std::enable_if_t<std::is_convertible< | ||||
907 | typename std::iterator_traits<ItTy>::iterator_category, | ||||
908 | std::input_iterator_tag>::value>> | ||||
909 | SmallVector(ItTy S, ItTy E) : SmallVectorImpl<T>(N) { | ||||
910 | this->append(S, E); | ||||
911 | } | ||||
912 | |||||
913 | template <typename RangeTy> | ||||
914 | explicit SmallVector(const iterator_range<RangeTy> &R) | ||||
915 | : SmallVectorImpl<T>(N) { | ||||
916 | this->append(R.begin(), R.end()); | ||||
917 | } | ||||
918 | |||||
919 | SmallVector(std::initializer_list<T> IL) : SmallVectorImpl<T>(N) { | ||||
920 | this->assign(IL); | ||||
921 | } | ||||
922 | |||||
923 | SmallVector(const SmallVector &RHS) : SmallVectorImpl<T>(N) { | ||||
924 | if (!RHS.empty()) | ||||
925 | SmallVectorImpl<T>::operator=(RHS); | ||||
926 | } | ||||
927 | |||||
928 | const SmallVector &operator=(const SmallVector &RHS) { | ||||
929 | SmallVectorImpl<T>::operator=(RHS); | ||||
930 | return *this; | ||||
931 | } | ||||
932 | |||||
933 | SmallVector(SmallVector &&RHS) : SmallVectorImpl<T>(N) { | ||||
934 | if (!RHS.empty()) | ||||
935 | SmallVectorImpl<T>::operator=(::std::move(RHS)); | ||||
936 | } | ||||
937 | |||||
938 | SmallVector(SmallVectorImpl<T> &&RHS) : SmallVectorImpl<T>(N) { | ||||
939 | if (!RHS.empty()) | ||||
940 | SmallVectorImpl<T>::operator=(::std::move(RHS)); | ||||
941 | } | ||||
942 | |||||
943 | const SmallVector &operator=(SmallVector &&RHS) { | ||||
944 | SmallVectorImpl<T>::operator=(::std::move(RHS)); | ||||
945 | return *this; | ||||
946 | } | ||||
947 | |||||
948 | const SmallVector &operator=(SmallVectorImpl<T> &&RHS) { | ||||
949 | SmallVectorImpl<T>::operator=(::std::move(RHS)); | ||||
950 | return *this; | ||||
951 | } | ||||
952 | |||||
953 | const SmallVector &operator=(std::initializer_list<T> IL) { | ||||
954 | this->assign(IL); | ||||
955 | return *this; | ||||
956 | } | ||||
957 | }; | ||||
958 | |||||
959 | template <typename T, unsigned N> | ||||
960 | inline size_t capacity_in_bytes(const SmallVector<T, N> &X) { | ||||
961 | return X.capacity_in_bytes(); | ||||
962 | } | ||||
963 | |||||
964 | /// Given a range of type R, iterate the entire range and return a | ||||
965 | /// SmallVector with elements of the vector. This is useful, for example, | ||||
966 | /// when you want to iterate a range and then sort the results. | ||||
967 | template <unsigned Size, typename R> | ||||
968 | SmallVector<typename std::remove_const<typename std::remove_reference< | ||||
969 | decltype(*std::begin(std::declval<R &>()))>::type>::type, | ||||
970 | Size> | ||||
971 | to_vector(R &&Range) { | ||||
972 | return {std::begin(Range), std::end(Range)}; | ||||
973 | } | ||||
974 | |||||
975 | } // end namespace llvm | ||||
976 | |||||
977 | namespace std { | ||||
978 | |||||
979 | /// Implement std::swap in terms of SmallVector swap. | ||||
980 | template<typename T> | ||||
981 | inline void | ||||
982 | swap(llvm::SmallVectorImpl<T> &LHS, llvm::SmallVectorImpl<T> &RHS) { | ||||
983 | LHS.swap(RHS); | ||||
984 | } | ||||
985 | |||||
986 | /// Implement std::swap in terms of SmallVector swap. | ||||
987 | template<typename T, unsigned N> | ||||
988 | inline void | ||||
989 | swap(llvm::SmallVector<T, N> &LHS, llvm::SmallVector<T, N> &RHS) { | ||||
990 | LHS.swap(RHS); | ||||
991 | } | ||||
992 | |||||
993 | } // end namespace std | ||||
994 | |||||
995 | #endif // LLVM_ADT_SMALLVECTOR_H |