File: | lib/Transforms/InstCombine/InstCombineCompares.cpp |
Warning: | line 2287, column 35 Called C++ object pointer is null |
1 | //===- InstCombineCompares.cpp --------------------------------------------===// | |||
2 | // | |||
3 | // The LLVM Compiler Infrastructure | |||
4 | // | |||
5 | // This file is distributed under the University of Illinois Open Source | |||
6 | // License. See LICENSE.TXT for details. | |||
7 | // | |||
8 | //===----------------------------------------------------------------------===// | |||
9 | // | |||
10 | // This file implements the visitICmp and visitFCmp functions. | |||
11 | // | |||
12 | //===----------------------------------------------------------------------===// | |||
13 | ||||
14 | #include "InstCombineInternal.h" | |||
15 | #include "llvm/ADT/APSInt.h" | |||
16 | #include "llvm/ADT/SetVector.h" | |||
17 | #include "llvm/ADT/Statistic.h" | |||
18 | #include "llvm/Analysis/ConstantFolding.h" | |||
19 | #include "llvm/Analysis/InstructionSimplify.h" | |||
20 | #include "llvm/Analysis/MemoryBuiltins.h" | |||
21 | #include "llvm/Analysis/TargetLibraryInfo.h" | |||
22 | #include "llvm/Analysis/VectorUtils.h" | |||
23 | #include "llvm/IR/ConstantRange.h" | |||
24 | #include "llvm/IR/DataLayout.h" | |||
25 | #include "llvm/IR/GetElementPtrTypeIterator.h" | |||
26 | #include "llvm/IR/IntrinsicInst.h" | |||
27 | #include "llvm/IR/PatternMatch.h" | |||
28 | #include "llvm/Support/Debug.h" | |||
29 | #include "llvm/Support/KnownBits.h" | |||
30 | ||||
31 | using namespace llvm; | |||
32 | using namespace PatternMatch; | |||
33 | ||||
34 | #define DEBUG_TYPE"instcombine" "instcombine" | |||
35 | ||||
36 | // How many times is a select replaced by one of its operands? | |||
37 | STATISTIC(NumSel, "Number of select opts")static llvm::Statistic NumSel = {"instcombine", "NumSel", "Number of select opts" , {0}, false}; | |||
38 | ||||
39 | ||||
40 | static ConstantInt *extractElement(Constant *V, Constant *Idx) { | |||
41 | return cast<ConstantInt>(ConstantExpr::getExtractElement(V, Idx)); | |||
42 | } | |||
43 | ||||
44 | static bool hasAddOverflow(ConstantInt *Result, | |||
45 | ConstantInt *In1, ConstantInt *In2, | |||
46 | bool IsSigned) { | |||
47 | if (!IsSigned) | |||
48 | return Result->getValue().ult(In1->getValue()); | |||
49 | ||||
50 | if (In2->isNegative()) | |||
51 | return Result->getValue().sgt(In1->getValue()); | |||
52 | return Result->getValue().slt(In1->getValue()); | |||
53 | } | |||
54 | ||||
55 | /// Compute Result = In1+In2, returning true if the result overflowed for this | |||
56 | /// type. | |||
57 | static bool addWithOverflow(Constant *&Result, Constant *In1, | |||
58 | Constant *In2, bool IsSigned = false) { | |||
59 | Result = ConstantExpr::getAdd(In1, In2); | |||
60 | ||||
61 | if (VectorType *VTy = dyn_cast<VectorType>(In1->getType())) { | |||
62 | for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) { | |||
63 | Constant *Idx = ConstantInt::get(Type::getInt32Ty(In1->getContext()), i); | |||
64 | if (hasAddOverflow(extractElement(Result, Idx), | |||
65 | extractElement(In1, Idx), | |||
66 | extractElement(In2, Idx), | |||
67 | IsSigned)) | |||
68 | return true; | |||
69 | } | |||
70 | return false; | |||
71 | } | |||
72 | ||||
73 | return hasAddOverflow(cast<ConstantInt>(Result), | |||
74 | cast<ConstantInt>(In1), cast<ConstantInt>(In2), | |||
75 | IsSigned); | |||
76 | } | |||
77 | ||||
78 | static bool hasSubOverflow(ConstantInt *Result, | |||
79 | ConstantInt *In1, ConstantInt *In2, | |||
80 | bool IsSigned) { | |||
81 | if (!IsSigned) | |||
82 | return Result->getValue().ugt(In1->getValue()); | |||
83 | ||||
84 | if (In2->isNegative()) | |||
85 | return Result->getValue().slt(In1->getValue()); | |||
86 | ||||
87 | return Result->getValue().sgt(In1->getValue()); | |||
88 | } | |||
89 | ||||
90 | /// Compute Result = In1-In2, returning true if the result overflowed for this | |||
91 | /// type. | |||
92 | static bool subWithOverflow(Constant *&Result, Constant *In1, | |||
93 | Constant *In2, bool IsSigned = false) { | |||
94 | Result = ConstantExpr::getSub(In1, In2); | |||
95 | ||||
96 | if (VectorType *VTy = dyn_cast<VectorType>(In1->getType())) { | |||
97 | for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) { | |||
98 | Constant *Idx = ConstantInt::get(Type::getInt32Ty(In1->getContext()), i); | |||
99 | if (hasSubOverflow(extractElement(Result, Idx), | |||
100 | extractElement(In1, Idx), | |||
101 | extractElement(In2, Idx), | |||
102 | IsSigned)) | |||
103 | return true; | |||
104 | } | |||
105 | return false; | |||
106 | } | |||
107 | ||||
108 | return hasSubOverflow(cast<ConstantInt>(Result), | |||
109 | cast<ConstantInt>(In1), cast<ConstantInt>(In2), | |||
110 | IsSigned); | |||
111 | } | |||
112 | ||||
113 | /// Given an icmp instruction, return true if any use of this comparison is a | |||
114 | /// branch on sign bit comparison. | |||
115 | static bool isBranchOnSignBitCheck(ICmpInst &I, bool isSignBit) { | |||
116 | for (auto *U : I.users()) | |||
117 | if (isa<BranchInst>(U)) | |||
118 | return isSignBit; | |||
119 | return false; | |||
120 | } | |||
121 | ||||
122 | /// Given an exploded icmp instruction, return true if the comparison only | |||
123 | /// checks the sign bit. If it only checks the sign bit, set TrueIfSigned if the | |||
124 | /// result of the comparison is true when the input value is signed. | |||
125 | static bool isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS, | |||
126 | bool &TrueIfSigned) { | |||
127 | switch (Pred) { | |||
128 | case ICmpInst::ICMP_SLT: // True if LHS s< 0 | |||
129 | TrueIfSigned = true; | |||
130 | return RHS == 0; | |||
131 | case ICmpInst::ICMP_SLE: // True if LHS s<= RHS and RHS == -1 | |||
132 | TrueIfSigned = true; | |||
133 | return RHS.isAllOnesValue(); | |||
134 | case ICmpInst::ICMP_SGT: // True if LHS s> -1 | |||
135 | TrueIfSigned = false; | |||
136 | return RHS.isAllOnesValue(); | |||
137 | case ICmpInst::ICMP_UGT: | |||
138 | // True if LHS u> RHS and RHS == high-bit-mask - 1 | |||
139 | TrueIfSigned = true; | |||
140 | return RHS.isMaxSignedValue(); | |||
141 | case ICmpInst::ICMP_UGE: | |||
142 | // True if LHS u>= RHS and RHS == high-bit-mask (2^7, 2^15, 2^31, etc) | |||
143 | TrueIfSigned = true; | |||
144 | return RHS.isSignMask(); | |||
145 | default: | |||
146 | return false; | |||
147 | } | |||
148 | } | |||
149 | ||||
150 | /// Returns true if the exploded icmp can be expressed as a signed comparison | |||
151 | /// to zero and updates the predicate accordingly. | |||
152 | /// The signedness of the comparison is preserved. | |||
153 | /// TODO: Refactor with decomposeBitTestICmp()? | |||
154 | static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) { | |||
155 | if (!ICmpInst::isSigned(Pred)) | |||
156 | return false; | |||
157 | ||||
158 | if (C == 0) | |||
159 | return ICmpInst::isRelational(Pred); | |||
160 | ||||
161 | if (C == 1) { | |||
162 | if (Pred == ICmpInst::ICMP_SLT) { | |||
163 | Pred = ICmpInst::ICMP_SLE; | |||
164 | return true; | |||
165 | } | |||
166 | } else if (C.isAllOnesValue()) { | |||
167 | if (Pred == ICmpInst::ICMP_SGT) { | |||
168 | Pred = ICmpInst::ICMP_SGE; | |||
169 | return true; | |||
170 | } | |||
171 | } | |||
172 | ||||
173 | return false; | |||
174 | } | |||
175 | ||||
176 | /// Given a signed integer type and a set of known zero and one bits, compute | |||
177 | /// the maximum and minimum values that could have the specified known zero and | |||
178 | /// known one bits, returning them in Min/Max. | |||
179 | /// TODO: Move to method on KnownBits struct? | |||
180 | static void computeSignedMinMaxValuesFromKnownBits(const KnownBits &Known, | |||
181 | APInt &Min, APInt &Max) { | |||
182 | assert(Known.getBitWidth() == Min.getBitWidth() &&((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 184, __PRETTY_FUNCTION__)) | |||
183 | Known.getBitWidth() == Max.getBitWidth() &&((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 184, __PRETTY_FUNCTION__)) | |||
184 | "KnownZero, KnownOne and Min, Max must have equal bitwidth.")((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 184, __PRETTY_FUNCTION__)); | |||
185 | APInt UnknownBits = ~(Known.Zero|Known.One); | |||
186 | ||||
187 | // The minimum value is when all unknown bits are zeros, EXCEPT for the sign | |||
188 | // bit if it is unknown. | |||
189 | Min = Known.One; | |||
190 | Max = Known.One|UnknownBits; | |||
191 | ||||
192 | if (UnknownBits.isNegative()) { // Sign bit is unknown | |||
193 | Min.setSignBit(); | |||
194 | Max.clearSignBit(); | |||
195 | } | |||
196 | } | |||
197 | ||||
198 | /// Given an unsigned integer type and a set of known zero and one bits, compute | |||
199 | /// the maximum and minimum values that could have the specified known zero and | |||
200 | /// known one bits, returning them in Min/Max. | |||
201 | /// TODO: Move to method on KnownBits struct? | |||
202 | static void computeUnsignedMinMaxValuesFromKnownBits(const KnownBits &Known, | |||
203 | APInt &Min, APInt &Max) { | |||
204 | assert(Known.getBitWidth() == Min.getBitWidth() &&((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 206, __PRETTY_FUNCTION__)) | |||
205 | Known.getBitWidth() == Max.getBitWidth() &&((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 206, __PRETTY_FUNCTION__)) | |||
206 | "Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth.")((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 206, __PRETTY_FUNCTION__)); | |||
207 | APInt UnknownBits = ~(Known.Zero|Known.One); | |||
208 | ||||
209 | // The minimum value is when the unknown bits are all zeros. | |||
210 | Min = Known.One; | |||
211 | // The maximum value is when the unknown bits are all ones. | |||
212 | Max = Known.One|UnknownBits; | |||
213 | } | |||
214 | ||||
215 | /// This is called when we see this pattern: | |||
216 | /// cmp pred (load (gep GV, ...)), cmpcst | |||
217 | /// where GV is a global variable with a constant initializer. Try to simplify | |||
218 | /// this into some simple computation that does not need the load. For example | |||
219 | /// we can optimize "icmp eq (load (gep "foo", 0, i)), 0" into "icmp eq i, 3". | |||
220 | /// | |||
221 | /// If AndCst is non-null, then the loaded value is masked with that constant | |||
222 | /// before doing the comparison. This handles cases like "A[i]&4 == 0". | |||
223 | Instruction *InstCombiner::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, | |||
224 | GlobalVariable *GV, | |||
225 | CmpInst &ICI, | |||
226 | ConstantInt *AndCst) { | |||
227 | Constant *Init = GV->getInitializer(); | |||
228 | if (!isa<ConstantArray>(Init) && !isa<ConstantDataArray>(Init)) | |||
229 | return nullptr; | |||
230 | ||||
231 | uint64_t ArrayElementCount = Init->getType()->getArrayNumElements(); | |||
232 | // Don't blow up on huge arrays. | |||
233 | if (ArrayElementCount > MaxArraySizeForCombine) | |||
234 | return nullptr; | |||
235 | ||||
236 | // There are many forms of this optimization we can handle, for now, just do | |||
237 | // the simple index into a single-dimensional array. | |||
238 | // | |||
239 | // Require: GEP GV, 0, i {{, constant indices}} | |||
240 | if (GEP->getNumOperands() < 3 || | |||
241 | !isa<ConstantInt>(GEP->getOperand(1)) || | |||
242 | !cast<ConstantInt>(GEP->getOperand(1))->isZero() || | |||
243 | isa<Constant>(GEP->getOperand(2))) | |||
244 | return nullptr; | |||
245 | ||||
246 | // Check that indices after the variable are constants and in-range for the | |||
247 | // type they index. Collect the indices. This is typically for arrays of | |||
248 | // structs. | |||
249 | SmallVector<unsigned, 4> LaterIndices; | |||
250 | ||||
251 | Type *EltTy = Init->getType()->getArrayElementType(); | |||
252 | for (unsigned i = 3, e = GEP->getNumOperands(); i != e; ++i) { | |||
253 | ConstantInt *Idx = dyn_cast<ConstantInt>(GEP->getOperand(i)); | |||
254 | if (!Idx) return nullptr; // Variable index. | |||
255 | ||||
256 | uint64_t IdxVal = Idx->getZExtValue(); | |||
257 | if ((unsigned)IdxVal != IdxVal) return nullptr; // Too large array index. | |||
258 | ||||
259 | if (StructType *STy = dyn_cast<StructType>(EltTy)) | |||
260 | EltTy = STy->getElementType(IdxVal); | |||
261 | else if (ArrayType *ATy = dyn_cast<ArrayType>(EltTy)) { | |||
262 | if (IdxVal >= ATy->getNumElements()) return nullptr; | |||
263 | EltTy = ATy->getElementType(); | |||
264 | } else { | |||
265 | return nullptr; // Unknown type. | |||
266 | } | |||
267 | ||||
268 | LaterIndices.push_back(IdxVal); | |||
269 | } | |||
270 | ||||
271 | enum { Overdefined = -3, Undefined = -2 }; | |||
272 | ||||
273 | // Variables for our state machines. | |||
274 | ||||
275 | // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form | |||
276 | // "i == 47 | i == 87", where 47 is the first index the condition is true for, | |||
277 | // and 87 is the second (and last) index. FirstTrueElement is -2 when | |||
278 | // undefined, otherwise set to the first true element. SecondTrueElement is | |||
279 | // -2 when undefined, -3 when overdefined and >= 0 when that index is true. | |||
280 | int FirstTrueElement = Undefined, SecondTrueElement = Undefined; | |||
281 | ||||
282 | // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the | |||
283 | // form "i != 47 & i != 87". Same state transitions as for true elements. | |||
284 | int FirstFalseElement = Undefined, SecondFalseElement = Undefined; | |||
285 | ||||
286 | /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these | |||
287 | /// define a state machine that triggers for ranges of values that the index | |||
288 | /// is true or false for. This triggers on things like "abbbbc"[i] == 'b'. | |||
289 | /// This is -2 when undefined, -3 when overdefined, and otherwise the last | |||
290 | /// index in the range (inclusive). We use -2 for undefined here because we | |||
291 | /// use relative comparisons and don't want 0-1 to match -1. | |||
292 | int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined; | |||
293 | ||||
294 | // MagicBitvector - This is a magic bitvector where we set a bit if the | |||
295 | // comparison is true for element 'i'. If there are 64 elements or less in | |||
296 | // the array, this will fully represent all the comparison results. | |||
297 | uint64_t MagicBitvector = 0; | |||
298 | ||||
299 | // Scan the array and see if one of our patterns matches. | |||
300 | Constant *CompareRHS = cast<Constant>(ICI.getOperand(1)); | |||
301 | for (unsigned i = 0, e = ArrayElementCount; i != e; ++i) { | |||
302 | Constant *Elt = Init->getAggregateElement(i); | |||
303 | if (!Elt) return nullptr; | |||
304 | ||||
305 | // If this is indexing an array of structures, get the structure element. | |||
306 | if (!LaterIndices.empty()) | |||
307 | Elt = ConstantExpr::getExtractValue(Elt, LaterIndices); | |||
308 | ||||
309 | // If the element is masked, handle it. | |||
310 | if (AndCst) Elt = ConstantExpr::getAnd(Elt, AndCst); | |||
311 | ||||
312 | // Find out if the comparison would be true or false for the i'th element. | |||
313 | Constant *C = ConstantFoldCompareInstOperands(ICI.getPredicate(), Elt, | |||
314 | CompareRHS, DL, &TLI); | |||
315 | // If the result is undef for this element, ignore it. | |||
316 | if (isa<UndefValue>(C)) { | |||
317 | // Extend range state machines to cover this element in case there is an | |||
318 | // undef in the middle of the range. | |||
319 | if (TrueRangeEnd == (int)i-1) | |||
320 | TrueRangeEnd = i; | |||
321 | if (FalseRangeEnd == (int)i-1) | |||
322 | FalseRangeEnd = i; | |||
323 | continue; | |||
324 | } | |||
325 | ||||
326 | // If we can't compute the result for any of the elements, we have to give | |||
327 | // up evaluating the entire conditional. | |||
328 | if (!isa<ConstantInt>(C)) return nullptr; | |||
329 | ||||
330 | // Otherwise, we know if the comparison is true or false for this element, | |||
331 | // update our state machines. | |||
332 | bool IsTrueForElt = !cast<ConstantInt>(C)->isZero(); | |||
333 | ||||
334 | // State machine for single/double/range index comparison. | |||
335 | if (IsTrueForElt) { | |||
336 | // Update the TrueElement state machine. | |||
337 | if (FirstTrueElement == Undefined) | |||
338 | FirstTrueElement = TrueRangeEnd = i; // First true element. | |||
339 | else { | |||
340 | // Update double-compare state machine. | |||
341 | if (SecondTrueElement == Undefined) | |||
342 | SecondTrueElement = i; | |||
343 | else | |||
344 | SecondTrueElement = Overdefined; | |||
345 | ||||
346 | // Update range state machine. | |||
347 | if (TrueRangeEnd == (int)i-1) | |||
348 | TrueRangeEnd = i; | |||
349 | else | |||
350 | TrueRangeEnd = Overdefined; | |||
351 | } | |||
352 | } else { | |||
353 | // Update the FalseElement state machine. | |||
354 | if (FirstFalseElement == Undefined) | |||
355 | FirstFalseElement = FalseRangeEnd = i; // First false element. | |||
356 | else { | |||
357 | // Update double-compare state machine. | |||
358 | if (SecondFalseElement == Undefined) | |||
359 | SecondFalseElement = i; | |||
360 | else | |||
361 | SecondFalseElement = Overdefined; | |||
362 | ||||
363 | // Update range state machine. | |||
364 | if (FalseRangeEnd == (int)i-1) | |||
365 | FalseRangeEnd = i; | |||
366 | else | |||
367 | FalseRangeEnd = Overdefined; | |||
368 | } | |||
369 | } | |||
370 | ||||
371 | // If this element is in range, update our magic bitvector. | |||
372 | if (i < 64 && IsTrueForElt) | |||
373 | MagicBitvector |= 1ULL << i; | |||
374 | ||||
375 | // If all of our states become overdefined, bail out early. Since the | |||
376 | // predicate is expensive, only check it every 8 elements. This is only | |||
377 | // really useful for really huge arrays. | |||
378 | if ((i & 8) == 0 && i >= 64 && SecondTrueElement == Overdefined && | |||
379 | SecondFalseElement == Overdefined && TrueRangeEnd == Overdefined && | |||
380 | FalseRangeEnd == Overdefined) | |||
381 | return nullptr; | |||
382 | } | |||
383 | ||||
384 | // Now that we've scanned the entire array, emit our new comparison(s). We | |||
385 | // order the state machines in complexity of the generated code. | |||
386 | Value *Idx = GEP->getOperand(2); | |||
387 | ||||
388 | // If the index is larger than the pointer size of the target, truncate the | |||
389 | // index down like the GEP would do implicitly. We don't have to do this for | |||
390 | // an inbounds GEP because the index can't be out of range. | |||
391 | if (!GEP->isInBounds()) { | |||
392 | Type *IntPtrTy = DL.getIntPtrType(GEP->getType()); | |||
393 | unsigned PtrSize = IntPtrTy->getIntegerBitWidth(); | |||
394 | if (Idx->getType()->getPrimitiveSizeInBits() > PtrSize) | |||
395 | Idx = Builder->CreateTrunc(Idx, IntPtrTy); | |||
396 | } | |||
397 | ||||
398 | // If the comparison is only true for one or two elements, emit direct | |||
399 | // comparisons. | |||
400 | if (SecondTrueElement != Overdefined) { | |||
401 | // None true -> false. | |||
402 | if (FirstTrueElement == Undefined) | |||
403 | return replaceInstUsesWith(ICI, Builder->getFalse()); | |||
404 | ||||
405 | Value *FirstTrueIdx = ConstantInt::get(Idx->getType(), FirstTrueElement); | |||
406 | ||||
407 | // True for one element -> 'i == 47'. | |||
408 | if (SecondTrueElement == Undefined) | |||
409 | return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx); | |||
410 | ||||
411 | // True for two elements -> 'i == 47 | i == 72'. | |||
412 | Value *C1 = Builder->CreateICmpEQ(Idx, FirstTrueIdx); | |||
413 | Value *SecondTrueIdx = ConstantInt::get(Idx->getType(), SecondTrueElement); | |||
414 | Value *C2 = Builder->CreateICmpEQ(Idx, SecondTrueIdx); | |||
415 | return BinaryOperator::CreateOr(C1, C2); | |||
416 | } | |||
417 | ||||
418 | // If the comparison is only false for one or two elements, emit direct | |||
419 | // comparisons. | |||
420 | if (SecondFalseElement != Overdefined) { | |||
421 | // None false -> true. | |||
422 | if (FirstFalseElement == Undefined) | |||
423 | return replaceInstUsesWith(ICI, Builder->getTrue()); | |||
424 | ||||
425 | Value *FirstFalseIdx = ConstantInt::get(Idx->getType(), FirstFalseElement); | |||
426 | ||||
427 | // False for one element -> 'i != 47'. | |||
428 | if (SecondFalseElement == Undefined) | |||
429 | return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx); | |||
430 | ||||
431 | // False for two elements -> 'i != 47 & i != 72'. | |||
432 | Value *C1 = Builder->CreateICmpNE(Idx, FirstFalseIdx); | |||
433 | Value *SecondFalseIdx = ConstantInt::get(Idx->getType(),SecondFalseElement); | |||
434 | Value *C2 = Builder->CreateICmpNE(Idx, SecondFalseIdx); | |||
435 | return BinaryOperator::CreateAnd(C1, C2); | |||
436 | } | |||
437 | ||||
438 | // If the comparison can be replaced with a range comparison for the elements | |||
439 | // where it is true, emit the range check. | |||
440 | if (TrueRangeEnd != Overdefined) { | |||
441 | assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare")((TrueRangeEnd != FirstTrueElement && "Should emit single compare" ) ? static_cast<void> (0) : __assert_fail ("TrueRangeEnd != FirstTrueElement && \"Should emit single compare\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 441, __PRETTY_FUNCTION__)); | |||
442 | ||||
443 | // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1). | |||
444 | if (FirstTrueElement) { | |||
445 | Value *Offs = ConstantInt::get(Idx->getType(), -FirstTrueElement); | |||
446 | Idx = Builder->CreateAdd(Idx, Offs); | |||
447 | } | |||
448 | ||||
449 | Value *End = ConstantInt::get(Idx->getType(), | |||
450 | TrueRangeEnd-FirstTrueElement+1); | |||
451 | return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End); | |||
452 | } | |||
453 | ||||
454 | // False range check. | |||
455 | if (FalseRangeEnd != Overdefined) { | |||
456 | assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare")((FalseRangeEnd != FirstFalseElement && "Should emit single compare" ) ? static_cast<void> (0) : __assert_fail ("FalseRangeEnd != FirstFalseElement && \"Should emit single compare\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 456, __PRETTY_FUNCTION__)); | |||
457 | // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse). | |||
458 | if (FirstFalseElement) { | |||
459 | Value *Offs = ConstantInt::get(Idx->getType(), -FirstFalseElement); | |||
460 | Idx = Builder->CreateAdd(Idx, Offs); | |||
461 | } | |||
462 | ||||
463 | Value *End = ConstantInt::get(Idx->getType(), | |||
464 | FalseRangeEnd-FirstFalseElement); | |||
465 | return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End); | |||
466 | } | |||
467 | ||||
468 | // If a magic bitvector captures the entire comparison state | |||
469 | // of this load, replace it with computation that does: | |||
470 | // ((magic_cst >> i) & 1) != 0 | |||
471 | { | |||
472 | Type *Ty = nullptr; | |||
473 | ||||
474 | // Look for an appropriate type: | |||
475 | // - The type of Idx if the magic fits | |||
476 | // - The smallest fitting legal type if we have a DataLayout | |||
477 | // - Default to i32 | |||
478 | if (ArrayElementCount <= Idx->getType()->getIntegerBitWidth()) | |||
479 | Ty = Idx->getType(); | |||
480 | else | |||
481 | Ty = DL.getSmallestLegalIntType(Init->getContext(), ArrayElementCount); | |||
482 | ||||
483 | if (Ty) { | |||
484 | Value *V = Builder->CreateIntCast(Idx, Ty, false); | |||
485 | V = Builder->CreateLShr(ConstantInt::get(Ty, MagicBitvector), V); | |||
486 | V = Builder->CreateAnd(ConstantInt::get(Ty, 1), V); | |||
487 | return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, 0)); | |||
488 | } | |||
489 | } | |||
490 | ||||
491 | return nullptr; | |||
492 | } | |||
493 | ||||
494 | /// Return a value that can be used to compare the *offset* implied by a GEP to | |||
495 | /// zero. For example, if we have &A[i], we want to return 'i' for | |||
496 | /// "icmp ne i, 0". Note that, in general, indices can be complex, and scales | |||
497 | /// are involved. The above expression would also be legal to codegen as | |||
498 | /// "icmp ne (i*4), 0" (assuming A is a pointer to i32). | |||
499 | /// This latter form is less amenable to optimization though, and we are allowed | |||
500 | /// to generate the first by knowing that pointer arithmetic doesn't overflow. | |||
501 | /// | |||
502 | /// If we can't emit an optimized form for this expression, this returns null. | |||
503 | /// | |||
504 | static Value *evaluateGEPOffsetExpression(User *GEP, InstCombiner &IC, | |||
505 | const DataLayout &DL) { | |||
506 | gep_type_iterator GTI = gep_type_begin(GEP); | |||
507 | ||||
508 | // Check to see if this gep only has a single variable index. If so, and if | |||
509 | // any constant indices are a multiple of its scale, then we can compute this | |||
510 | // in terms of the scale of the variable index. For example, if the GEP | |||
511 | // implies an offset of "12 + i*4", then we can codegen this as "3 + i", | |||
512 | // because the expression will cross zero at the same point. | |||
513 | unsigned i, e = GEP->getNumOperands(); | |||
514 | int64_t Offset = 0; | |||
515 | for (i = 1; i != e; ++i, ++GTI) { | |||
516 | if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) { | |||
517 | // Compute the aggregate offset of constant indices. | |||
518 | if (CI->isZero()) continue; | |||
519 | ||||
520 | // Handle a struct index, which adds its field offset to the pointer. | |||
521 | if (StructType *STy = GTI.getStructTypeOrNull()) { | |||
522 | Offset += DL.getStructLayout(STy)->getElementOffset(CI->getZExtValue()); | |||
523 | } else { | |||
524 | uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType()); | |||
525 | Offset += Size*CI->getSExtValue(); | |||
526 | } | |||
527 | } else { | |||
528 | // Found our variable index. | |||
529 | break; | |||
530 | } | |||
531 | } | |||
532 | ||||
533 | // If there are no variable indices, we must have a constant offset, just | |||
534 | // evaluate it the general way. | |||
535 | if (i == e) return nullptr; | |||
536 | ||||
537 | Value *VariableIdx = GEP->getOperand(i); | |||
538 | // Determine the scale factor of the variable element. For example, this is | |||
539 | // 4 if the variable index is into an array of i32. | |||
540 | uint64_t VariableScale = DL.getTypeAllocSize(GTI.getIndexedType()); | |||
541 | ||||
542 | // Verify that there are no other variable indices. If so, emit the hard way. | |||
543 | for (++i, ++GTI; i != e; ++i, ++GTI) { | |||
544 | ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i)); | |||
545 | if (!CI) return nullptr; | |||
546 | ||||
547 | // Compute the aggregate offset of constant indices. | |||
548 | if (CI->isZero()) continue; | |||
549 | ||||
550 | // Handle a struct index, which adds its field offset to the pointer. | |||
551 | if (StructType *STy = GTI.getStructTypeOrNull()) { | |||
552 | Offset += DL.getStructLayout(STy)->getElementOffset(CI->getZExtValue()); | |||
553 | } else { | |||
554 | uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType()); | |||
555 | Offset += Size*CI->getSExtValue(); | |||
556 | } | |||
557 | } | |||
558 | ||||
559 | // Okay, we know we have a single variable index, which must be a | |||
560 | // pointer/array/vector index. If there is no offset, life is simple, return | |||
561 | // the index. | |||
562 | Type *IntPtrTy = DL.getIntPtrType(GEP->getOperand(0)->getType()); | |||
563 | unsigned IntPtrWidth = IntPtrTy->getIntegerBitWidth(); | |||
564 | if (Offset == 0) { | |||
565 | // Cast to intptrty in case a truncation occurs. If an extension is needed, | |||
566 | // we don't need to bother extending: the extension won't affect where the | |||
567 | // computation crosses zero. | |||
568 | if (VariableIdx->getType()->getPrimitiveSizeInBits() > IntPtrWidth) { | |||
569 | VariableIdx = IC.Builder->CreateTrunc(VariableIdx, IntPtrTy); | |||
570 | } | |||
571 | return VariableIdx; | |||
572 | } | |||
573 | ||||
574 | // Otherwise, there is an index. The computation we will do will be modulo | |||
575 | // the pointer size, so get it. | |||
576 | uint64_t PtrSizeMask = ~0ULL >> (64-IntPtrWidth); | |||
577 | ||||
578 | Offset &= PtrSizeMask; | |||
579 | VariableScale &= PtrSizeMask; | |||
580 | ||||
581 | // To do this transformation, any constant index must be a multiple of the | |||
582 | // variable scale factor. For example, we can evaluate "12 + 4*i" as "3 + i", | |||
583 | // but we can't evaluate "10 + 3*i" in terms of i. Check that the offset is a | |||
584 | // multiple of the variable scale. | |||
585 | int64_t NewOffs = Offset / (int64_t)VariableScale; | |||
586 | if (Offset != NewOffs*(int64_t)VariableScale) | |||
587 | return nullptr; | |||
588 | ||||
589 | // Okay, we can do this evaluation. Start by converting the index to intptr. | |||
590 | if (VariableIdx->getType() != IntPtrTy) | |||
591 | VariableIdx = IC.Builder->CreateIntCast(VariableIdx, IntPtrTy, | |||
592 | true /*Signed*/); | |||
593 | Constant *OffsetVal = ConstantInt::get(IntPtrTy, NewOffs); | |||
594 | return IC.Builder->CreateAdd(VariableIdx, OffsetVal, "offset"); | |||
595 | } | |||
596 | ||||
597 | /// Returns true if we can rewrite Start as a GEP with pointer Base | |||
598 | /// and some integer offset. The nodes that need to be re-written | |||
599 | /// for this transformation will be added to Explored. | |||
600 | static bool canRewriteGEPAsOffset(Value *Start, Value *Base, | |||
601 | const DataLayout &DL, | |||
602 | SetVector<Value *> &Explored) { | |||
603 | SmallVector<Value *, 16> WorkList(1, Start); | |||
604 | Explored.insert(Base); | |||
605 | ||||
606 | // The following traversal gives us an order which can be used | |||
607 | // when doing the final transformation. Since in the final | |||
608 | // transformation we create the PHI replacement instructions first, | |||
609 | // we don't have to get them in any particular order. | |||
610 | // | |||
611 | // However, for other instructions we will have to traverse the | |||
612 | // operands of an instruction first, which means that we have to | |||
613 | // do a post-order traversal. | |||
614 | while (!WorkList.empty()) { | |||
615 | SetVector<PHINode *> PHIs; | |||
616 | ||||
617 | while (!WorkList.empty()) { | |||
618 | if (Explored.size() >= 100) | |||
619 | return false; | |||
620 | ||||
621 | Value *V = WorkList.back(); | |||
622 | ||||
623 | if (Explored.count(V) != 0) { | |||
624 | WorkList.pop_back(); | |||
625 | continue; | |||
626 | } | |||
627 | ||||
628 | if (!isa<IntToPtrInst>(V) && !isa<PtrToIntInst>(V) && | |||
629 | !isa<GetElementPtrInst>(V) && !isa<PHINode>(V)) | |||
630 | // We've found some value that we can't explore which is different from | |||
631 | // the base. Therefore we can't do this transformation. | |||
632 | return false; | |||
633 | ||||
634 | if (isa<IntToPtrInst>(V) || isa<PtrToIntInst>(V)) { | |||
635 | auto *CI = dyn_cast<CastInst>(V); | |||
636 | if (!CI->isNoopCast(DL)) | |||
637 | return false; | |||
638 | ||||
639 | if (Explored.count(CI->getOperand(0)) == 0) | |||
640 | WorkList.push_back(CI->getOperand(0)); | |||
641 | } | |||
642 | ||||
643 | if (auto *GEP = dyn_cast<GEPOperator>(V)) { | |||
644 | // We're limiting the GEP to having one index. This will preserve | |||
645 | // the original pointer type. We could handle more cases in the | |||
646 | // future. | |||
647 | if (GEP->getNumIndices() != 1 || !GEP->isInBounds() || | |||
648 | GEP->getType() != Start->getType()) | |||
649 | return false; | |||
650 | ||||
651 | if (Explored.count(GEP->getOperand(0)) == 0) | |||
652 | WorkList.push_back(GEP->getOperand(0)); | |||
653 | } | |||
654 | ||||
655 | if (WorkList.back() == V) { | |||
656 | WorkList.pop_back(); | |||
657 | // We've finished visiting this node, mark it as such. | |||
658 | Explored.insert(V); | |||
659 | } | |||
660 | ||||
661 | if (auto *PN = dyn_cast<PHINode>(V)) { | |||
662 | // We cannot transform PHIs on unsplittable basic blocks. | |||
663 | if (isa<CatchSwitchInst>(PN->getParent()->getTerminator())) | |||
664 | return false; | |||
665 | Explored.insert(PN); | |||
666 | PHIs.insert(PN); | |||
667 | } | |||
668 | } | |||
669 | ||||
670 | // Explore the PHI nodes further. | |||
671 | for (auto *PN : PHIs) | |||
672 | for (Value *Op : PN->incoming_values()) | |||
673 | if (Explored.count(Op) == 0) | |||
674 | WorkList.push_back(Op); | |||
675 | } | |||
676 | ||||
677 | // Make sure that we can do this. Since we can't insert GEPs in a basic | |||
678 | // block before a PHI node, we can't easily do this transformation if | |||
679 | // we have PHI node users of transformed instructions. | |||
680 | for (Value *Val : Explored) { | |||
681 | for (Value *Use : Val->uses()) { | |||
682 | ||||
683 | auto *PHI = dyn_cast<PHINode>(Use); | |||
684 | auto *Inst = dyn_cast<Instruction>(Val); | |||
685 | ||||
686 | if (Inst == Base || Inst == PHI || !Inst || !PHI || | |||
687 | Explored.count(PHI) == 0) | |||
688 | continue; | |||
689 | ||||
690 | if (PHI->getParent() == Inst->getParent()) | |||
691 | return false; | |||
692 | } | |||
693 | } | |||
694 | return true; | |||
695 | } | |||
696 | ||||
697 | // Sets the appropriate insert point on Builder where we can add | |||
698 | // a replacement Instruction for V (if that is possible). | |||
699 | static void setInsertionPoint(IRBuilder<> &Builder, Value *V, | |||
700 | bool Before = true) { | |||
701 | if (auto *PHI = dyn_cast<PHINode>(V)) { | |||
702 | Builder.SetInsertPoint(&*PHI->getParent()->getFirstInsertionPt()); | |||
703 | return; | |||
704 | } | |||
705 | if (auto *I = dyn_cast<Instruction>(V)) { | |||
706 | if (!Before) | |||
707 | I = &*std::next(I->getIterator()); | |||
708 | Builder.SetInsertPoint(I); | |||
709 | return; | |||
710 | } | |||
711 | if (auto *A = dyn_cast<Argument>(V)) { | |||
712 | // Set the insertion point in the entry block. | |||
713 | BasicBlock &Entry = A->getParent()->getEntryBlock(); | |||
714 | Builder.SetInsertPoint(&*Entry.getFirstInsertionPt()); | |||
715 | return; | |||
716 | } | |||
717 | // Otherwise, this is a constant and we don't need to set a new | |||
718 | // insertion point. | |||
719 | assert(isa<Constant>(V) && "Setting insertion point for unknown value!")((isa<Constant>(V) && "Setting insertion point for unknown value!" ) ? static_cast<void> (0) : __assert_fail ("isa<Constant>(V) && \"Setting insertion point for unknown value!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 719, __PRETTY_FUNCTION__)); | |||
720 | } | |||
721 | ||||
722 | /// Returns a re-written value of Start as an indexed GEP using Base as a | |||
723 | /// pointer. | |||
724 | static Value *rewriteGEPAsOffset(Value *Start, Value *Base, | |||
725 | const DataLayout &DL, | |||
726 | SetVector<Value *> &Explored) { | |||
727 | // Perform all the substitutions. This is a bit tricky because we can | |||
728 | // have cycles in our use-def chains. | |||
729 | // 1. Create the PHI nodes without any incoming values. | |||
730 | // 2. Create all the other values. | |||
731 | // 3. Add the edges for the PHI nodes. | |||
732 | // 4. Emit GEPs to get the original pointers. | |||
733 | // 5. Remove the original instructions. | |||
734 | Type *IndexType = IntegerType::get( | |||
735 | Base->getContext(), DL.getPointerTypeSizeInBits(Start->getType())); | |||
736 | ||||
737 | DenseMap<Value *, Value *> NewInsts; | |||
738 | NewInsts[Base] = ConstantInt::getNullValue(IndexType); | |||
739 | ||||
740 | // Create the new PHI nodes, without adding any incoming values. | |||
741 | for (Value *Val : Explored) { | |||
742 | if (Val == Base) | |||
743 | continue; | |||
744 | // Create empty phi nodes. This avoids cyclic dependencies when creating | |||
745 | // the remaining instructions. | |||
746 | if (auto *PHI = dyn_cast<PHINode>(Val)) | |||
747 | NewInsts[PHI] = PHINode::Create(IndexType, PHI->getNumIncomingValues(), | |||
748 | PHI->getName() + ".idx", PHI); | |||
749 | } | |||
750 | IRBuilder<> Builder(Base->getContext()); | |||
751 | ||||
752 | // Create all the other instructions. | |||
753 | for (Value *Val : Explored) { | |||
754 | ||||
755 | if (NewInsts.find(Val) != NewInsts.end()) | |||
756 | continue; | |||
757 | ||||
758 | if (auto *CI = dyn_cast<CastInst>(Val)) { | |||
759 | NewInsts[CI] = NewInsts[CI->getOperand(0)]; | |||
760 | continue; | |||
761 | } | |||
762 | if (auto *GEP = dyn_cast<GEPOperator>(Val)) { | |||
763 | Value *Index = NewInsts[GEP->getOperand(1)] ? NewInsts[GEP->getOperand(1)] | |||
764 | : GEP->getOperand(1); | |||
765 | setInsertionPoint(Builder, GEP); | |||
766 | // Indices might need to be sign extended. GEPs will magically do | |||
767 | // this, but we need to do it ourselves here. | |||
768 | if (Index->getType()->getScalarSizeInBits() != | |||
769 | NewInsts[GEP->getOperand(0)]->getType()->getScalarSizeInBits()) { | |||
770 | Index = Builder.CreateSExtOrTrunc( | |||
771 | Index, NewInsts[GEP->getOperand(0)]->getType(), | |||
772 | GEP->getOperand(0)->getName() + ".sext"); | |||
773 | } | |||
774 | ||||
775 | auto *Op = NewInsts[GEP->getOperand(0)]; | |||
776 | if (isa<ConstantInt>(Op) && dyn_cast<ConstantInt>(Op)->isZero()) | |||
777 | NewInsts[GEP] = Index; | |||
778 | else | |||
779 | NewInsts[GEP] = Builder.CreateNSWAdd( | |||
780 | Op, Index, GEP->getOperand(0)->getName() + ".add"); | |||
781 | continue; | |||
782 | } | |||
783 | if (isa<PHINode>(Val)) | |||
784 | continue; | |||
785 | ||||
786 | llvm_unreachable("Unexpected instruction type")::llvm::llvm_unreachable_internal("Unexpected instruction type" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 786); | |||
787 | } | |||
788 | ||||
789 | // Add the incoming values to the PHI nodes. | |||
790 | for (Value *Val : Explored) { | |||
791 | if (Val == Base) | |||
792 | continue; | |||
793 | // All the instructions have been created, we can now add edges to the | |||
794 | // phi nodes. | |||
795 | if (auto *PHI = dyn_cast<PHINode>(Val)) { | |||
796 | PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]); | |||
797 | for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) { | |||
798 | Value *NewIncoming = PHI->getIncomingValue(I); | |||
799 | ||||
800 | if (NewInsts.find(NewIncoming) != NewInsts.end()) | |||
801 | NewIncoming = NewInsts[NewIncoming]; | |||
802 | ||||
803 | NewPhi->addIncoming(NewIncoming, PHI->getIncomingBlock(I)); | |||
804 | } | |||
805 | } | |||
806 | } | |||
807 | ||||
808 | for (Value *Val : Explored) { | |||
809 | if (Val == Base) | |||
810 | continue; | |||
811 | ||||
812 | // Depending on the type, for external users we have to emit | |||
813 | // a GEP or a GEP + ptrtoint. | |||
814 | setInsertionPoint(Builder, Val, false); | |||
815 | ||||
816 | // If required, create an inttoptr instruction for Base. | |||
817 | Value *NewBase = Base; | |||
818 | if (!Base->getType()->isPointerTy()) | |||
819 | NewBase = Builder.CreateBitOrPointerCast(Base, Start->getType(), | |||
820 | Start->getName() + "to.ptr"); | |||
821 | ||||
822 | Value *GEP = Builder.CreateInBoundsGEP( | |||
823 | Start->getType()->getPointerElementType(), NewBase, | |||
824 | makeArrayRef(NewInsts[Val]), Val->getName() + ".ptr"); | |||
825 | ||||
826 | if (!Val->getType()->isPointerTy()) { | |||
827 | Value *Cast = Builder.CreatePointerCast(GEP, Val->getType(), | |||
828 | Val->getName() + ".conv"); | |||
829 | GEP = Cast; | |||
830 | } | |||
831 | Val->replaceAllUsesWith(GEP); | |||
832 | } | |||
833 | ||||
834 | return NewInsts[Start]; | |||
835 | } | |||
836 | ||||
837 | /// Looks through GEPs, IntToPtrInsts and PtrToIntInsts in order to express | |||
838 | /// the input Value as a constant indexed GEP. Returns a pair containing | |||
839 | /// the GEPs Pointer and Index. | |||
840 | static std::pair<Value *, Value *> | |||
841 | getAsConstantIndexedAddress(Value *V, const DataLayout &DL) { | |||
842 | Type *IndexType = IntegerType::get(V->getContext(), | |||
843 | DL.getPointerTypeSizeInBits(V->getType())); | |||
844 | ||||
845 | Constant *Index = ConstantInt::getNullValue(IndexType); | |||
846 | while (true) { | |||
847 | if (GEPOperator *GEP = dyn_cast<GEPOperator>(V)) { | |||
848 | // We accept only inbouds GEPs here to exclude the possibility of | |||
849 | // overflow. | |||
850 | if (!GEP->isInBounds()) | |||
851 | break; | |||
852 | if (GEP->hasAllConstantIndices() && GEP->getNumIndices() == 1 && | |||
853 | GEP->getType() == V->getType()) { | |||
854 | V = GEP->getOperand(0); | |||
855 | Constant *GEPIndex = static_cast<Constant *>(GEP->getOperand(1)); | |||
856 | Index = ConstantExpr::getAdd( | |||
857 | Index, ConstantExpr::getSExtOrBitCast(GEPIndex, IndexType)); | |||
858 | continue; | |||
859 | } | |||
860 | break; | |||
861 | } | |||
862 | if (auto *CI = dyn_cast<IntToPtrInst>(V)) { | |||
863 | if (!CI->isNoopCast(DL)) | |||
864 | break; | |||
865 | V = CI->getOperand(0); | |||
866 | continue; | |||
867 | } | |||
868 | if (auto *CI = dyn_cast<PtrToIntInst>(V)) { | |||
869 | if (!CI->isNoopCast(DL)) | |||
870 | break; | |||
871 | V = CI->getOperand(0); | |||
872 | continue; | |||
873 | } | |||
874 | break; | |||
875 | } | |||
876 | return {V, Index}; | |||
877 | } | |||
878 | ||||
879 | /// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant. | |||
880 | /// We can look through PHIs, GEPs and casts in order to determine a common base | |||
881 | /// between GEPLHS and RHS. | |||
882 | static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS, | |||
883 | ICmpInst::Predicate Cond, | |||
884 | const DataLayout &DL) { | |||
885 | if (!GEPLHS->hasAllConstantIndices()) | |||
886 | return nullptr; | |||
887 | ||||
888 | // Make sure the pointers have the same type. | |||
889 | if (GEPLHS->getType() != RHS->getType()) | |||
890 | return nullptr; | |||
891 | ||||
892 | Value *PtrBase, *Index; | |||
893 | std::tie(PtrBase, Index) = getAsConstantIndexedAddress(GEPLHS, DL); | |||
894 | ||||
895 | // The set of nodes that will take part in this transformation. | |||
896 | SetVector<Value *> Nodes; | |||
897 | ||||
898 | if (!canRewriteGEPAsOffset(RHS, PtrBase, DL, Nodes)) | |||
899 | return nullptr; | |||
900 | ||||
901 | // We know we can re-write this as | |||
902 | // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) | |||
903 | // Since we've only looked through inbouds GEPs we know that we | |||
904 | // can't have overflow on either side. We can therefore re-write | |||
905 | // this as: | |||
906 | // OFFSET1 cmp OFFSET2 | |||
907 | Value *NewRHS = rewriteGEPAsOffset(RHS, PtrBase, DL, Nodes); | |||
908 | ||||
909 | // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written | |||
910 | // GEP having PtrBase as the pointer base, and has returned in NewRHS the | |||
911 | // offset. Since Index is the offset of LHS to the base pointer, we will now | |||
912 | // compare the offsets instead of comparing the pointers. | |||
913 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Index, NewRHS); | |||
914 | } | |||
915 | ||||
916 | /// Fold comparisons between a GEP instruction and something else. At this point | |||
917 | /// we know that the GEP is on the LHS of the comparison. | |||
918 | Instruction *InstCombiner::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, | |||
919 | ICmpInst::Predicate Cond, | |||
920 | Instruction &I) { | |||
921 | // Don't transform signed compares of GEPs into index compares. Even if the | |||
922 | // GEP is inbounds, the final add of the base pointer can have signed overflow | |||
923 | // and would change the result of the icmp. | |||
924 | // e.g. "&foo[0] <s &foo[1]" can't be folded to "true" because "foo" could be | |||
925 | // the maximum signed value for the pointer type. | |||
926 | if (ICmpInst::isSigned(Cond)) | |||
927 | return nullptr; | |||
928 | ||||
929 | // Look through bitcasts and addrspacecasts. We do not however want to remove | |||
930 | // 0 GEPs. | |||
931 | if (!isa<GetElementPtrInst>(RHS)) | |||
932 | RHS = RHS->stripPointerCasts(); | |||
933 | ||||
934 | Value *PtrBase = GEPLHS->getOperand(0); | |||
935 | if (PtrBase == RHS && GEPLHS->isInBounds()) { | |||
936 | // ((gep Ptr, OFFSET) cmp Ptr) ---> (OFFSET cmp 0). | |||
937 | // This transformation (ignoring the base and scales) is valid because we | |||
938 | // know pointers can't overflow since the gep is inbounds. See if we can | |||
939 | // output an optimized form. | |||
940 | Value *Offset = evaluateGEPOffsetExpression(GEPLHS, *this, DL); | |||
941 | ||||
942 | // If not, synthesize the offset the hard way. | |||
943 | if (!Offset) | |||
944 | Offset = EmitGEPOffset(GEPLHS); | |||
945 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Offset, | |||
946 | Constant::getNullValue(Offset->getType())); | |||
947 | } else if (GEPOperator *GEPRHS = dyn_cast<GEPOperator>(RHS)) { | |||
948 | // If the base pointers are different, but the indices are the same, just | |||
949 | // compare the base pointer. | |||
950 | if (PtrBase != GEPRHS->getOperand(0)) { | |||
951 | bool IndicesTheSame = GEPLHS->getNumOperands()==GEPRHS->getNumOperands(); | |||
952 | IndicesTheSame &= GEPLHS->getOperand(0)->getType() == | |||
953 | GEPRHS->getOperand(0)->getType(); | |||
954 | if (IndicesTheSame) | |||
955 | for (unsigned i = 1, e = GEPLHS->getNumOperands(); i != e; ++i) | |||
956 | if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) { | |||
957 | IndicesTheSame = false; | |||
958 | break; | |||
959 | } | |||
960 | ||||
961 | // If all indices are the same, just compare the base pointers. | |||
962 | if (IndicesTheSame) | |||
963 | return new ICmpInst(Cond, GEPLHS->getOperand(0), GEPRHS->getOperand(0)); | |||
964 | ||||
965 | // If we're comparing GEPs with two base pointers that only differ in type | |||
966 | // and both GEPs have only constant indices or just one use, then fold | |||
967 | // the compare with the adjusted indices. | |||
968 | if (GEPLHS->isInBounds() && GEPRHS->isInBounds() && | |||
969 | (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) && | |||
970 | (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse()) && | |||
971 | PtrBase->stripPointerCasts() == | |||
972 | GEPRHS->getOperand(0)->stripPointerCasts()) { | |||
973 | Value *LOffset = EmitGEPOffset(GEPLHS); | |||
974 | Value *ROffset = EmitGEPOffset(GEPRHS); | |||
975 | ||||
976 | // If we looked through an addrspacecast between different sized address | |||
977 | // spaces, the LHS and RHS pointers are different sized | |||
978 | // integers. Truncate to the smaller one. | |||
979 | Type *LHSIndexTy = LOffset->getType(); | |||
980 | Type *RHSIndexTy = ROffset->getType(); | |||
981 | if (LHSIndexTy != RHSIndexTy) { | |||
982 | if (LHSIndexTy->getPrimitiveSizeInBits() < | |||
983 | RHSIndexTy->getPrimitiveSizeInBits()) { | |||
984 | ROffset = Builder->CreateTrunc(ROffset, LHSIndexTy); | |||
985 | } else | |||
986 | LOffset = Builder->CreateTrunc(LOffset, RHSIndexTy); | |||
987 | } | |||
988 | ||||
989 | Value *Cmp = Builder->CreateICmp(ICmpInst::getSignedPredicate(Cond), | |||
990 | LOffset, ROffset); | |||
991 | return replaceInstUsesWith(I, Cmp); | |||
992 | } | |||
993 | ||||
994 | // Otherwise, the base pointers are different and the indices are | |||
995 | // different. Try convert this to an indexed compare by looking through | |||
996 | // PHIs/casts. | |||
997 | return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); | |||
998 | } | |||
999 | ||||
1000 | // If one of the GEPs has all zero indices, recurse. | |||
1001 | if (GEPLHS->hasAllZeroIndices()) | |||
1002 | return foldGEPICmp(GEPRHS, GEPLHS->getOperand(0), | |||
1003 | ICmpInst::getSwappedPredicate(Cond), I); | |||
1004 | ||||
1005 | // If the other GEP has all zero indices, recurse. | |||
1006 | if (GEPRHS->hasAllZeroIndices()) | |||
1007 | return foldGEPICmp(GEPLHS, GEPRHS->getOperand(0), Cond, I); | |||
1008 | ||||
1009 | bool GEPsInBounds = GEPLHS->isInBounds() && GEPRHS->isInBounds(); | |||
1010 | if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands()) { | |||
1011 | // If the GEPs only differ by one index, compare it. | |||
1012 | unsigned NumDifferences = 0; // Keep track of # differences. | |||
1013 | unsigned DiffOperand = 0; // The operand that differs. | |||
1014 | for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i) | |||
1015 | if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) { | |||
1016 | if (GEPLHS->getOperand(i)->getType()->getPrimitiveSizeInBits() != | |||
1017 | GEPRHS->getOperand(i)->getType()->getPrimitiveSizeInBits()) { | |||
1018 | // Irreconcilable differences. | |||
1019 | NumDifferences = 2; | |||
1020 | break; | |||
1021 | } else { | |||
1022 | if (NumDifferences++) break; | |||
1023 | DiffOperand = i; | |||
1024 | } | |||
1025 | } | |||
1026 | ||||
1027 | if (NumDifferences == 0) // SAME GEP? | |||
1028 | return replaceInstUsesWith(I, // No comparison is needed here. | |||
1029 | Builder->getInt1(ICmpInst::isTrueWhenEqual(Cond))); | |||
1030 | ||||
1031 | else if (NumDifferences == 1 && GEPsInBounds) { | |||
1032 | Value *LHSV = GEPLHS->getOperand(DiffOperand); | |||
1033 | Value *RHSV = GEPRHS->getOperand(DiffOperand); | |||
1034 | // Make sure we do a signed comparison here. | |||
1035 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), LHSV, RHSV); | |||
1036 | } | |||
1037 | } | |||
1038 | ||||
1039 | // Only lower this if the icmp is the only user of the GEP or if we expect | |||
1040 | // the result to fold to a constant! | |||
1041 | if (GEPsInBounds && (isa<ConstantExpr>(GEPLHS) || GEPLHS->hasOneUse()) && | |||
1042 | (isa<ConstantExpr>(GEPRHS) || GEPRHS->hasOneUse())) { | |||
1043 | // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2) | |||
1044 | Value *L = EmitGEPOffset(GEPLHS); | |||
1045 | Value *R = EmitGEPOffset(GEPRHS); | |||
1046 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), L, R); | |||
1047 | } | |||
1048 | } | |||
1049 | ||||
1050 | // Try convert this to an indexed compare by looking through PHIs/casts as a | |||
1051 | // last resort. | |||
1052 | return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); | |||
1053 | } | |||
1054 | ||||
1055 | Instruction *InstCombiner::foldAllocaCmp(ICmpInst &ICI, | |||
1056 | const AllocaInst *Alloca, | |||
1057 | const Value *Other) { | |||
1058 | assert(ICI.isEquality() && "Cannot fold non-equality comparison.")((ICI.isEquality() && "Cannot fold non-equality comparison." ) ? static_cast<void> (0) : __assert_fail ("ICI.isEquality() && \"Cannot fold non-equality comparison.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1058, __PRETTY_FUNCTION__)); | |||
1059 | ||||
1060 | // It would be tempting to fold away comparisons between allocas and any | |||
1061 | // pointer not based on that alloca (e.g. an argument). However, even | |||
1062 | // though such pointers cannot alias, they can still compare equal. | |||
1063 | // | |||
1064 | // But LLVM doesn't specify where allocas get their memory, so if the alloca | |||
1065 | // doesn't escape we can argue that it's impossible to guess its value, and we | |||
1066 | // can therefore act as if any such guesses are wrong. | |||
1067 | // | |||
1068 | // The code below checks that the alloca doesn't escape, and that it's only | |||
1069 | // used in a comparison once (the current instruction). The | |||
1070 | // single-comparison-use condition ensures that we're trivially folding all | |||
1071 | // comparisons against the alloca consistently, and avoids the risk of | |||
1072 | // erroneously folding a comparison of the pointer with itself. | |||
1073 | ||||
1074 | unsigned MaxIter = 32; // Break cycles and bound to constant-time. | |||
1075 | ||||
1076 | SmallVector<const Use *, 32> Worklist; | |||
1077 | for (const Use &U : Alloca->uses()) { | |||
1078 | if (Worklist.size() >= MaxIter) | |||
1079 | return nullptr; | |||
1080 | Worklist.push_back(&U); | |||
1081 | } | |||
1082 | ||||
1083 | unsigned NumCmps = 0; | |||
1084 | while (!Worklist.empty()) { | |||
1085 | assert(Worklist.size() <= MaxIter)((Worklist.size() <= MaxIter) ? static_cast<void> (0 ) : __assert_fail ("Worklist.size() <= MaxIter", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1085, __PRETTY_FUNCTION__)); | |||
1086 | const Use *U = Worklist.pop_back_val(); | |||
1087 | const Value *V = U->getUser(); | |||
1088 | --MaxIter; | |||
1089 | ||||
1090 | if (isa<BitCastInst>(V) || isa<GetElementPtrInst>(V) || isa<PHINode>(V) || | |||
1091 | isa<SelectInst>(V)) { | |||
1092 | // Track the uses. | |||
1093 | } else if (isa<LoadInst>(V)) { | |||
1094 | // Loading from the pointer doesn't escape it. | |||
1095 | continue; | |||
1096 | } else if (const auto *SI = dyn_cast<StoreInst>(V)) { | |||
1097 | // Storing *to* the pointer is fine, but storing the pointer escapes it. | |||
1098 | if (SI->getValueOperand() == U->get()) | |||
1099 | return nullptr; | |||
1100 | continue; | |||
1101 | } else if (isa<ICmpInst>(V)) { | |||
1102 | if (NumCmps++) | |||
1103 | return nullptr; // Found more than one cmp. | |||
1104 | continue; | |||
1105 | } else if (const auto *Intrin = dyn_cast<IntrinsicInst>(V)) { | |||
1106 | switch (Intrin->getIntrinsicID()) { | |||
1107 | // These intrinsics don't escape or compare the pointer. Memset is safe | |||
1108 | // because we don't allow ptrtoint. Memcpy and memmove are safe because | |||
1109 | // we don't allow stores, so src cannot point to V. | |||
1110 | case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: | |||
1111 | case Intrinsic::dbg_declare: case Intrinsic::dbg_value: | |||
1112 | case Intrinsic::memcpy: case Intrinsic::memmove: case Intrinsic::memset: | |||
1113 | continue; | |||
1114 | default: | |||
1115 | return nullptr; | |||
1116 | } | |||
1117 | } else { | |||
1118 | return nullptr; | |||
1119 | } | |||
1120 | for (const Use &U : V->uses()) { | |||
1121 | if (Worklist.size() >= MaxIter) | |||
1122 | return nullptr; | |||
1123 | Worklist.push_back(&U); | |||
1124 | } | |||
1125 | } | |||
1126 | ||||
1127 | Type *CmpTy = CmpInst::makeCmpResultType(Other->getType()); | |||
1128 | return replaceInstUsesWith( | |||
1129 | ICI, | |||
1130 | ConstantInt::get(CmpTy, !CmpInst::isTrueWhenEqual(ICI.getPredicate()))); | |||
1131 | } | |||
1132 | ||||
1133 | /// Fold "icmp pred (X+CI), X". | |||
1134 | Instruction *InstCombiner::foldICmpAddOpConst(Instruction &ICI, | |||
1135 | Value *X, ConstantInt *CI, | |||
1136 | ICmpInst::Predicate Pred) { | |||
1137 | // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0, | |||
1138 | // so the values can never be equal. Similarly for all other "or equals" | |||
1139 | // operators. | |||
1140 | ||||
1141 | // (X+1) <u X --> X >u (MAXUINT-1) --> X == 255 | |||
1142 | // (X+2) <u X --> X >u (MAXUINT-2) --> X > 253 | |||
1143 | // (X+MAXUINT) <u X --> X >u (MAXUINT-MAXUINT) --> X != 0 | |||
1144 | if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) { | |||
1145 | Value *R = | |||
1146 | ConstantExpr::getSub(ConstantInt::getAllOnesValue(CI->getType()), CI); | |||
1147 | return new ICmpInst(ICmpInst::ICMP_UGT, X, R); | |||
1148 | } | |||
1149 | ||||
1150 | // (X+1) >u X --> X <u (0-1) --> X != 255 | |||
1151 | // (X+2) >u X --> X <u (0-2) --> X <u 254 | |||
1152 | // (X+MAXUINT) >u X --> X <u (0-MAXUINT) --> X <u 1 --> X == 0 | |||
1153 | if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) | |||
1154 | return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantExpr::getNeg(CI)); | |||
1155 | ||||
1156 | unsigned BitWidth = CI->getType()->getPrimitiveSizeInBits(); | |||
1157 | ConstantInt *SMax = ConstantInt::get(X->getContext(), | |||
1158 | APInt::getSignedMaxValue(BitWidth)); | |||
1159 | ||||
1160 | // (X+ 1) <s X --> X >s (MAXSINT-1) --> X == 127 | |||
1161 | // (X+ 2) <s X --> X >s (MAXSINT-2) --> X >s 125 | |||
1162 | // (X+MAXSINT) <s X --> X >s (MAXSINT-MAXSINT) --> X >s 0 | |||
1163 | // (X+MINSINT) <s X --> X >s (MAXSINT-MINSINT) --> X >s -1 | |||
1164 | // (X+ -2) <s X --> X >s (MAXSINT- -2) --> X >s 126 | |||
1165 | // (X+ -1) <s X --> X >s (MAXSINT- -1) --> X != 127 | |||
1166 | if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) | |||
1167 | return new ICmpInst(ICmpInst::ICMP_SGT, X, ConstantExpr::getSub(SMax, CI)); | |||
1168 | ||||
1169 | // (X+ 1) >s X --> X <s (MAXSINT-(1-1)) --> X != 127 | |||
1170 | // (X+ 2) >s X --> X <s (MAXSINT-(2-1)) --> X <s 126 | |||
1171 | // (X+MAXSINT) >s X --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1 | |||
1172 | // (X+MINSINT) >s X --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2 | |||
1173 | // (X+ -2) >s X --> X <s (MAXSINT-(-2-1)) --> X <s -126 | |||
1174 | // (X+ -1) >s X --> X <s (MAXSINT-(-1-1)) --> X == -128 | |||
1175 | ||||
1176 | assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE)((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) ? static_cast<void> (0) : __assert_fail ("Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1176, __PRETTY_FUNCTION__)); | |||
1177 | Constant *C = Builder->getInt(CI->getValue()-1); | |||
1178 | return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantExpr::getSub(SMax, C)); | |||
1179 | } | |||
1180 | ||||
1181 | /// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" -> | |||
1182 | /// (icmp eq/ne A, Log2(AP2/AP1)) -> | |||
1183 | /// (icmp eq/ne A, Log2(AP2) - Log2(AP1)). | |||
1184 | Instruction *InstCombiner::foldICmpShrConstConst(ICmpInst &I, Value *A, | |||
1185 | const APInt &AP1, | |||
1186 | const APInt &AP2) { | |||
1187 | assert(I.isEquality() && "Cannot fold icmp gt/lt")((I.isEquality() && "Cannot fold icmp gt/lt") ? static_cast <void> (0) : __assert_fail ("I.isEquality() && \"Cannot fold icmp gt/lt\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1187, __PRETTY_FUNCTION__)); | |||
1188 | ||||
1189 | auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { | |||
1190 | if (I.getPredicate() == I.ICMP_NE) | |||
1191 | Pred = CmpInst::getInversePredicate(Pred); | |||
1192 | return new ICmpInst(Pred, LHS, RHS); | |||
1193 | }; | |||
1194 | ||||
1195 | // Don't bother doing any work for cases which InstSimplify handles. | |||
1196 | if (AP2 == 0) | |||
1197 | return nullptr; | |||
1198 | ||||
1199 | bool IsAShr = isa<AShrOperator>(I.getOperand(0)); | |||
1200 | if (IsAShr) { | |||
1201 | if (AP2.isAllOnesValue()) | |||
1202 | return nullptr; | |||
1203 | if (AP2.isNegative() != AP1.isNegative()) | |||
1204 | return nullptr; | |||
1205 | if (AP2.sgt(AP1)) | |||
1206 | return nullptr; | |||
1207 | } | |||
1208 | ||||
1209 | if (!AP1) | |||
1210 | // 'A' must be large enough to shift out the highest set bit. | |||
1211 | return getICmp(I.ICMP_UGT, A, | |||
1212 | ConstantInt::get(A->getType(), AP2.logBase2())); | |||
1213 | ||||
1214 | if (AP1 == AP2) | |||
1215 | return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType())); | |||
1216 | ||||
1217 | int Shift; | |||
1218 | if (IsAShr && AP1.isNegative()) | |||
1219 | Shift = AP1.countLeadingOnes() - AP2.countLeadingOnes(); | |||
1220 | else | |||
1221 | Shift = AP1.countLeadingZeros() - AP2.countLeadingZeros(); | |||
1222 | ||||
1223 | if (Shift > 0) { | |||
1224 | if (IsAShr && AP1 == AP2.ashr(Shift)) { | |||
1225 | // There are multiple solutions if we are comparing against -1 and the LHS | |||
1226 | // of the ashr is not a power of two. | |||
1227 | if (AP1.isAllOnesValue() && !AP2.isPowerOf2()) | |||
1228 | return getICmp(I.ICMP_UGE, A, ConstantInt::get(A->getType(), Shift)); | |||
1229 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); | |||
1230 | } else if (AP1 == AP2.lshr(Shift)) { | |||
1231 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); | |||
1232 | } | |||
1233 | } | |||
1234 | ||||
1235 | // Shifting const2 will never be equal to const1. | |||
1236 | // FIXME: This should always be handled by InstSimplify? | |||
1237 | auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE); | |||
1238 | return replaceInstUsesWith(I, TorF); | |||
1239 | } | |||
1240 | ||||
1241 | /// Handle "(icmp eq/ne (shl AP2, A), AP1)" -> | |||
1242 | /// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)). | |||
1243 | Instruction *InstCombiner::foldICmpShlConstConst(ICmpInst &I, Value *A, | |||
1244 | const APInt &AP1, | |||
1245 | const APInt &AP2) { | |||
1246 | assert(I.isEquality() && "Cannot fold icmp gt/lt")((I.isEquality() && "Cannot fold icmp gt/lt") ? static_cast <void> (0) : __assert_fail ("I.isEquality() && \"Cannot fold icmp gt/lt\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1246, __PRETTY_FUNCTION__)); | |||
1247 | ||||
1248 | auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { | |||
1249 | if (I.getPredicate() == I.ICMP_NE) | |||
1250 | Pred = CmpInst::getInversePredicate(Pred); | |||
1251 | return new ICmpInst(Pred, LHS, RHS); | |||
1252 | }; | |||
1253 | ||||
1254 | // Don't bother doing any work for cases which InstSimplify handles. | |||
1255 | if (AP2 == 0) | |||
1256 | return nullptr; | |||
1257 | ||||
1258 | unsigned AP2TrailingZeros = AP2.countTrailingZeros(); | |||
1259 | ||||
1260 | if (!AP1 && AP2TrailingZeros != 0) | |||
1261 | return getICmp( | |||
1262 | I.ICMP_UGE, A, | |||
1263 | ConstantInt::get(A->getType(), AP2.getBitWidth() - AP2TrailingZeros)); | |||
1264 | ||||
1265 | if (AP1 == AP2) | |||
1266 | return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType())); | |||
1267 | ||||
1268 | // Get the distance between the lowest bits that are set. | |||
1269 | int Shift = AP1.countTrailingZeros() - AP2TrailingZeros; | |||
1270 | ||||
1271 | if (Shift > 0 && AP2.shl(Shift) == AP1) | |||
1272 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); | |||
1273 | ||||
1274 | // Shifting const2 will never be equal to const1. | |||
1275 | // FIXME: This should always be handled by InstSimplify? | |||
1276 | auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE); | |||
1277 | return replaceInstUsesWith(I, TorF); | |||
1278 | } | |||
1279 | ||||
1280 | /// The caller has matched a pattern of the form: | |||
1281 | /// I = icmp ugt (add (add A, B), CI2), CI1 | |||
1282 | /// If this is of the form: | |||
1283 | /// sum = a + b | |||
1284 | /// if (sum+128 >u 255) | |||
1285 | /// Then replace it with llvm.sadd.with.overflow.i8. | |||
1286 | /// | |||
1287 | static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B, | |||
1288 | ConstantInt *CI2, ConstantInt *CI1, | |||
1289 | InstCombiner &IC) { | |||
1290 | // The transformation we're trying to do here is to transform this into an | |||
1291 | // llvm.sadd.with.overflow. To do this, we have to replace the original add | |||
1292 | // with a narrower add, and discard the add-with-constant that is part of the | |||
1293 | // range check (if we can't eliminate it, this isn't profitable). | |||
1294 | ||||
1295 | // In order to eliminate the add-with-constant, the compare can be its only | |||
1296 | // use. | |||
1297 | Instruction *AddWithCst = cast<Instruction>(I.getOperand(0)); | |||
1298 | if (!AddWithCst->hasOneUse()) | |||
1299 | return nullptr; | |||
1300 | ||||
1301 | // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow. | |||
1302 | if (!CI2->getValue().isPowerOf2()) | |||
1303 | return nullptr; | |||
1304 | unsigned NewWidth = CI2->getValue().countTrailingZeros(); | |||
1305 | if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31) | |||
1306 | return nullptr; | |||
1307 | ||||
1308 | // The width of the new add formed is 1 more than the bias. | |||
1309 | ++NewWidth; | |||
1310 | ||||
1311 | // Check to see that CI1 is an all-ones value with NewWidth bits. | |||
1312 | if (CI1->getBitWidth() == NewWidth || | |||
1313 | CI1->getValue() != APInt::getLowBitsSet(CI1->getBitWidth(), NewWidth)) | |||
1314 | return nullptr; | |||
1315 | ||||
1316 | // This is only really a signed overflow check if the inputs have been | |||
1317 | // sign-extended; check for that condition. For example, if CI2 is 2^31 and | |||
1318 | // the operands of the add are 64 bits wide, we need at least 33 sign bits. | |||
1319 | unsigned NeededSignBits = CI1->getBitWidth() - NewWidth + 1; | |||
1320 | if (IC.ComputeNumSignBits(A, 0, &I) < NeededSignBits || | |||
1321 | IC.ComputeNumSignBits(B, 0, &I) < NeededSignBits) | |||
1322 | return nullptr; | |||
1323 | ||||
1324 | // In order to replace the original add with a narrower | |||
1325 | // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant | |||
1326 | // and truncates that discard the high bits of the add. Verify that this is | |||
1327 | // the case. | |||
1328 | Instruction *OrigAdd = cast<Instruction>(AddWithCst->getOperand(0)); | |||
1329 | for (User *U : OrigAdd->users()) { | |||
1330 | if (U == AddWithCst) | |||
1331 | continue; | |||
1332 | ||||
1333 | // Only accept truncates for now. We would really like a nice recursive | |||
1334 | // predicate like SimplifyDemandedBits, but which goes downwards the use-def | |||
1335 | // chain to see which bits of a value are actually demanded. If the | |||
1336 | // original add had another add which was then immediately truncated, we | |||
1337 | // could still do the transformation. | |||
1338 | TruncInst *TI = dyn_cast<TruncInst>(U); | |||
1339 | if (!TI || TI->getType()->getPrimitiveSizeInBits() > NewWidth) | |||
1340 | return nullptr; | |||
1341 | } | |||
1342 | ||||
1343 | // If the pattern matches, truncate the inputs to the narrower type and | |||
1344 | // use the sadd_with_overflow intrinsic to efficiently compute both the | |||
1345 | // result and the overflow bit. | |||
1346 | Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth); | |||
1347 | Value *F = Intrinsic::getDeclaration(I.getModule(), | |||
1348 | Intrinsic::sadd_with_overflow, NewType); | |||
1349 | ||||
1350 | InstCombiner::BuilderTy *Builder = IC.Builder; | |||
1351 | ||||
1352 | // Put the new code above the original add, in case there are any uses of the | |||
1353 | // add between the add and the compare. | |||
1354 | Builder->SetInsertPoint(OrigAdd); | |||
1355 | ||||
1356 | Value *TruncA = Builder->CreateTrunc(A, NewType, A->getName() + ".trunc"); | |||
1357 | Value *TruncB = Builder->CreateTrunc(B, NewType, B->getName() + ".trunc"); | |||
1358 | CallInst *Call = Builder->CreateCall(F, {TruncA, TruncB}, "sadd"); | |||
1359 | Value *Add = Builder->CreateExtractValue(Call, 0, "sadd.result"); | |||
1360 | Value *ZExt = Builder->CreateZExt(Add, OrigAdd->getType()); | |||
1361 | ||||
1362 | // The inner add was the result of the narrow add, zero extended to the | |||
1363 | // wider type. Replace it with the result computed by the intrinsic. | |||
1364 | IC.replaceInstUsesWith(*OrigAdd, ZExt); | |||
1365 | ||||
1366 | // The original icmp gets replaced with the overflow value. | |||
1367 | return ExtractValueInst::Create(Call, 1, "sadd.overflow"); | |||
1368 | } | |||
1369 | ||||
1370 | // Fold icmp Pred X, C. | |||
1371 | Instruction *InstCombiner::foldICmpWithConstant(ICmpInst &Cmp) { | |||
1372 | CmpInst::Predicate Pred = Cmp.getPredicate(); | |||
1373 | Value *X = Cmp.getOperand(0); | |||
1374 | ||||
1375 | const APInt *C; | |||
1376 | if (!match(Cmp.getOperand(1), m_APInt(C))) | |||
1377 | return nullptr; | |||
1378 | ||||
1379 | Value *A = nullptr, *B = nullptr; | |||
1380 | ||||
1381 | // Match the following pattern, which is a common idiom when writing | |||
1382 | // overflow-safe integer arithmetic functions. The source performs an addition | |||
1383 | // in wider type and explicitly checks for overflow using comparisons against | |||
1384 | // INT_MIN and INT_MAX. Simplify by using the sadd_with_overflow intrinsic. | |||
1385 | // | |||
1386 | // TODO: This could probably be generalized to handle other overflow-safe | |||
1387 | // operations if we worked out the formulas to compute the appropriate magic | |||
1388 | // constants. | |||
1389 | // | |||
1390 | // sum = a + b | |||
1391 | // if (sum+128 >u 255) ... -> llvm.sadd.with.overflow.i8 | |||
1392 | { | |||
1393 | ConstantInt *CI2; // I = icmp ugt (add (add A, B), CI2), CI | |||
1394 | if (Pred == ICmpInst::ICMP_UGT && | |||
1395 | match(X, m_Add(m_Add(m_Value(A), m_Value(B)), m_ConstantInt(CI2)))) | |||
1396 | if (Instruction *Res = processUGT_ADDCST_ADD( | |||
1397 | Cmp, A, B, CI2, cast<ConstantInt>(Cmp.getOperand(1)), *this)) | |||
1398 | return Res; | |||
1399 | } | |||
1400 | ||||
1401 | // (icmp sgt smin(PosA, B) 0) -> (icmp sgt B 0) | |||
1402 | if (*C == 0 && Pred == ICmpInst::ICMP_SGT) { | |||
1403 | SelectPatternResult SPR = matchSelectPattern(X, A, B); | |||
1404 | if (SPR.Flavor == SPF_SMIN) { | |||
1405 | if (isKnownPositive(A, DL)) | |||
1406 | return new ICmpInst(Pred, B, Cmp.getOperand(1)); | |||
1407 | if (isKnownPositive(B, DL)) | |||
1408 | return new ICmpInst(Pred, A, Cmp.getOperand(1)); | |||
1409 | } | |||
1410 | } | |||
1411 | ||||
1412 | // FIXME: Use m_APInt to allow folds for splat constants. | |||
1413 | ConstantInt *CI = dyn_cast<ConstantInt>(Cmp.getOperand(1)); | |||
1414 | if (!CI) | |||
1415 | return nullptr; | |||
1416 | ||||
1417 | // Canonicalize icmp instructions based on dominating conditions. | |||
1418 | BasicBlock *Parent = Cmp.getParent(); | |||
1419 | BasicBlock *Dom = Parent->getSinglePredecessor(); | |||
1420 | auto *BI = Dom ? dyn_cast<BranchInst>(Dom->getTerminator()) : nullptr; | |||
1421 | ICmpInst::Predicate Pred2; | |||
1422 | BasicBlock *TrueBB, *FalseBB; | |||
1423 | ConstantInt *CI2; | |||
1424 | if (BI && match(BI, m_Br(m_ICmp(Pred2, m_Specific(X), m_ConstantInt(CI2)), | |||
1425 | TrueBB, FalseBB)) && | |||
1426 | TrueBB != FalseBB) { | |||
1427 | ConstantRange CR = | |||
1428 | ConstantRange::makeAllowedICmpRegion(Pred, CI->getValue()); | |||
1429 | ConstantRange DominatingCR = | |||
1430 | (Parent == TrueBB) | |||
1431 | ? ConstantRange::makeExactICmpRegion(Pred2, CI2->getValue()) | |||
1432 | : ConstantRange::makeExactICmpRegion( | |||
1433 | CmpInst::getInversePredicate(Pred2), CI2->getValue()); | |||
1434 | ConstantRange Intersection = DominatingCR.intersectWith(CR); | |||
1435 | ConstantRange Difference = DominatingCR.difference(CR); | |||
1436 | if (Intersection.isEmptySet()) | |||
1437 | return replaceInstUsesWith(Cmp, Builder->getFalse()); | |||
1438 | if (Difference.isEmptySet()) | |||
1439 | return replaceInstUsesWith(Cmp, Builder->getTrue()); | |||
1440 | ||||
1441 | // If this is a normal comparison, it demands all bits. If it is a sign | |||
1442 | // bit comparison, it only demands the sign bit. | |||
1443 | bool UnusedBit; | |||
1444 | bool IsSignBit = isSignBitCheck(Pred, CI->getValue(), UnusedBit); | |||
1445 | ||||
1446 | // Canonicalizing a sign bit comparison that gets used in a branch, | |||
1447 | // pessimizes codegen by generating branch on zero instruction instead | |||
1448 | // of a test and branch. So we avoid canonicalizing in such situations | |||
1449 | // because test and branch instruction has better branch displacement | |||
1450 | // than compare and branch instruction. | |||
1451 | if (!isBranchOnSignBitCheck(Cmp, IsSignBit) && !Cmp.isEquality()) { | |||
1452 | if (auto *AI = Intersection.getSingleElement()) | |||
1453 | return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder->getInt(*AI)); | |||
1454 | if (auto *AD = Difference.getSingleElement()) | |||
1455 | return new ICmpInst(ICmpInst::ICMP_NE, X, Builder->getInt(*AD)); | |||
1456 | } | |||
1457 | } | |||
1458 | ||||
1459 | return nullptr; | |||
1460 | } | |||
1461 | ||||
1462 | /// Fold icmp (trunc X, Y), C. | |||
1463 | Instruction *InstCombiner::foldICmpTruncConstant(ICmpInst &Cmp, | |||
1464 | Instruction *Trunc, | |||
1465 | const APInt *C) { | |||
1466 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | |||
1467 | Value *X = Trunc->getOperand(0); | |||
1468 | if (*C == 1 && C->getBitWidth() > 1) { | |||
1469 | // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1 | |||
1470 | Value *V = nullptr; | |||
1471 | if (Pred == ICmpInst::ICMP_SLT && match(X, m_Signum(m_Value(V)))) | |||
1472 | return new ICmpInst(ICmpInst::ICMP_SLT, V, | |||
1473 | ConstantInt::get(V->getType(), 1)); | |||
1474 | } | |||
1475 | ||||
1476 | if (Cmp.isEquality() && Trunc->hasOneUse()) { | |||
1477 | // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all | |||
1478 | // of the high bits truncated out of x are known. | |||
1479 | unsigned DstBits = Trunc->getType()->getScalarSizeInBits(), | |||
1480 | SrcBits = X->getType()->getScalarSizeInBits(); | |||
1481 | KnownBits Known(SrcBits); | |||
1482 | computeKnownBits(X, Known, 0, &Cmp); | |||
1483 | ||||
1484 | // If all the high bits are known, we can do this xform. | |||
1485 | if ((Known.Zero | Known.One).countLeadingOnes() >= SrcBits - DstBits) { | |||
1486 | // Pull in the high bits from known-ones set. | |||
1487 | APInt NewRHS = C->zext(SrcBits); | |||
1488 | NewRHS |= Known.One & APInt::getHighBitsSet(SrcBits, SrcBits - DstBits); | |||
1489 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), NewRHS)); | |||
1490 | } | |||
1491 | } | |||
1492 | ||||
1493 | return nullptr; | |||
1494 | } | |||
1495 | ||||
1496 | /// Fold icmp (xor X, Y), C. | |||
1497 | Instruction *InstCombiner::foldICmpXorConstant(ICmpInst &Cmp, | |||
1498 | BinaryOperator *Xor, | |||
1499 | const APInt *C) { | |||
1500 | Value *X = Xor->getOperand(0); | |||
1501 | Value *Y = Xor->getOperand(1); | |||
1502 | const APInt *XorC; | |||
1503 | if (!match(Y, m_APInt(XorC))) | |||
1504 | return nullptr; | |||
1505 | ||||
1506 | // If this is a comparison that tests the signbit (X < 0) or (x > -1), | |||
1507 | // fold the xor. | |||
1508 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | |||
1509 | if ((Pred == ICmpInst::ICMP_SLT && *C == 0) || | |||
1510 | (Pred == ICmpInst::ICMP_SGT && C->isAllOnesValue())) { | |||
1511 | ||||
1512 | // If the sign bit of the XorCst is not set, there is no change to | |||
1513 | // the operation, just stop using the Xor. | |||
1514 | if (!XorC->isNegative()) { | |||
1515 | Cmp.setOperand(0, X); | |||
1516 | Worklist.Add(Xor); | |||
1517 | return &Cmp; | |||
1518 | } | |||
1519 | ||||
1520 | // Was the old condition true if the operand is positive? | |||
1521 | bool isTrueIfPositive = Pred == ICmpInst::ICMP_SGT; | |||
1522 | ||||
1523 | // If so, the new one isn't. | |||
1524 | isTrueIfPositive ^= true; | |||
1525 | ||||
1526 | Constant *CmpConstant = cast<Constant>(Cmp.getOperand(1)); | |||
1527 | if (isTrueIfPositive) | |||
1528 | return new ICmpInst(ICmpInst::ICMP_SGT, X, SubOne(CmpConstant)); | |||
1529 | else | |||
1530 | return new ICmpInst(ICmpInst::ICMP_SLT, X, AddOne(CmpConstant)); | |||
1531 | } | |||
1532 | ||||
1533 | if (Xor->hasOneUse()) { | |||
1534 | // (icmp u/s (xor X SignMask), C) -> (icmp s/u X, (xor C SignMask)) | |||
1535 | if (!Cmp.isEquality() && XorC->isSignMask()) { | |||
1536 | Pred = Cmp.isSigned() ? Cmp.getUnsignedPredicate() | |||
1537 | : Cmp.getSignedPredicate(); | |||
1538 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), *C ^ *XorC)); | |||
1539 | } | |||
1540 | ||||
1541 | // (icmp u/s (xor X ~SignMask), C) -> (icmp s/u X, (xor C ~SignMask)) | |||
1542 | if (!Cmp.isEquality() && XorC->isMaxSignedValue()) { | |||
1543 | Pred = Cmp.isSigned() ? Cmp.getUnsignedPredicate() | |||
1544 | : Cmp.getSignedPredicate(); | |||
1545 | Pred = Cmp.getSwappedPredicate(Pred); | |||
1546 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), *C ^ *XorC)); | |||
1547 | } | |||
1548 | } | |||
1549 | ||||
1550 | // (icmp ugt (xor X, C), ~C) -> (icmp ult X, C) | |||
1551 | // iff -C is a power of 2 | |||
1552 | if (Pred == ICmpInst::ICMP_UGT && *XorC == ~(*C) && (*C + 1).isPowerOf2()) | |||
1553 | return new ICmpInst(ICmpInst::ICMP_ULT, X, Y); | |||
1554 | ||||
1555 | // (icmp ult (xor X, C), -C) -> (icmp uge X, C) | |||
1556 | // iff -C is a power of 2 | |||
1557 | if (Pred == ICmpInst::ICMP_ULT && *XorC == -(*C) && C->isPowerOf2()) | |||
1558 | return new ICmpInst(ICmpInst::ICMP_UGE, X, Y); | |||
1559 | ||||
1560 | return nullptr; | |||
1561 | } | |||
1562 | ||||
1563 | /// Fold icmp (and (sh X, Y), C2), C1. | |||
1564 | Instruction *InstCombiner::foldICmpAndShift(ICmpInst &Cmp, BinaryOperator *And, | |||
1565 | const APInt *C1, const APInt *C2) { | |||
1566 | BinaryOperator *Shift = dyn_cast<BinaryOperator>(And->getOperand(0)); | |||
1567 | if (!Shift || !Shift->isShift()) | |||
1568 | return nullptr; | |||
1569 | ||||
1570 | // If this is: (X >> C3) & C2 != C1 (where any shift and any compare could | |||
1571 | // exist), turn it into (X & (C2 << C3)) != (C1 << C3). This happens a LOT in | |||
1572 | // code produced by the clang front-end, for bitfield access. | |||
1573 | // This seemingly simple opportunity to fold away a shift turns out to be | |||
1574 | // rather complicated. See PR17827 for details. | |||
1575 | unsigned ShiftOpcode = Shift->getOpcode(); | |||
1576 | bool IsShl = ShiftOpcode == Instruction::Shl; | |||
1577 | const APInt *C3; | |||
1578 | if (match(Shift->getOperand(1), m_APInt(C3))) { | |||
1579 | bool CanFold = false; | |||
1580 | if (ShiftOpcode == Instruction::AShr) { | |||
1581 | // There may be some constraints that make this possible, but nothing | |||
1582 | // simple has been discovered yet. | |||
1583 | CanFold = false; | |||
1584 | } else if (ShiftOpcode == Instruction::Shl) { | |||
1585 | // For a left shift, we can fold if the comparison is not signed. We can | |||
1586 | // also fold a signed comparison if the mask value and comparison value | |||
1587 | // are not negative. These constraints may not be obvious, but we can | |||
1588 | // prove that they are correct using an SMT solver. | |||
1589 | if (!Cmp.isSigned() || (!C2->isNegative() && !C1->isNegative())) | |||
1590 | CanFold = true; | |||
1591 | } else if (ShiftOpcode == Instruction::LShr) { | |||
1592 | // For a logical right shift, we can fold if the comparison is not signed. | |||
1593 | // We can also fold a signed comparison if the shifted mask value and the | |||
1594 | // shifted comparison value are not negative. These constraints may not be | |||
1595 | // obvious, but we can prove that they are correct using an SMT solver. | |||
1596 | if (!Cmp.isSigned() || | |||
1597 | (!C2->shl(*C3).isNegative() && !C1->shl(*C3).isNegative())) | |||
1598 | CanFold = true; | |||
1599 | } | |||
1600 | ||||
1601 | if (CanFold) { | |||
1602 | APInt NewCst = IsShl ? C1->lshr(*C3) : C1->shl(*C3); | |||
1603 | APInt SameAsC1 = IsShl ? NewCst.shl(*C3) : NewCst.lshr(*C3); | |||
1604 | // Check to see if we are shifting out any of the bits being compared. | |||
1605 | if (SameAsC1 != *C1) { | |||
1606 | // If we shifted bits out, the fold is not going to work out. As a | |||
1607 | // special case, check to see if this means that the result is always | |||
1608 | // true or false now. | |||
1609 | if (Cmp.getPredicate() == ICmpInst::ICMP_EQ) | |||
1610 | return replaceInstUsesWith(Cmp, ConstantInt::getFalse(Cmp.getType())); | |||
1611 | if (Cmp.getPredicate() == ICmpInst::ICMP_NE) | |||
1612 | return replaceInstUsesWith(Cmp, ConstantInt::getTrue(Cmp.getType())); | |||
1613 | } else { | |||
1614 | Cmp.setOperand(1, ConstantInt::get(And->getType(), NewCst)); | |||
1615 | APInt NewAndCst = IsShl ? C2->lshr(*C3) : C2->shl(*C3); | |||
1616 | And->setOperand(1, ConstantInt::get(And->getType(), NewAndCst)); | |||
1617 | And->setOperand(0, Shift->getOperand(0)); | |||
1618 | Worklist.Add(Shift); // Shift is dead. | |||
1619 | return &Cmp; | |||
1620 | } | |||
1621 | } | |||
1622 | } | |||
1623 | ||||
1624 | // Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is | |||
1625 | // preferable because it allows the C2 << Y expression to be hoisted out of a | |||
1626 | // loop if Y is invariant and X is not. | |||
1627 | if (Shift->hasOneUse() && *C1 == 0 && Cmp.isEquality() && | |||
1628 | !Shift->isArithmeticShift() && !isa<Constant>(Shift->getOperand(0))) { | |||
1629 | // Compute C2 << Y. | |||
1630 | Value *NewShift = | |||
1631 | IsShl ? Builder->CreateLShr(And->getOperand(1), Shift->getOperand(1)) | |||
1632 | : Builder->CreateShl(And->getOperand(1), Shift->getOperand(1)); | |||
1633 | ||||
1634 | // Compute X & (C2 << Y). | |||
1635 | Value *NewAnd = Builder->CreateAnd(Shift->getOperand(0), NewShift); | |||
1636 | Cmp.setOperand(0, NewAnd); | |||
1637 | return &Cmp; | |||
1638 | } | |||
1639 | ||||
1640 | return nullptr; | |||
1641 | } | |||
1642 | ||||
1643 | /// Fold icmp (and X, C2), C1. | |||
1644 | Instruction *InstCombiner::foldICmpAndConstConst(ICmpInst &Cmp, | |||
1645 | BinaryOperator *And, | |||
1646 | const APInt *C1) { | |||
1647 | const APInt *C2; | |||
1648 | if (!match(And->getOperand(1), m_APInt(C2))) | |||
1649 | return nullptr; | |||
1650 | ||||
1651 | if (!And->hasOneUse() || !And->getOperand(0)->hasOneUse()) | |||
1652 | return nullptr; | |||
1653 | ||||
1654 | // If the LHS is an 'and' of a truncate and we can widen the and/compare to | |||
1655 | // the input width without changing the value produced, eliminate the cast: | |||
1656 | // | |||
1657 | // icmp (and (trunc W), C2), C1 -> icmp (and W, C2'), C1' | |||
1658 | // | |||
1659 | // We can do this transformation if the constants do not have their sign bits | |||
1660 | // set or if it is an equality comparison. Extending a relational comparison | |||
1661 | // when we're checking the sign bit would not work. | |||
1662 | Value *W; | |||
1663 | if (match(And->getOperand(0), m_Trunc(m_Value(W))) && | |||
1664 | (Cmp.isEquality() || (!C1->isNegative() && !C2->isNegative()))) { | |||
1665 | // TODO: Is this a good transform for vectors? Wider types may reduce | |||
1666 | // throughput. Should this transform be limited (even for scalars) by using | |||
1667 | // shouldChangeType()? | |||
1668 | if (!Cmp.getType()->isVectorTy()) { | |||
1669 | Type *WideType = W->getType(); | |||
1670 | unsigned WideScalarBits = WideType->getScalarSizeInBits(); | |||
1671 | Constant *ZextC1 = ConstantInt::get(WideType, C1->zext(WideScalarBits)); | |||
1672 | Constant *ZextC2 = ConstantInt::get(WideType, C2->zext(WideScalarBits)); | |||
1673 | Value *NewAnd = Builder->CreateAnd(W, ZextC2, And->getName()); | |||
1674 | return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1); | |||
1675 | } | |||
1676 | } | |||
1677 | ||||
1678 | if (Instruction *I = foldICmpAndShift(Cmp, And, C1, C2)) | |||
1679 | return I; | |||
1680 | ||||
1681 | // (icmp pred (and (or (lshr A, B), A), 1), 0) --> | |||
1682 | // (icmp pred (and A, (or (shl 1, B), 1), 0)) | |||
1683 | // | |||
1684 | // iff pred isn't signed | |||
1685 | if (!Cmp.isSigned() && *C1 == 0 && match(And->getOperand(1), m_One())) { | |||
1686 | Constant *One = cast<Constant>(And->getOperand(1)); | |||
1687 | Value *Or = And->getOperand(0); | |||
1688 | Value *A, *B, *LShr; | |||
1689 | if (match(Or, m_Or(m_Value(LShr), m_Value(A))) && | |||
1690 | match(LShr, m_LShr(m_Specific(A), m_Value(B)))) { | |||
1691 | unsigned UsesRemoved = 0; | |||
1692 | if (And->hasOneUse()) | |||
1693 | ++UsesRemoved; | |||
1694 | if (Or->hasOneUse()) | |||
1695 | ++UsesRemoved; | |||
1696 | if (LShr->hasOneUse()) | |||
1697 | ++UsesRemoved; | |||
1698 | ||||
1699 | // Compute A & ((1 << B) | 1) | |||
1700 | Value *NewOr = nullptr; | |||
1701 | if (auto *C = dyn_cast<Constant>(B)) { | |||
1702 | if (UsesRemoved >= 1) | |||
1703 | NewOr = ConstantExpr::getOr(ConstantExpr::getNUWShl(One, C), One); | |||
1704 | } else { | |||
1705 | if (UsesRemoved >= 3) | |||
1706 | NewOr = Builder->CreateOr(Builder->CreateShl(One, B, LShr->getName(), | |||
1707 | /*HasNUW=*/true), | |||
1708 | One, Or->getName()); | |||
1709 | } | |||
1710 | if (NewOr) { | |||
1711 | Value *NewAnd = Builder->CreateAnd(A, NewOr, And->getName()); | |||
1712 | Cmp.setOperand(0, NewAnd); | |||
1713 | return &Cmp; | |||
1714 | } | |||
1715 | } | |||
1716 | } | |||
1717 | ||||
1718 | // (X & C2) > C1 --> (X & C2) != 0, if any bit set in (X & C2) will produce a | |||
1719 | // result greater than C1. | |||
1720 | unsigned NumTZ = C2->countTrailingZeros(); | |||
1721 | if (Cmp.getPredicate() == ICmpInst::ICMP_UGT && NumTZ < C2->getBitWidth() && | |||
1722 | APInt::getOneBitSet(C2->getBitWidth(), NumTZ).ugt(*C1)) { | |||
1723 | Constant *Zero = Constant::getNullValue(And->getType()); | |||
1724 | return new ICmpInst(ICmpInst::ICMP_NE, And, Zero); | |||
1725 | } | |||
1726 | ||||
1727 | return nullptr; | |||
1728 | } | |||
1729 | ||||
1730 | /// Fold icmp (and X, Y), C. | |||
1731 | Instruction *InstCombiner::foldICmpAndConstant(ICmpInst &Cmp, | |||
1732 | BinaryOperator *And, | |||
1733 | const APInt *C) { | |||
1734 | if (Instruction *I = foldICmpAndConstConst(Cmp, And, C)) | |||
1735 | return I; | |||
1736 | ||||
1737 | // TODO: These all require that Y is constant too, so refactor with the above. | |||
1738 | ||||
1739 | // Try to optimize things like "A[i] & 42 == 0" to index computations. | |||
1740 | Value *X = And->getOperand(0); | |||
1741 | Value *Y = And->getOperand(1); | |||
1742 | if (auto *LI = dyn_cast<LoadInst>(X)) | |||
1743 | if (auto *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0))) | |||
1744 | if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) | |||
1745 | if (GV->isConstant() && GV->hasDefinitiveInitializer() && | |||
1746 | !LI->isVolatile() && isa<ConstantInt>(Y)) { | |||
1747 | ConstantInt *C2 = cast<ConstantInt>(Y); | |||
1748 | if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, Cmp, C2)) | |||
1749 | return Res; | |||
1750 | } | |||
1751 | ||||
1752 | if (!Cmp.isEquality()) | |||
1753 | return nullptr; | |||
1754 | ||||
1755 | // X & -C == -C -> X > u ~C | |||
1756 | // X & -C != -C -> X <= u ~C | |||
1757 | // iff C is a power of 2 | |||
1758 | if (Cmp.getOperand(1) == Y && (-(*C)).isPowerOf2()) { | |||
1759 | auto NewPred = Cmp.getPredicate() == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGT | |||
1760 | : CmpInst::ICMP_ULE; | |||
1761 | return new ICmpInst(NewPred, X, SubOne(cast<Constant>(Cmp.getOperand(1)))); | |||
1762 | } | |||
1763 | ||||
1764 | // (X & C2) == 0 -> (trunc X) >= 0 | |||
1765 | // (X & C2) != 0 -> (trunc X) < 0 | |||
1766 | // iff C2 is a power of 2 and it masks the sign bit of a legal integer type. | |||
1767 | const APInt *C2; | |||
1768 | if (And->hasOneUse() && *C == 0 && match(Y, m_APInt(C2))) { | |||
1769 | int32_t ExactLogBase2 = C2->exactLogBase2(); | |||
1770 | if (ExactLogBase2 != -1 && DL.isLegalInteger(ExactLogBase2 + 1)) { | |||
1771 | Type *NTy = IntegerType::get(Cmp.getContext(), ExactLogBase2 + 1); | |||
1772 | if (And->getType()->isVectorTy()) | |||
1773 | NTy = VectorType::get(NTy, And->getType()->getVectorNumElements()); | |||
1774 | Value *Trunc = Builder->CreateTrunc(X, NTy); | |||
1775 | auto NewPred = Cmp.getPredicate() == CmpInst::ICMP_EQ ? CmpInst::ICMP_SGE | |||
1776 | : CmpInst::ICMP_SLT; | |||
1777 | return new ICmpInst(NewPred, Trunc, Constant::getNullValue(NTy)); | |||
1778 | } | |||
1779 | } | |||
1780 | ||||
1781 | return nullptr; | |||
1782 | } | |||
1783 | ||||
1784 | /// Fold icmp (or X, Y), C. | |||
1785 | Instruction *InstCombiner::foldICmpOrConstant(ICmpInst &Cmp, BinaryOperator *Or, | |||
1786 | const APInt *C) { | |||
1787 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | |||
1788 | if (*C == 1) { | |||
1789 | // icmp slt signum(V) 1 --> icmp slt V, 1 | |||
1790 | Value *V = nullptr; | |||
1791 | if (Pred == ICmpInst::ICMP_SLT && match(Or, m_Signum(m_Value(V)))) | |||
1792 | return new ICmpInst(ICmpInst::ICMP_SLT, V, | |||
1793 | ConstantInt::get(V->getType(), 1)); | |||
1794 | } | |||
1795 | ||||
1796 | // X | C == C --> X <=u C | |||
1797 | // X | C != C --> X >u C | |||
1798 | // iff C+1 is a power of 2 (C is a bitmask of the low bits) | |||
1799 | if (Cmp.isEquality() && Cmp.getOperand(1) == Or->getOperand(1) && | |||
1800 | (*C + 1).isPowerOf2()) { | |||
1801 | Pred = (Pred == CmpInst::ICMP_EQ) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT; | |||
1802 | return new ICmpInst(Pred, Or->getOperand(0), Or->getOperand(1)); | |||
1803 | } | |||
1804 | ||||
1805 | if (!Cmp.isEquality() || *C != 0 || !Or->hasOneUse()) | |||
1806 | return nullptr; | |||
1807 | ||||
1808 | Value *P, *Q; | |||
1809 | if (match(Or, m_Or(m_PtrToInt(m_Value(P)), m_PtrToInt(m_Value(Q))))) { | |||
1810 | // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0 | |||
1811 | // -> and (icmp eq P, null), (icmp eq Q, null). | |||
1812 | Value *CmpP = | |||
1813 | Builder->CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType())); | |||
1814 | Value *CmpQ = | |||
1815 | Builder->CreateICmp(Pred, Q, ConstantInt::getNullValue(Q->getType())); | |||
1816 | auto LogicOpc = Pred == ICmpInst::Predicate::ICMP_EQ ? Instruction::And | |||
1817 | : Instruction::Or; | |||
1818 | return BinaryOperator::Create(LogicOpc, CmpP, CmpQ); | |||
1819 | } | |||
1820 | ||||
1821 | return nullptr; | |||
1822 | } | |||
1823 | ||||
1824 | /// Fold icmp (mul X, Y), C. | |||
1825 | Instruction *InstCombiner::foldICmpMulConstant(ICmpInst &Cmp, | |||
1826 | BinaryOperator *Mul, | |||
1827 | const APInt *C) { | |||
1828 | const APInt *MulC; | |||
1829 | if (!match(Mul->getOperand(1), m_APInt(MulC))) | |||
1830 | return nullptr; | |||
1831 | ||||
1832 | // If this is a test of the sign bit and the multiply is sign-preserving with | |||
1833 | // a constant operand, use the multiply LHS operand instead. | |||
1834 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | |||
1835 | if (isSignTest(Pred, *C) && Mul->hasNoSignedWrap()) { | |||
1836 | if (MulC->isNegative()) | |||
1837 | Pred = ICmpInst::getSwappedPredicate(Pred); | |||
1838 | return new ICmpInst(Pred, Mul->getOperand(0), | |||
1839 | Constant::getNullValue(Mul->getType())); | |||
1840 | } | |||
1841 | ||||
1842 | return nullptr; | |||
1843 | } | |||
1844 | ||||
1845 | /// Fold icmp (shl 1, Y), C. | |||
1846 | static Instruction *foldICmpShlOne(ICmpInst &Cmp, Instruction *Shl, | |||
1847 | const APInt *C) { | |||
1848 | Value *Y; | |||
1849 | if (!match(Shl, m_Shl(m_One(), m_Value(Y)))) | |||
1850 | return nullptr; | |||
1851 | ||||
1852 | Type *ShiftType = Shl->getType(); | |||
1853 | uint32_t TypeBits = C->getBitWidth(); | |||
1854 | bool CIsPowerOf2 = C->isPowerOf2(); | |||
1855 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | |||
1856 | if (Cmp.isUnsigned()) { | |||
1857 | // (1 << Y) pred C -> Y pred Log2(C) | |||
1858 | if (!CIsPowerOf2) { | |||
1859 | // (1 << Y) < 30 -> Y <= 4 | |||
1860 | // (1 << Y) <= 30 -> Y <= 4 | |||
1861 | // (1 << Y) >= 30 -> Y > 4 | |||
1862 | // (1 << Y) > 30 -> Y > 4 | |||
1863 | if (Pred == ICmpInst::ICMP_ULT) | |||
1864 | Pred = ICmpInst::ICMP_ULE; | |||
1865 | else if (Pred == ICmpInst::ICMP_UGE) | |||
1866 | Pred = ICmpInst::ICMP_UGT; | |||
1867 | } | |||
1868 | ||||
1869 | // (1 << Y) >= 2147483648 -> Y >= 31 -> Y == 31 | |||
1870 | // (1 << Y) < 2147483648 -> Y < 31 -> Y != 31 | |||
1871 | unsigned CLog2 = C->logBase2(); | |||
1872 | if (CLog2 == TypeBits - 1) { | |||
1873 | if (Pred == ICmpInst::ICMP_UGE) | |||
1874 | Pred = ICmpInst::ICMP_EQ; | |||
1875 | else if (Pred == ICmpInst::ICMP_ULT) | |||
1876 | Pred = ICmpInst::ICMP_NE; | |||
1877 | } | |||
1878 | return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, CLog2)); | |||
1879 | } else if (Cmp.isSigned()) { | |||
1880 | Constant *BitWidthMinusOne = ConstantInt::get(ShiftType, TypeBits - 1); | |||
1881 | if (C->isAllOnesValue()) { | |||
1882 | // (1 << Y) <= -1 -> Y == 31 | |||
1883 | if (Pred == ICmpInst::ICMP_SLE) | |||
1884 | return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); | |||
1885 | ||||
1886 | // (1 << Y) > -1 -> Y != 31 | |||
1887 | if (Pred == ICmpInst::ICMP_SGT) | |||
1888 | return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); | |||
1889 | } else if (!(*C)) { | |||
1890 | // (1 << Y) < 0 -> Y == 31 | |||
1891 | // (1 << Y) <= 0 -> Y == 31 | |||
1892 | if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) | |||
1893 | return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); | |||
1894 | ||||
1895 | // (1 << Y) >= 0 -> Y != 31 | |||
1896 | // (1 << Y) > 0 -> Y != 31 | |||
1897 | if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) | |||
1898 | return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); | |||
1899 | } | |||
1900 | } else if (Cmp.isEquality() && CIsPowerOf2) { | |||
1901 | return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, C->logBase2())); | |||
1902 | } | |||
1903 | ||||
1904 | return nullptr; | |||
1905 | } | |||
1906 | ||||
1907 | /// Fold icmp (shl X, Y), C. | |||
1908 | Instruction *InstCombiner::foldICmpShlConstant(ICmpInst &Cmp, | |||
1909 | BinaryOperator *Shl, | |||
1910 | const APInt *C) { | |||
1911 | const APInt *ShiftVal; | |||
1912 | if (Cmp.isEquality() && match(Shl->getOperand(0), m_APInt(ShiftVal))) | |||
1913 | return foldICmpShlConstConst(Cmp, Shl->getOperand(1), *C, *ShiftVal); | |||
1914 | ||||
1915 | const APInt *ShiftAmt; | |||
1916 | if (!match(Shl->getOperand(1), m_APInt(ShiftAmt))) | |||
1917 | return foldICmpShlOne(Cmp, Shl, C); | |||
1918 | ||||
1919 | // Check that the shift amount is in range. If not, don't perform undefined | |||
1920 | // shifts. When the shift is visited, it will be simplified. | |||
1921 | unsigned TypeBits = C->getBitWidth(); | |||
1922 | if (ShiftAmt->uge(TypeBits)) | |||
1923 | return nullptr; | |||
1924 | ||||
1925 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | |||
1926 | Value *X = Shl->getOperand(0); | |||
1927 | Type *ShType = Shl->getType(); | |||
1928 | ||||
1929 | // NSW guarantees that we are only shifting out sign bits from the high bits, | |||
1930 | // so we can ASHR the compare constant without needing a mask and eliminate | |||
1931 | // the shift. | |||
1932 | if (Shl->hasNoSignedWrap()) { | |||
1933 | if (Pred == ICmpInst::ICMP_SGT) { | |||
1934 | // icmp Pred (shl nsw X, ShiftAmt), C --> icmp Pred X, (C >>s ShiftAmt) | |||
1935 | APInt ShiftedC = C->ashr(*ShiftAmt); | |||
1936 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | |||
1937 | } | |||
1938 | if (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) { | |||
1939 | // This is the same code as the SGT case, but assert the pre-condition | |||
1940 | // that is needed for this to work with equality predicates. | |||
1941 | assert(C->ashr(*ShiftAmt).shl(*ShiftAmt) == *C &&((C->ashr(*ShiftAmt).shl(*ShiftAmt) == *C && "Compare known true or false was not folded" ) ? static_cast<void> (0) : __assert_fail ("C->ashr(*ShiftAmt).shl(*ShiftAmt) == *C && \"Compare known true or false was not folded\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1942, __PRETTY_FUNCTION__)) | |||
1942 | "Compare known true or false was not folded")((C->ashr(*ShiftAmt).shl(*ShiftAmt) == *C && "Compare known true or false was not folded" ) ? static_cast<void> (0) : __assert_fail ("C->ashr(*ShiftAmt).shl(*ShiftAmt) == *C && \"Compare known true or false was not folded\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1942, __PRETTY_FUNCTION__)); | |||
1943 | APInt ShiftedC = C->ashr(*ShiftAmt); | |||
1944 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | |||
1945 | } | |||
1946 | if (Pred == ICmpInst::ICMP_SLT) { | |||
1947 | // SLE is the same as above, but SLE is canonicalized to SLT, so convert: | |||
1948 | // (X << S) <=s C is equiv to X <=s (C >> S) for all C | |||
1949 | // (X << S) <s (C + 1) is equiv to X <s (C >> S) + 1 if C <s SMAX | |||
1950 | // (X << S) <s C is equiv to X <s ((C - 1) >> S) + 1 if C >s SMIN | |||
1951 | assert(!C->isMinSignedValue() && "Unexpected icmp slt")((!C->isMinSignedValue() && "Unexpected icmp slt") ? static_cast<void> (0) : __assert_fail ("!C->isMinSignedValue() && \"Unexpected icmp slt\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1951, __PRETTY_FUNCTION__)); | |||
1952 | APInt ShiftedC = (*C - 1).ashr(*ShiftAmt) + 1; | |||
1953 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | |||
1954 | } | |||
1955 | // If this is a signed comparison to 0 and the shift is sign preserving, | |||
1956 | // use the shift LHS operand instead; isSignTest may change 'Pred', so only | |||
1957 | // do that if we're sure to not continue on in this function. | |||
1958 | if (isSignTest(Pred, *C)) | |||
1959 | return new ICmpInst(Pred, X, Constant::getNullValue(ShType)); | |||
1960 | } | |||
1961 | ||||
1962 | // NUW guarantees that we are only shifting out zero bits from the high bits, | |||
1963 | // so we can LSHR the compare constant without needing a mask and eliminate | |||
1964 | // the shift. | |||
1965 | if (Shl->hasNoUnsignedWrap()) { | |||
1966 | if (Pred == ICmpInst::ICMP_UGT) { | |||
1967 | // icmp Pred (shl nuw X, ShiftAmt), C --> icmp Pred X, (C >>u ShiftAmt) | |||
1968 | APInt ShiftedC = C->lshr(*ShiftAmt); | |||
1969 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | |||
1970 | } | |||
1971 | if (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) { | |||
1972 | // This is the same code as the UGT case, but assert the pre-condition | |||
1973 | // that is needed for this to work with equality predicates. | |||
1974 | assert(C->lshr(*ShiftAmt).shl(*ShiftAmt) == *C &&((C->lshr(*ShiftAmt).shl(*ShiftAmt) == *C && "Compare known true or false was not folded" ) ? static_cast<void> (0) : __assert_fail ("C->lshr(*ShiftAmt).shl(*ShiftAmt) == *C && \"Compare known true or false was not folded\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1975, __PRETTY_FUNCTION__)) | |||
1975 | "Compare known true or false was not folded")((C->lshr(*ShiftAmt).shl(*ShiftAmt) == *C && "Compare known true or false was not folded" ) ? static_cast<void> (0) : __assert_fail ("C->lshr(*ShiftAmt).shl(*ShiftAmt) == *C && \"Compare known true or false was not folded\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1975, __PRETTY_FUNCTION__)); | |||
1976 | APInt ShiftedC = C->lshr(*ShiftAmt); | |||
1977 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | |||
1978 | } | |||
1979 | if (Pred == ICmpInst::ICMP_ULT) { | |||
1980 | // ULE is the same as above, but ULE is canonicalized to ULT, so convert: | |||
1981 | // (X << S) <=u C is equiv to X <=u (C >> S) for all C | |||
1982 | // (X << S) <u (C + 1) is equiv to X <u (C >> S) + 1 if C <u ~0u | |||
1983 | // (X << S) <u C is equiv to X <u ((C - 1) >> S) + 1 if C >u 0 | |||
1984 | assert(C->ugt(0) && "ult 0 should have been eliminated")((C->ugt(0) && "ult 0 should have been eliminated" ) ? static_cast<void> (0) : __assert_fail ("C->ugt(0) && \"ult 0 should have been eliminated\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1984, __PRETTY_FUNCTION__)); | |||
1985 | APInt ShiftedC = (*C - 1).lshr(*ShiftAmt) + 1; | |||
1986 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | |||
1987 | } | |||
1988 | } | |||
1989 | ||||
1990 | if (Cmp.isEquality() && Shl->hasOneUse()) { | |||
1991 | // Strength-reduce the shift into an 'and'. | |||
1992 | Constant *Mask = ConstantInt::get( | |||
1993 | ShType, | |||
1994 | APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt->getZExtValue())); | |||
1995 | Value *And = Builder->CreateAnd(X, Mask, Shl->getName() + ".mask"); | |||
1996 | Constant *LShrC = ConstantInt::get(ShType, C->lshr(*ShiftAmt)); | |||
1997 | return new ICmpInst(Pred, And, LShrC); | |||
1998 | } | |||
1999 | ||||
2000 | // Otherwise, if this is a comparison of the sign bit, simplify to and/test. | |||
2001 | bool TrueIfSigned = false; | |||
2002 | if (Shl->hasOneUse() && isSignBitCheck(Pred, *C, TrueIfSigned)) { | |||
2003 | // (X << 31) <s 0 --> (X & 1) != 0 | |||
2004 | Constant *Mask = ConstantInt::get( | |||
2005 | ShType, | |||
2006 | APInt::getOneBitSet(TypeBits, TypeBits - ShiftAmt->getZExtValue() - 1)); | |||
2007 | Value *And = Builder->CreateAnd(X, Mask, Shl->getName() + ".mask"); | |||
2008 | return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ, | |||
2009 | And, Constant::getNullValue(ShType)); | |||
2010 | } | |||
2011 | ||||
2012 | // Transform (icmp pred iM (shl iM %v, N), C) | |||
2013 | // -> (icmp pred i(M-N) (trunc %v iM to i(M-N)), (trunc (C>>N)) | |||
2014 | // Transform the shl to a trunc if (trunc (C>>N)) has no loss and M-N. | |||
2015 | // This enables us to get rid of the shift in favor of a trunc that may be | |||
2016 | // free on the target. It has the additional benefit of comparing to a | |||
2017 | // smaller constant that may be more target-friendly. | |||
2018 | unsigned Amt = ShiftAmt->getLimitedValue(TypeBits - 1); | |||
2019 | if (Shl->hasOneUse() && Amt != 0 && C->countTrailingZeros() >= Amt && | |||
2020 | DL.isLegalInteger(TypeBits - Amt)) { | |||
2021 | Type *TruncTy = IntegerType::get(Cmp.getContext(), TypeBits - Amt); | |||
2022 | if (ShType->isVectorTy()) | |||
2023 | TruncTy = VectorType::get(TruncTy, ShType->getVectorNumElements()); | |||
2024 | Constant *NewC = | |||
2025 | ConstantInt::get(TruncTy, C->ashr(*ShiftAmt).trunc(TypeBits - Amt)); | |||
2026 | return new ICmpInst(Pred, Builder->CreateTrunc(X, TruncTy), NewC); | |||
2027 | } | |||
2028 | ||||
2029 | return nullptr; | |||
2030 | } | |||
2031 | ||||
2032 | /// Fold icmp ({al}shr X, Y), C. | |||
2033 | Instruction *InstCombiner::foldICmpShrConstant(ICmpInst &Cmp, | |||
2034 | BinaryOperator *Shr, | |||
2035 | const APInt *C) { | |||
2036 | // An exact shr only shifts out zero bits, so: | |||
2037 | // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0 | |||
2038 | Value *X = Shr->getOperand(0); | |||
2039 | CmpInst::Predicate Pred = Cmp.getPredicate(); | |||
2040 | if (Cmp.isEquality() && Shr->isExact() && Shr->hasOneUse() && *C == 0) | |||
2041 | return new ICmpInst(Pred, X, Cmp.getOperand(1)); | |||
2042 | ||||
2043 | const APInt *ShiftVal; | |||
2044 | if (Cmp.isEquality() && match(Shr->getOperand(0), m_APInt(ShiftVal))) | |||
2045 | return foldICmpShrConstConst(Cmp, Shr->getOperand(1), *C, *ShiftVal); | |||
2046 | ||||
2047 | const APInt *ShiftAmt; | |||
2048 | if (!match(Shr->getOperand(1), m_APInt(ShiftAmt))) | |||
2049 | return nullptr; | |||
2050 | ||||
2051 | // Check that the shift amount is in range. If not, don't perform undefined | |||
2052 | // shifts. When the shift is visited it will be simplified. | |||
2053 | unsigned TypeBits = C->getBitWidth(); | |||
2054 | unsigned ShAmtVal = ShiftAmt->getLimitedValue(TypeBits); | |||
2055 | if (ShAmtVal >= TypeBits || ShAmtVal == 0) | |||
2056 | return nullptr; | |||
2057 | ||||
2058 | bool IsAShr = Shr->getOpcode() == Instruction::AShr; | |||
2059 | if (!Cmp.isEquality()) { | |||
2060 | // If we have an unsigned comparison and an ashr, we can't simplify this. | |||
2061 | // Similarly for signed comparisons with lshr. | |||
2062 | if (Cmp.isSigned() != IsAShr) | |||
2063 | return nullptr; | |||
2064 | ||||
2065 | // Otherwise, all lshr and most exact ashr's are equivalent to a udiv/sdiv | |||
2066 | // by a power of 2. Since we already have logic to simplify these, | |||
2067 | // transform to div and then simplify the resultant comparison. | |||
2068 | if (IsAShr && (!Shr->isExact() || ShAmtVal == TypeBits - 1)) | |||
2069 | return nullptr; | |||
2070 | ||||
2071 | // Revisit the shift (to delete it). | |||
2072 | Worklist.Add(Shr); | |||
2073 | ||||
2074 | Constant *DivCst = ConstantInt::get( | |||
2075 | Shr->getType(), APInt::getOneBitSet(TypeBits, ShAmtVal)); | |||
2076 | ||||
2077 | Value *Tmp = IsAShr ? Builder->CreateSDiv(X, DivCst, "", Shr->isExact()) | |||
2078 | : Builder->CreateUDiv(X, DivCst, "", Shr->isExact()); | |||
2079 | ||||
2080 | Cmp.setOperand(0, Tmp); | |||
2081 | ||||
2082 | // If the builder folded the binop, just return it. | |||
2083 | BinaryOperator *TheDiv = dyn_cast<BinaryOperator>(Tmp); | |||
2084 | if (!TheDiv) | |||
2085 | return &Cmp; | |||
2086 | ||||
2087 | // Otherwise, fold this div/compare. | |||
2088 | assert(TheDiv->getOpcode() == Instruction::SDiv ||((TheDiv->getOpcode() == Instruction::SDiv || TheDiv->getOpcode () == Instruction::UDiv) ? static_cast<void> (0) : __assert_fail ("TheDiv->getOpcode() == Instruction::SDiv || TheDiv->getOpcode() == Instruction::UDiv" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2089, __PRETTY_FUNCTION__)) | |||
2089 | TheDiv->getOpcode() == Instruction::UDiv)((TheDiv->getOpcode() == Instruction::SDiv || TheDiv->getOpcode () == Instruction::UDiv) ? static_cast<void> (0) : __assert_fail ("TheDiv->getOpcode() == Instruction::SDiv || TheDiv->getOpcode() == Instruction::UDiv" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2089, __PRETTY_FUNCTION__)); | |||
2090 | ||||
2091 | Instruction *Res = foldICmpDivConstant(Cmp, TheDiv, C); | |||
2092 | assert(Res && "This div/cst should have folded!")((Res && "This div/cst should have folded!") ? static_cast <void> (0) : __assert_fail ("Res && \"This div/cst should have folded!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2092, __PRETTY_FUNCTION__)); | |||
2093 | return Res; | |||
2094 | } | |||
2095 | ||||
2096 | // Handle equality comparisons of shift-by-constant. | |||
2097 | ||||
2098 | // If the comparison constant changes with the shift, the comparison cannot | |||
2099 | // succeed (bits of the comparison constant cannot match the shifted value). | |||
2100 | // This should be known by InstSimplify and already be folded to true/false. | |||
2101 | assert(((IsAShr && C->shl(ShAmtVal).ashr(ShAmtVal) == *C) ||((((IsAShr && C->shl(ShAmtVal).ashr(ShAmtVal) == * C) || (!IsAShr && C->shl(ShAmtVal).lshr(ShAmtVal) == *C)) && "Expected icmp+shr simplify did not occur.") ? static_cast<void> (0) : __assert_fail ("((IsAShr && C->shl(ShAmtVal).ashr(ShAmtVal) == *C) || (!IsAShr && C->shl(ShAmtVal).lshr(ShAmtVal) == *C)) && \"Expected icmp+shr simplify did not occur.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2103, __PRETTY_FUNCTION__)) | |||
2102 | (!IsAShr && C->shl(ShAmtVal).lshr(ShAmtVal) == *C)) &&((((IsAShr && C->shl(ShAmtVal).ashr(ShAmtVal) == * C) || (!IsAShr && C->shl(ShAmtVal).lshr(ShAmtVal) == *C)) && "Expected icmp+shr simplify did not occur.") ? static_cast<void> (0) : __assert_fail ("((IsAShr && C->shl(ShAmtVal).ashr(ShAmtVal) == *C) || (!IsAShr && C->shl(ShAmtVal).lshr(ShAmtVal) == *C)) && \"Expected icmp+shr simplify did not occur.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2103, __PRETTY_FUNCTION__)) | |||
2103 | "Expected icmp+shr simplify did not occur.")((((IsAShr && C->shl(ShAmtVal).ashr(ShAmtVal) == * C) || (!IsAShr && C->shl(ShAmtVal).lshr(ShAmtVal) == *C)) && "Expected icmp+shr simplify did not occur.") ? static_cast<void> (0) : __assert_fail ("((IsAShr && C->shl(ShAmtVal).ashr(ShAmtVal) == *C) || (!IsAShr && C->shl(ShAmtVal).lshr(ShAmtVal) == *C)) && \"Expected icmp+shr simplify did not occur.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2103, __PRETTY_FUNCTION__)); | |||
2104 | ||||
2105 | // Check if the bits shifted out are known to be zero. If so, we can compare | |||
2106 | // against the unshifted value: | |||
2107 | // (X & 4) >> 1 == 2 --> (X & 4) == 4. | |||
2108 | Constant *ShiftedCmpRHS = ConstantInt::get(Shr->getType(), *C << ShAmtVal); | |||
2109 | if (Shr->hasOneUse()) { | |||
2110 | if (Shr->isExact()) | |||
2111 | return new ICmpInst(Pred, X, ShiftedCmpRHS); | |||
2112 | ||||
2113 | // Otherwise strength reduce the shift into an 'and'. | |||
2114 | APInt Val(APInt::getHighBitsSet(TypeBits, TypeBits - ShAmtVal)); | |||
2115 | Constant *Mask = ConstantInt::get(Shr->getType(), Val); | |||
2116 | Value *And = Builder->CreateAnd(X, Mask, Shr->getName() + ".mask"); | |||
2117 | return new ICmpInst(Pred, And, ShiftedCmpRHS); | |||
2118 | } | |||
2119 | ||||
2120 | return nullptr; | |||
2121 | } | |||
2122 | ||||
2123 | /// Fold icmp (udiv X, Y), C. | |||
2124 | Instruction *InstCombiner::foldICmpUDivConstant(ICmpInst &Cmp, | |||
2125 | BinaryOperator *UDiv, | |||
2126 | const APInt *C) { | |||
2127 | const APInt *C2; | |||
2128 | if (!match(UDiv->getOperand(0), m_APInt(C2))) | |||
2129 | return nullptr; | |||
2130 | ||||
2131 | assert(C2 != 0 && "udiv 0, X should have been simplified already.")((C2 != 0 && "udiv 0, X should have been simplified already." ) ? static_cast<void> (0) : __assert_fail ("C2 != 0 && \"udiv 0, X should have been simplified already.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2131, __PRETTY_FUNCTION__)); | |||
2132 | ||||
2133 | // (icmp ugt (udiv C2, Y), C) -> (icmp ule Y, C2/(C+1)) | |||
2134 | Value *Y = UDiv->getOperand(1); | |||
2135 | if (Cmp.getPredicate() == ICmpInst::ICMP_UGT) { | |||
2136 | assert(!C->isMaxValue() &&((!C->isMaxValue() && "icmp ugt X, UINT_MAX should have been simplified already." ) ? static_cast<void> (0) : __assert_fail ("!C->isMaxValue() && \"icmp ugt X, UINT_MAX should have been simplified already.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2137, __PRETTY_FUNCTION__)) | |||
2137 | "icmp ugt X, UINT_MAX should have been simplified already.")((!C->isMaxValue() && "icmp ugt X, UINT_MAX should have been simplified already." ) ? static_cast<void> (0) : __assert_fail ("!C->isMaxValue() && \"icmp ugt X, UINT_MAX should have been simplified already.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2137, __PRETTY_FUNCTION__)); | |||
2138 | return new ICmpInst(ICmpInst::ICMP_ULE, Y, | |||
2139 | ConstantInt::get(Y->getType(), C2->udiv(*C + 1))); | |||
2140 | } | |||
2141 | ||||
2142 | // (icmp ult (udiv C2, Y), C) -> (icmp ugt Y, C2/C) | |||
2143 | if (Cmp.getPredicate() == ICmpInst::ICMP_ULT) { | |||
2144 | assert(C != 0 && "icmp ult X, 0 should have been simplified already.")((C != 0 && "icmp ult X, 0 should have been simplified already." ) ? static_cast<void> (0) : __assert_fail ("C != 0 && \"icmp ult X, 0 should have been simplified already.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2144, __PRETTY_FUNCTION__)); | |||
2145 | return new ICmpInst(ICmpInst::ICMP_UGT, Y, | |||
2146 | ConstantInt::get(Y->getType(), C2->udiv(*C))); | |||
2147 | } | |||
2148 | ||||
2149 | return nullptr; | |||
2150 | } | |||
2151 | ||||
2152 | /// Fold icmp ({su}div X, Y), C. | |||
2153 | Instruction *InstCombiner::foldICmpDivConstant(ICmpInst &Cmp, | |||
2154 | BinaryOperator *Div, | |||
2155 | const APInt *C) { | |||
2156 | // Fold: icmp pred ([us]div X, C2), C -> range test | |||
2157 | // Fold this div into the comparison, producing a range check. | |||
2158 | // Determine, based on the divide type, what the range is being | |||
2159 | // checked. If there is an overflow on the low or high side, remember | |||
2160 | // it, otherwise compute the range [low, hi) bounding the new value. | |||
2161 | // See: InsertRangeTest above for the kinds of replacements possible. | |||
2162 | const APInt *C2; | |||
2163 | if (!match(Div->getOperand(1), m_APInt(C2))) | |||
| ||||
2164 | return nullptr; | |||
2165 | ||||
2166 | // FIXME: If the operand types don't match the type of the divide | |||
2167 | // then don't attempt this transform. The code below doesn't have the | |||
2168 | // logic to deal with a signed divide and an unsigned compare (and | |||
2169 | // vice versa). This is because (x /s C2) <s C produces different | |||
2170 | // results than (x /s C2) <u C or (x /u C2) <s C or even | |||
2171 | // (x /u C2) <u C. Simply casting the operands and result won't | |||
2172 | // work. :( The if statement below tests that condition and bails | |||
2173 | // if it finds it. | |||
2174 | bool DivIsSigned = Div->getOpcode() == Instruction::SDiv; | |||
2175 | if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned()) | |||
2176 | return nullptr; | |||
2177 | ||||
2178 | // The ProdOV computation fails on divide by 0 and divide by -1. Cases with | |||
2179 | // INT_MIN will also fail if the divisor is 1. Although folds of all these | |||
2180 | // division-by-constant cases should be present, we can not assert that they | |||
2181 | // have happened before we reach this icmp instruction. | |||
2182 | if (*C2 == 0 || *C2 == 1 || (DivIsSigned && C2->isAllOnesValue())) | |||
2183 | return nullptr; | |||
2184 | ||||
2185 | // TODO: We could do all of the computations below using APInt. | |||
2186 | Constant *CmpRHS = cast<Constant>(Cmp.getOperand(1)); | |||
2187 | Constant *DivRHS = cast<Constant>(Div->getOperand(1)); | |||
2188 | ||||
2189 | // Compute Prod = CmpRHS * DivRHS. We are essentially solving an equation of | |||
2190 | // form X / C2 = C. We solve for X by multiplying C2 (DivRHS) and C (CmpRHS). | |||
2191 | // By solving for X, we can turn this into a range check instead of computing | |||
2192 | // a divide. | |||
2193 | Constant *Prod = ConstantExpr::getMul(CmpRHS, DivRHS); | |||
2194 | ||||
2195 | // Determine if the product overflows by seeing if the product is not equal to | |||
2196 | // the divide. Make sure we do the same kind of divide as in the LHS | |||
2197 | // instruction that we're folding. | |||
2198 | bool ProdOV = (DivIsSigned ? ConstantExpr::getSDiv(Prod, DivRHS) | |||
2199 | : ConstantExpr::getUDiv(Prod, DivRHS)) != CmpRHS; | |||
2200 | ||||
2201 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | |||
2202 | ||||
2203 | // If the division is known to be exact, then there is no remainder from the | |||
2204 | // divide, so the covered range size is unit, otherwise it is the divisor. | |||
2205 | Constant *RangeSize = | |||
2206 | Div->isExact() ? ConstantInt::get(Div->getType(), 1) : DivRHS; | |||
2207 | ||||
2208 | // Figure out the interval that is being checked. For example, a comparison | |||
2209 | // like "X /u 5 == 0" is really checking that X is in the interval [0, 5). | |||
2210 | // Compute this interval based on the constants involved and the signedness of | |||
2211 | // the compare/divide. This computes a half-open interval, keeping track of | |||
2212 | // whether either value in the interval overflows. After analysis each | |||
2213 | // overflow variable is set to 0 if it's corresponding bound variable is valid | |||
2214 | // -1 if overflowed off the bottom end, or +1 if overflowed off the top end. | |||
2215 | int LoOverflow = 0, HiOverflow = 0; | |||
2216 | Constant *LoBound = nullptr, *HiBound = nullptr; | |||
2217 | ||||
2218 | if (!DivIsSigned) { // udiv | |||
2219 | // e.g. X/5 op 3 --> [15, 20) | |||
2220 | LoBound = Prod; | |||
2221 | HiOverflow = LoOverflow = ProdOV; | |||
2222 | if (!HiOverflow) { | |||
2223 | // If this is not an exact divide, then many values in the range collapse | |||
2224 | // to the same result value. | |||
2225 | HiOverflow = addWithOverflow(HiBound, LoBound, RangeSize, false); | |||
2226 | } | |||
2227 | } else if (C2->isStrictlyPositive()) { // Divisor is > 0. | |||
2228 | if (*C == 0) { // (X / pos) op 0 | |||
2229 | // Can't overflow. e.g. X/2 op 0 --> [-1, 2) | |||
2230 | LoBound = ConstantExpr::getNeg(SubOne(RangeSize)); | |||
2231 | HiBound = RangeSize; | |||
2232 | } else if (C->isStrictlyPositive()) { // (X / pos) op pos | |||
2233 | LoBound = Prod; // e.g. X/5 op 3 --> [15, 20) | |||
2234 | HiOverflow = LoOverflow = ProdOV; | |||
2235 | if (!HiOverflow) | |||
2236 | HiOverflow = addWithOverflow(HiBound, Prod, RangeSize, true); | |||
2237 | } else { // (X / pos) op neg | |||
2238 | // e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14) | |||
2239 | HiBound = AddOne(Prod); | |||
2240 | LoOverflow = HiOverflow = ProdOV ? -1 : 0; | |||
2241 | if (!LoOverflow) { | |||
2242 | Constant *DivNeg = ConstantExpr::getNeg(RangeSize); | |||
2243 | LoOverflow = addWithOverflow(LoBound, HiBound, DivNeg, true) ? -1 : 0; | |||
2244 | } | |||
2245 | } | |||
2246 | } else if (C2->isNegative()) { // Divisor is < 0. | |||
2247 | if (Div->isExact()) | |||
2248 | RangeSize = ConstantExpr::getNeg(RangeSize); | |||
2249 | if (*C == 0) { // (X / neg) op 0 | |||
2250 | // e.g. X/-5 op 0 --> [-4, 5) | |||
2251 | LoBound = AddOne(RangeSize); | |||
2252 | HiBound = ConstantExpr::getNeg(RangeSize); | |||
2253 | if (HiBound == DivRHS) { // -INTMIN = INTMIN | |||
2254 | HiOverflow = 1; // [INTMIN+1, overflow) | |||
2255 | HiBound = nullptr; // e.g. X/INTMIN = 0 --> X > INTMIN | |||
2256 | } | |||
2257 | } else if (C->isStrictlyPositive()) { // (X / neg) op pos | |||
2258 | // e.g. X/-5 op 3 --> [-19, -14) | |||
2259 | HiBound = AddOne(Prod); | |||
2260 | HiOverflow = LoOverflow = ProdOV ? -1 : 0; | |||
2261 | if (!LoOverflow) | |||
2262 | LoOverflow = addWithOverflow(LoBound, HiBound, RangeSize, true) ? -1:0; | |||
2263 | } else { // (X / neg) op neg | |||
2264 | LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20) | |||
2265 | LoOverflow = HiOverflow = ProdOV; | |||
2266 | if (!HiOverflow) | |||
2267 | HiOverflow = subWithOverflow(HiBound, Prod, RangeSize, true); | |||
2268 | } | |||
2269 | ||||
2270 | // Dividing by a negative swaps the condition. LT <-> GT | |||
2271 | Pred = ICmpInst::getSwappedPredicate(Pred); | |||
2272 | } | |||
2273 | ||||
2274 | Value *X = Div->getOperand(0); | |||
2275 | switch (Pred) { | |||
2276 | default: llvm_unreachable("Unhandled icmp opcode!")::llvm::llvm_unreachable_internal("Unhandled icmp opcode!", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2276); | |||
2277 | case ICmpInst::ICMP_EQ: | |||
2278 | if (LoOverflow && HiOverflow) | |||
2279 | return replaceInstUsesWith(Cmp, Builder->getFalse()); | |||
2280 | if (HiOverflow) | |||
2281 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : | |||
2282 | ICmpInst::ICMP_UGE, X, LoBound); | |||
2283 | if (LoOverflow) | |||
2284 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : | |||
2285 | ICmpInst::ICMP_ULT, X, HiBound); | |||
2286 | return replaceInstUsesWith( | |||
2287 | Cmp, insertRangeTest(X, LoBound->getUniqueInteger(), | |||
| ||||
2288 | HiBound->getUniqueInteger(), DivIsSigned, true)); | |||
2289 | case ICmpInst::ICMP_NE: | |||
2290 | if (LoOverflow && HiOverflow) | |||
2291 | return replaceInstUsesWith(Cmp, Builder->getTrue()); | |||
2292 | if (HiOverflow) | |||
2293 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : | |||
2294 | ICmpInst::ICMP_ULT, X, LoBound); | |||
2295 | if (LoOverflow) | |||
2296 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : | |||
2297 | ICmpInst::ICMP_UGE, X, HiBound); | |||
2298 | return replaceInstUsesWith(Cmp, | |||
2299 | insertRangeTest(X, LoBound->getUniqueInteger(), | |||
2300 | HiBound->getUniqueInteger(), | |||
2301 | DivIsSigned, false)); | |||
2302 | case ICmpInst::ICMP_ULT: | |||
2303 | case ICmpInst::ICMP_SLT: | |||
2304 | if (LoOverflow == +1) // Low bound is greater than input range. | |||
2305 | return replaceInstUsesWith(Cmp, Builder->getTrue()); | |||
2306 | if (LoOverflow == -1) // Low bound is less than input range. | |||
2307 | return replaceInstUsesWith(Cmp, Builder->getFalse()); | |||
2308 | return new ICmpInst(Pred, X, LoBound); | |||
2309 | case ICmpInst::ICMP_UGT: | |||
2310 | case ICmpInst::ICMP_SGT: | |||
2311 | if (HiOverflow == +1) // High bound greater than input range. | |||
2312 | return replaceInstUsesWith(Cmp, Builder->getFalse()); | |||
2313 | if (HiOverflow == -1) // High bound less than input range. | |||
2314 | return replaceInstUsesWith(Cmp, Builder->getTrue()); | |||
2315 | if (Pred == ICmpInst::ICMP_UGT) | |||
2316 | return new ICmpInst(ICmpInst::ICMP_UGE, X, HiBound); | |||
2317 | return new ICmpInst(ICmpInst::ICMP_SGE, X, HiBound); | |||
2318 | } | |||
2319 | ||||
2320 | return nullptr; | |||
2321 | } | |||
2322 | ||||
2323 | /// Fold icmp (sub X, Y), C. | |||
2324 | Instruction *InstCombiner::foldICmpSubConstant(ICmpInst &Cmp, | |||
2325 | BinaryOperator *Sub, | |||
2326 | const APInt *C) { | |||
2327 | Value *X = Sub->getOperand(0), *Y = Sub->getOperand(1); | |||
2328 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | |||
2329 | ||||
2330 | // The following transforms are only worth it if the only user of the subtract | |||
2331 | // is the icmp. | |||
2332 | if (!Sub->hasOneUse()) | |||
2333 | return nullptr; | |||
2334 | ||||
2335 | if (Sub->hasNoSignedWrap()) { | |||
2336 | // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y) | |||
2337 | if (Pred == ICmpInst::ICMP_SGT && C->isAllOnesValue()) | |||
2338 | return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); | |||
2339 | ||||
2340 | // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y) | |||
2341 | if (Pred == ICmpInst::ICMP_SGT && *C == 0) | |||
2342 | return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); | |||
2343 | ||||
2344 | // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y) | |||
2345 | if (Pred == ICmpInst::ICMP_SLT && *C == 0) | |||
2346 | return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); | |||
2347 | ||||
2348 | // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y) | |||
2349 | if (Pred == ICmpInst::ICMP_SLT && *C == 1) | |||
2350 | return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); | |||
2351 | } | |||
2352 | ||||
2353 | const APInt *C2; | |||
2354 | if (!match(X, m_APInt(C2))) | |||
2355 | return nullptr; | |||
2356 | ||||
2357 | // C2 - Y <u C -> (Y | (C - 1)) == C2 | |||
2358 | // iff (C2 & (C - 1)) == C - 1 and C is a power of 2 | |||
2359 | if (Pred == ICmpInst::ICMP_ULT && C->isPowerOf2() && | |||
2360 | (*C2 & (*C - 1)) == (*C - 1)) | |||
2361 | return new ICmpInst(ICmpInst::ICMP_EQ, Builder->CreateOr(Y, *C - 1), X); | |||
2362 | ||||
2363 | // C2 - Y >u C -> (Y | C) != C2 | |||
2364 | // iff C2 & C == C and C + 1 is a power of 2 | |||
2365 | if (Pred == ICmpInst::ICMP_UGT && (*C + 1).isPowerOf2() && (*C2 & *C) == *C) | |||
2366 | return new ICmpInst(ICmpInst::ICMP_NE, Builder->CreateOr(Y, *C), X); | |||
2367 | ||||
2368 | return nullptr; | |||
2369 | } | |||
2370 | ||||
2371 | /// Fold icmp (add X, Y), C. | |||
2372 | Instruction *InstCombiner::foldICmpAddConstant(ICmpInst &Cmp, | |||
2373 | BinaryOperator *Add, | |||
2374 | const APInt *C) { | |||
2375 | Value *Y = Add->getOperand(1); | |||
2376 | const APInt *C2; | |||
2377 | if (Cmp.isEquality() || !match(Y, m_APInt(C2))) | |||
2378 | return nullptr; | |||
2379 | ||||
2380 | // Fold icmp pred (add X, C2), C. | |||
2381 | Value *X = Add->getOperand(0); | |||
2382 | Type *Ty = Add->getType(); | |||
2383 | CmpInst::Predicate Pred = Cmp.getPredicate(); | |||
2384 | ||||
2385 | // If the add does not wrap, we can always adjust the compare by subtracting | |||
2386 | // the constants. Equality comparisons are handled elsewhere. SGE/SLE are | |||
2387 | // canonicalized to SGT/SLT. | |||
2388 | if (Add->hasNoSignedWrap() && | |||
2389 | (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT)) { | |||
2390 | bool Overflow; | |||
2391 | APInt NewC = C->ssub_ov(*C2, Overflow); | |||
2392 | // If there is overflow, the result must be true or false. | |||
2393 | // TODO: Can we assert there is no overflow because InstSimplify always | |||
2394 | // handles those cases? | |||
2395 | if (!Overflow) | |||
2396 | // icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2) | |||
2397 | return new ICmpInst(Pred, X, ConstantInt::get(Ty, NewC)); | |||
2398 | } | |||
2399 | ||||
2400 | auto CR = ConstantRange::makeExactICmpRegion(Pred, *C).subtract(*C2); | |||
2401 | const APInt &Upper = CR.getUpper(); | |||
2402 | const APInt &Lower = CR.getLower(); | |||
2403 | if (Cmp.isSigned()) { | |||
2404 | if (Lower.isSignMask()) | |||
2405 | return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, Upper)); | |||
2406 | if (Upper.isSignMask()) | |||
2407 | return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, Lower)); | |||
2408 | } else { | |||
2409 | if (Lower.isMinValue()) | |||
2410 | return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, Upper)); | |||
2411 | if (Upper.isMinValue()) | |||
2412 | return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, Lower)); | |||
2413 | } | |||
2414 | ||||
2415 | if (!Add->hasOneUse()) | |||
2416 | return nullptr; | |||
2417 | ||||
2418 | // X+C <u C2 -> (X & -C2) == C | |||
2419 | // iff C & (C2-1) == 0 | |||
2420 | // C2 is a power of 2 | |||
2421 | if (Pred == ICmpInst::ICMP_ULT && C->isPowerOf2() && (*C2 & (*C - 1)) == 0) | |||
2422 | return new ICmpInst(ICmpInst::ICMP_EQ, Builder->CreateAnd(X, -(*C)), | |||
2423 | ConstantExpr::getNeg(cast<Constant>(Y))); | |||
2424 | ||||
2425 | // X+C >u C2 -> (X & ~C2) != C | |||
2426 | // iff C & C2 == 0 | |||
2427 | // C2+1 is a power of 2 | |||
2428 | if (Pred == ICmpInst::ICMP_UGT && (*C + 1).isPowerOf2() && (*C2 & *C) == 0) | |||
2429 | return new ICmpInst(ICmpInst::ICMP_NE, Builder->CreateAnd(X, ~(*C)), | |||
2430 | ConstantExpr::getNeg(cast<Constant>(Y))); | |||
2431 | ||||
2432 | return nullptr; | |||
2433 | } | |||
2434 | ||||
2435 | /// Try to fold integer comparisons with a constant operand: icmp Pred X, C | |||
2436 | /// where X is some kind of instruction. | |||
2437 | Instruction *InstCombiner::foldICmpInstWithConstant(ICmpInst &Cmp) { | |||
2438 | const APInt *C; | |||
2439 | if (!match(Cmp.getOperand(1), m_APInt(C))) | |||
2440 | return nullptr; | |||
2441 | ||||
2442 | BinaryOperator *BO; | |||
2443 | if (match(Cmp.getOperand(0), m_BinOp(BO))) { | |||
2444 | switch (BO->getOpcode()) { | |||
2445 | case Instruction::Xor: | |||
2446 | if (Instruction *I = foldICmpXorConstant(Cmp, BO, C)) | |||
2447 | return I; | |||
2448 | break; | |||
2449 | case Instruction::And: | |||
2450 | if (Instruction *I = foldICmpAndConstant(Cmp, BO, C)) | |||
2451 | return I; | |||
2452 | break; | |||
2453 | case Instruction::Or: | |||
2454 | if (Instruction *I = foldICmpOrConstant(Cmp, BO, C)) | |||
2455 | return I; | |||
2456 | break; | |||
2457 | case Instruction::Mul: | |||
2458 | if (Instruction *I = foldICmpMulConstant(Cmp, BO, C)) | |||
2459 | return I; | |||
2460 | break; | |||
2461 | case Instruction::Shl: | |||
2462 | if (Instruction *I = foldICmpShlConstant(Cmp, BO, C)) | |||
2463 | return I; | |||
2464 | break; | |||
2465 | case Instruction::LShr: | |||
2466 | case Instruction::AShr: | |||
2467 | if (Instruction *I = foldICmpShrConstant(Cmp, BO, C)) | |||
2468 | return I; | |||
2469 | break; | |||
2470 | case Instruction::UDiv: | |||
2471 | if (Instruction *I = foldICmpUDivConstant(Cmp, BO, C)) | |||
2472 | return I; | |||
2473 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
2474 | case Instruction::SDiv: | |||
2475 | if (Instruction *I = foldICmpDivConstant(Cmp, BO, C)) | |||
2476 | return I; | |||
2477 | break; | |||
2478 | case Instruction::Sub: | |||
2479 | if (Instruction *I = foldICmpSubConstant(Cmp, BO, C)) | |||
2480 | return I; | |||
2481 | break; | |||
2482 | case Instruction::Add: | |||
2483 | if (Instruction *I = foldICmpAddConstant(Cmp, BO, C)) | |||
2484 | return I; | |||
2485 | break; | |||
2486 | default: | |||
2487 | break; | |||
2488 | } | |||
2489 | // TODO: These folds could be refactored to be part of the above calls. | |||
2490 | if (Instruction *I = foldICmpBinOpEqualityWithConstant(Cmp, BO, C)) | |||
2491 | return I; | |||
2492 | } | |||
2493 | ||||
2494 | Instruction *LHSI; | |||
2495 | if (match(Cmp.getOperand(0), m_Instruction(LHSI)) && | |||
2496 | LHSI->getOpcode() == Instruction::Trunc) | |||
2497 | if (Instruction *I = foldICmpTruncConstant(Cmp, LHSI, C)) | |||
2498 | return I; | |||
2499 | ||||
2500 | if (Instruction *I = foldICmpIntrinsicWithConstant(Cmp, C)) | |||
2501 | return I; | |||
2502 | ||||
2503 | return nullptr; | |||
2504 | } | |||
2505 | ||||
2506 | /// Fold an icmp equality instruction with binary operator LHS and constant RHS: | |||
2507 | /// icmp eq/ne BO, C. | |||
2508 | Instruction *InstCombiner::foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp, | |||
2509 | BinaryOperator *BO, | |||
2510 | const APInt *C) { | |||
2511 | // TODO: Some of these folds could work with arbitrary constants, but this | |||
2512 | // function is limited to scalar and vector splat constants. | |||
2513 | if (!Cmp.isEquality()) | |||
2514 | return nullptr; | |||
2515 | ||||
2516 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | |||
2517 | bool isICMP_NE = Pred == ICmpInst::ICMP_NE; | |||
2518 | Constant *RHS = cast<Constant>(Cmp.getOperand(1)); | |||
2519 | Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1); | |||
2520 | ||||
2521 | switch (BO->getOpcode()) { | |||
2522 | case Instruction::SRem: | |||
2523 | // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one. | |||
2524 | if (*C == 0 && BO->hasOneUse()) { | |||
2525 | const APInt *BOC; | |||
2526 | if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) { | |||
2527 | Value *NewRem = Builder->CreateURem(BOp0, BOp1, BO->getName()); | |||
2528 | return new ICmpInst(Pred, NewRem, | |||
2529 | Constant::getNullValue(BO->getType())); | |||
2530 | } | |||
2531 | } | |||
2532 | break; | |||
2533 | case Instruction::Add: { | |||
2534 | // Replace ((add A, B) != C) with (A != C-B) if B & C are constants. | |||
2535 | const APInt *BOC; | |||
2536 | if (match(BOp1, m_APInt(BOC))) { | |||
2537 | if (BO->hasOneUse()) { | |||
2538 | Constant *SubC = ConstantExpr::getSub(RHS, cast<Constant>(BOp1)); | |||
2539 | return new ICmpInst(Pred, BOp0, SubC); | |||
2540 | } | |||
2541 | } else if (*C == 0) { | |||
2542 | // Replace ((add A, B) != 0) with (A != -B) if A or B is | |||
2543 | // efficiently invertible, or if the add has just this one use. | |||
2544 | if (Value *NegVal = dyn_castNegVal(BOp1)) | |||
2545 | return new ICmpInst(Pred, BOp0, NegVal); | |||
2546 | if (Value *NegVal = dyn_castNegVal(BOp0)) | |||
2547 | return new ICmpInst(Pred, NegVal, BOp1); | |||
2548 | if (BO->hasOneUse()) { | |||
2549 | Value *Neg = Builder->CreateNeg(BOp1); | |||
2550 | Neg->takeName(BO); | |||
2551 | return new ICmpInst(Pred, BOp0, Neg); | |||
2552 | } | |||
2553 | } | |||
2554 | break; | |||
2555 | } | |||
2556 | case Instruction::Xor: | |||
2557 | if (BO->hasOneUse()) { | |||
2558 | if (Constant *BOC = dyn_cast<Constant>(BOp1)) { | |||
2559 | // For the xor case, we can xor two constants together, eliminating | |||
2560 | // the explicit xor. | |||
2561 | return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC)); | |||
2562 | } else if (*C == 0) { | |||
2563 | // Replace ((xor A, B) != 0) with (A != B) | |||
2564 | return new ICmpInst(Pred, BOp0, BOp1); | |||
2565 | } | |||
2566 | } | |||
2567 | break; | |||
2568 | case Instruction::Sub: | |||
2569 | if (BO->hasOneUse()) { | |||
2570 | const APInt *BOC; | |||
2571 | if (match(BOp0, m_APInt(BOC))) { | |||
2572 | // Replace ((sub BOC, B) != C) with (B != BOC-C). | |||
2573 | Constant *SubC = ConstantExpr::getSub(cast<Constant>(BOp0), RHS); | |||
2574 | return new ICmpInst(Pred, BOp1, SubC); | |||
2575 | } else if (*C == 0) { | |||
2576 | // Replace ((sub A, B) != 0) with (A != B). | |||
2577 | return new ICmpInst(Pred, BOp0, BOp1); | |||
2578 | } | |||
2579 | } | |||
2580 | break; | |||
2581 | case Instruction::Or: { | |||
2582 | const APInt *BOC; | |||
2583 | if (match(BOp1, m_APInt(BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) { | |||
2584 | // Comparing if all bits outside of a constant mask are set? | |||
2585 | // Replace (X | C) == -1 with (X & ~C) == ~C. | |||
2586 | // This removes the -1 constant. | |||
2587 | Constant *NotBOC = ConstantExpr::getNot(cast<Constant>(BOp1)); | |||
2588 | Value *And = Builder->CreateAnd(BOp0, NotBOC); | |||
2589 | return new ICmpInst(Pred, And, NotBOC); | |||
2590 | } | |||
2591 | break; | |||
2592 | } | |||
2593 | case Instruction::And: { | |||
2594 | const APInt *BOC; | |||
2595 | if (match(BOp1, m_APInt(BOC))) { | |||
2596 | // If we have ((X & C) == C), turn it into ((X & C) != 0). | |||
2597 | if (C == BOC && C->isPowerOf2()) | |||
2598 | return new ICmpInst(isICMP_NE ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE, | |||
2599 | BO, Constant::getNullValue(RHS->getType())); | |||
2600 | ||||
2601 | // Don't perform the following transforms if the AND has multiple uses | |||
2602 | if (!BO->hasOneUse()) | |||
2603 | break; | |||
2604 | ||||
2605 | // Replace (and X, (1 << size(X)-1) != 0) with x s< 0 | |||
2606 | if (BOC->isSignMask()) { | |||
2607 | Constant *Zero = Constant::getNullValue(BOp0->getType()); | |||
2608 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE; | |||
2609 | return new ICmpInst(NewPred, BOp0, Zero); | |||
2610 | } | |||
2611 | ||||
2612 | // ((X & ~7) == 0) --> X < 8 | |||
2613 | if (*C == 0 && (~(*BOC) + 1).isPowerOf2()) { | |||
2614 | Constant *NegBOC = ConstantExpr::getNeg(cast<Constant>(BOp1)); | |||
2615 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; | |||
2616 | return new ICmpInst(NewPred, BOp0, NegBOC); | |||
2617 | } | |||
2618 | } | |||
2619 | break; | |||
2620 | } | |||
2621 | case Instruction::Mul: | |||
2622 | if (*C == 0 && BO->hasNoSignedWrap()) { | |||
2623 | const APInt *BOC; | |||
2624 | if (match(BOp1, m_APInt(BOC)) && *BOC != 0) { | |||
2625 | // The trivial case (mul X, 0) is handled by InstSimplify. | |||
2626 | // General case : (mul X, C) != 0 iff X != 0 | |||
2627 | // (mul X, C) == 0 iff X == 0 | |||
2628 | return new ICmpInst(Pred, BOp0, Constant::getNullValue(RHS->getType())); | |||
2629 | } | |||
2630 | } | |||
2631 | break; | |||
2632 | case Instruction::UDiv: | |||
2633 | if (*C == 0) { | |||
2634 | // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A) | |||
2635 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; | |||
2636 | return new ICmpInst(NewPred, BOp1, BOp0); | |||
2637 | } | |||
2638 | break; | |||
2639 | default: | |||
2640 | break; | |||
2641 | } | |||
2642 | return nullptr; | |||
2643 | } | |||
2644 | ||||
2645 | /// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C. | |||
2646 | Instruction *InstCombiner::foldICmpIntrinsicWithConstant(ICmpInst &Cmp, | |||
2647 | const APInt *C) { | |||
2648 | IntrinsicInst *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0)); | |||
2649 | if (!II || !Cmp.isEquality()) | |||
2650 | return nullptr; | |||
2651 | ||||
2652 | // Handle icmp {eq|ne} <intrinsic>, intcst. | |||
2653 | switch (II->getIntrinsicID()) { | |||
2654 | case Intrinsic::bswap: | |||
2655 | Worklist.Add(II); | |||
2656 | Cmp.setOperand(0, II->getArgOperand(0)); | |||
2657 | Cmp.setOperand(1, Builder->getInt(C->byteSwap())); | |||
2658 | return &Cmp; | |||
2659 | case Intrinsic::ctlz: | |||
2660 | case Intrinsic::cttz: | |||
2661 | // ctz(A) == bitwidth(A) -> A == 0 and likewise for != | |||
2662 | if (*C == C->getBitWidth()) { | |||
2663 | Worklist.Add(II); | |||
2664 | Cmp.setOperand(0, II->getArgOperand(0)); | |||
2665 | Cmp.setOperand(1, ConstantInt::getNullValue(II->getType())); | |||
2666 | return &Cmp; | |||
2667 | } | |||
2668 | break; | |||
2669 | case Intrinsic::ctpop: { | |||
2670 | // popcount(A) == 0 -> A == 0 and likewise for != | |||
2671 | // popcount(A) == bitwidth(A) -> A == -1 and likewise for != | |||
2672 | bool IsZero = *C == 0; | |||
2673 | if (IsZero || *C == C->getBitWidth()) { | |||
2674 | Worklist.Add(II); | |||
2675 | Cmp.setOperand(0, II->getArgOperand(0)); | |||
2676 | auto *NewOp = IsZero ? Constant::getNullValue(II->getType()) | |||
2677 | : Constant::getAllOnesValue(II->getType()); | |||
2678 | Cmp.setOperand(1, NewOp); | |||
2679 | return &Cmp; | |||
2680 | } | |||
2681 | break; | |||
2682 | } | |||
2683 | default: | |||
2684 | break; | |||
2685 | } | |||
2686 | return nullptr; | |||
2687 | } | |||
2688 | ||||
2689 | /// Handle icmp with constant (but not simple integer constant) RHS. | |||
2690 | Instruction *InstCombiner::foldICmpInstWithConstantNotInt(ICmpInst &I) { | |||
2691 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
2692 | Constant *RHSC = dyn_cast<Constant>(Op1); | |||
2693 | Instruction *LHSI = dyn_cast<Instruction>(Op0); | |||
2694 | if (!RHSC || !LHSI) | |||
2695 | return nullptr; | |||
2696 | ||||
2697 | switch (LHSI->getOpcode()) { | |||
2698 | case Instruction::GetElementPtr: | |||
2699 | // icmp pred GEP (P, int 0, int 0, int 0), null -> icmp pred P, null | |||
2700 | if (RHSC->isNullValue() && | |||
2701 | cast<GetElementPtrInst>(LHSI)->hasAllZeroIndices()) | |||
2702 | return new ICmpInst( | |||
2703 | I.getPredicate(), LHSI->getOperand(0), | |||
2704 | Constant::getNullValue(LHSI->getOperand(0)->getType())); | |||
2705 | break; | |||
2706 | case Instruction::PHI: | |||
2707 | // Only fold icmp into the PHI if the phi and icmp are in the same | |||
2708 | // block. If in the same block, we're encouraging jump threading. If | |||
2709 | // not, we are just pessimizing the code by making an i1 phi. | |||
2710 | if (LHSI->getParent() == I.getParent()) | |||
2711 | if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI))) | |||
2712 | return NV; | |||
2713 | break; | |||
2714 | case Instruction::Select: { | |||
2715 | // If either operand of the select is a constant, we can fold the | |||
2716 | // comparison into the select arms, which will cause one to be | |||
2717 | // constant folded and the select turned into a bitwise or. | |||
2718 | Value *Op1 = nullptr, *Op2 = nullptr; | |||
2719 | ConstantInt *CI = nullptr; | |||
2720 | if (Constant *C = dyn_cast<Constant>(LHSI->getOperand(1))) { | |||
2721 | Op1 = ConstantExpr::getICmp(I.getPredicate(), C, RHSC); | |||
2722 | CI = dyn_cast<ConstantInt>(Op1); | |||
2723 | } | |||
2724 | if (Constant *C = dyn_cast<Constant>(LHSI->getOperand(2))) { | |||
2725 | Op2 = ConstantExpr::getICmp(I.getPredicate(), C, RHSC); | |||
2726 | CI = dyn_cast<ConstantInt>(Op2); | |||
2727 | } | |||
2728 | ||||
2729 | // We only want to perform this transformation if it will not lead to | |||
2730 | // additional code. This is true if either both sides of the select | |||
2731 | // fold to a constant (in which case the icmp is replaced with a select | |||
2732 | // which will usually simplify) or this is the only user of the | |||
2733 | // select (in which case we are trading a select+icmp for a simpler | |||
2734 | // select+icmp) or all uses of the select can be replaced based on | |||
2735 | // dominance information ("Global cases"). | |||
2736 | bool Transform = false; | |||
2737 | if (Op1 && Op2) | |||
2738 | Transform = true; | |||
2739 | else if (Op1 || Op2) { | |||
2740 | // Local case | |||
2741 | if (LHSI->hasOneUse()) | |||
2742 | Transform = true; | |||
2743 | // Global cases | |||
2744 | else if (CI && !CI->isZero()) | |||
2745 | // When Op1 is constant try replacing select with second operand. | |||
2746 | // Otherwise Op2 is constant and try replacing select with first | |||
2747 | // operand. | |||
2748 | Transform = | |||
2749 | replacedSelectWithOperand(cast<SelectInst>(LHSI), &I, Op1 ? 2 : 1); | |||
2750 | } | |||
2751 | if (Transform) { | |||
2752 | if (!Op1) | |||
2753 | Op1 = Builder->CreateICmp(I.getPredicate(), LHSI->getOperand(1), RHSC, | |||
2754 | I.getName()); | |||
2755 | if (!Op2) | |||
2756 | Op2 = Builder->CreateICmp(I.getPredicate(), LHSI->getOperand(2), RHSC, | |||
2757 | I.getName()); | |||
2758 | return SelectInst::Create(LHSI->getOperand(0), Op1, Op2); | |||
2759 | } | |||
2760 | break; | |||
2761 | } | |||
2762 | case Instruction::IntToPtr: | |||
2763 | // icmp pred inttoptr(X), null -> icmp pred X, 0 | |||
2764 | if (RHSC->isNullValue() && | |||
2765 | DL.getIntPtrType(RHSC->getType()) == LHSI->getOperand(0)->getType()) | |||
2766 | return new ICmpInst( | |||
2767 | I.getPredicate(), LHSI->getOperand(0), | |||
2768 | Constant::getNullValue(LHSI->getOperand(0)->getType())); | |||
2769 | break; | |||
2770 | ||||
2771 | case Instruction::Load: | |||
2772 | // Try to optimize things like "A[i] > 4" to index computations. | |||
2773 | if (GetElementPtrInst *GEP = | |||
2774 | dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) { | |||
2775 | if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) | |||
2776 | if (GV->isConstant() && GV->hasDefinitiveInitializer() && | |||
2777 | !cast<LoadInst>(LHSI)->isVolatile()) | |||
2778 | if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, I)) | |||
2779 | return Res; | |||
2780 | } | |||
2781 | break; | |||
2782 | } | |||
2783 | ||||
2784 | return nullptr; | |||
2785 | } | |||
2786 | ||||
2787 | /// Try to fold icmp (binop), X or icmp X, (binop). | |||
2788 | /// TODO: A large part of this logic is duplicated in InstSimplify's | |||
2789 | /// simplifyICmpWithBinOp(). We should be able to share that and avoid the code | |||
2790 | /// duplication. | |||
2791 | Instruction *InstCombiner::foldICmpBinOp(ICmpInst &I) { | |||
2792 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
2793 | ||||
2794 | // Special logic for binary operators. | |||
2795 | BinaryOperator *BO0 = dyn_cast<BinaryOperator>(Op0); | |||
2796 | BinaryOperator *BO1 = dyn_cast<BinaryOperator>(Op1); | |||
2797 | if (!BO0 && !BO1) | |||
2798 | return nullptr; | |||
2799 | ||||
2800 | const CmpInst::Predicate Pred = I.getPredicate(); | |||
2801 | bool NoOp0WrapProblem = false, NoOp1WrapProblem = false; | |||
2802 | if (BO0 && isa<OverflowingBinaryOperator>(BO0)) | |||
2803 | NoOp0WrapProblem = | |||
2804 | ICmpInst::isEquality(Pred) || | |||
2805 | (CmpInst::isUnsigned(Pred) && BO0->hasNoUnsignedWrap()) || | |||
2806 | (CmpInst::isSigned(Pred) && BO0->hasNoSignedWrap()); | |||
2807 | if (BO1 && isa<OverflowingBinaryOperator>(BO1)) | |||
2808 | NoOp1WrapProblem = | |||
2809 | ICmpInst::isEquality(Pred) || | |||
2810 | (CmpInst::isUnsigned(Pred) && BO1->hasNoUnsignedWrap()) || | |||
2811 | (CmpInst::isSigned(Pred) && BO1->hasNoSignedWrap()); | |||
2812 | ||||
2813 | // Analyze the case when either Op0 or Op1 is an add instruction. | |||
2814 | // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null). | |||
2815 | Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr; | |||
2816 | if (BO0 && BO0->getOpcode() == Instruction::Add) { | |||
2817 | A = BO0->getOperand(0); | |||
2818 | B = BO0->getOperand(1); | |||
2819 | } | |||
2820 | if (BO1 && BO1->getOpcode() == Instruction::Add) { | |||
2821 | C = BO1->getOperand(0); | |||
2822 | D = BO1->getOperand(1); | |||
2823 | } | |||
2824 | ||||
2825 | // icmp (X+Y), X -> icmp Y, 0 for equalities or if there is no overflow. | |||
2826 | if ((A == Op1 || B == Op1) && NoOp0WrapProblem) | |||
2827 | return new ICmpInst(Pred, A == Op1 ? B : A, | |||
2828 | Constant::getNullValue(Op1->getType())); | |||
2829 | ||||
2830 | // icmp X, (X+Y) -> icmp 0, Y for equalities or if there is no overflow. | |||
2831 | if ((C == Op0 || D == Op0) && NoOp1WrapProblem) | |||
2832 | return new ICmpInst(Pred, Constant::getNullValue(Op0->getType()), | |||
2833 | C == Op0 ? D : C); | |||
2834 | ||||
2835 | // icmp (X+Y), (X+Z) -> icmp Y, Z for equalities or if there is no overflow. | |||
2836 | if (A && C && (A == C || A == D || B == C || B == D) && NoOp0WrapProblem && | |||
2837 | NoOp1WrapProblem && | |||
2838 | // Try not to increase register pressure. | |||
2839 | BO0->hasOneUse() && BO1->hasOneUse()) { | |||
2840 | // Determine Y and Z in the form icmp (X+Y), (X+Z). | |||
2841 | Value *Y, *Z; | |||
2842 | if (A == C) { | |||
2843 | // C + B == C + D -> B == D | |||
2844 | Y = B; | |||
2845 | Z = D; | |||
2846 | } else if (A == D) { | |||
2847 | // D + B == C + D -> B == C | |||
2848 | Y = B; | |||
2849 | Z = C; | |||
2850 | } else if (B == C) { | |||
2851 | // A + C == C + D -> A == D | |||
2852 | Y = A; | |||
2853 | Z = D; | |||
2854 | } else { | |||
2855 | assert(B == D)((B == D) ? static_cast<void> (0) : __assert_fail ("B == D" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2855, __PRETTY_FUNCTION__)); | |||
2856 | // A + D == C + D -> A == C | |||
2857 | Y = A; | |||
2858 | Z = C; | |||
2859 | } | |||
2860 | return new ICmpInst(Pred, Y, Z); | |||
2861 | } | |||
2862 | ||||
2863 | // icmp slt (X + -1), Y -> icmp sle X, Y | |||
2864 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLT && | |||
2865 | match(B, m_AllOnes())) | |||
2866 | return new ICmpInst(CmpInst::ICMP_SLE, A, Op1); | |||
2867 | ||||
2868 | // icmp sge (X + -1), Y -> icmp sgt X, Y | |||
2869 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGE && | |||
2870 | match(B, m_AllOnes())) | |||
2871 | return new ICmpInst(CmpInst::ICMP_SGT, A, Op1); | |||
2872 | ||||
2873 | // icmp sle (X + 1), Y -> icmp slt X, Y | |||
2874 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLE && match(B, m_One())) | |||
2875 | return new ICmpInst(CmpInst::ICMP_SLT, A, Op1); | |||
2876 | ||||
2877 | // icmp sgt (X + 1), Y -> icmp sge X, Y | |||
2878 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGT && match(B, m_One())) | |||
2879 | return new ICmpInst(CmpInst::ICMP_SGE, A, Op1); | |||
2880 | ||||
2881 | // icmp sgt X, (Y + -1) -> icmp sge X, Y | |||
2882 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGT && | |||
2883 | match(D, m_AllOnes())) | |||
2884 | return new ICmpInst(CmpInst::ICMP_SGE, Op0, C); | |||
2885 | ||||
2886 | // icmp sle X, (Y + -1) -> icmp slt X, Y | |||
2887 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLE && | |||
2888 | match(D, m_AllOnes())) | |||
2889 | return new ICmpInst(CmpInst::ICMP_SLT, Op0, C); | |||
2890 | ||||
2891 | // icmp sge X, (Y + 1) -> icmp sgt X, Y | |||
2892 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGE && match(D, m_One())) | |||
2893 | return new ICmpInst(CmpInst::ICMP_SGT, Op0, C); | |||
2894 | ||||
2895 | // icmp slt X, (Y + 1) -> icmp sle X, Y | |||
2896 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLT && match(D, m_One())) | |||
2897 | return new ICmpInst(CmpInst::ICMP_SLE, Op0, C); | |||
2898 | ||||
2899 | // TODO: The subtraction-related identities shown below also hold, but | |||
2900 | // canonicalization from (X -nuw 1) to (X + -1) means that the combinations | |||
2901 | // wouldn't happen even if they were implemented. | |||
2902 | // | |||
2903 | // icmp ult (X - 1), Y -> icmp ule X, Y | |||
2904 | // icmp uge (X - 1), Y -> icmp ugt X, Y | |||
2905 | // icmp ugt X, (Y - 1) -> icmp uge X, Y | |||
2906 | // icmp ule X, (Y - 1) -> icmp ult X, Y | |||
2907 | ||||
2908 | // icmp ule (X + 1), Y -> icmp ult X, Y | |||
2909 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_ULE && match(B, m_One())) | |||
2910 | return new ICmpInst(CmpInst::ICMP_ULT, A, Op1); | |||
2911 | ||||
2912 | // icmp ugt (X + 1), Y -> icmp uge X, Y | |||
2913 | if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_UGT && match(B, m_One())) | |||
2914 | return new ICmpInst(CmpInst::ICMP_UGE, A, Op1); | |||
2915 | ||||
2916 | // icmp uge X, (Y + 1) -> icmp ugt X, Y | |||
2917 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_UGE && match(D, m_One())) | |||
2918 | return new ICmpInst(CmpInst::ICMP_UGT, Op0, C); | |||
2919 | ||||
2920 | // icmp ult X, (Y + 1) -> icmp ule X, Y | |||
2921 | if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_ULT && match(D, m_One())) | |||
2922 | return new ICmpInst(CmpInst::ICMP_ULE, Op0, C); | |||
2923 | ||||
2924 | // if C1 has greater magnitude than C2: | |||
2925 | // icmp (X + C1), (Y + C2) -> icmp (X + C3), Y | |||
2926 | // s.t. C3 = C1 - C2 | |||
2927 | // | |||
2928 | // if C2 has greater magnitude than C1: | |||
2929 | // icmp (X + C1), (Y + C2) -> icmp X, (Y + C3) | |||
2930 | // s.t. C3 = C2 - C1 | |||
2931 | if (A && C && NoOp0WrapProblem && NoOp1WrapProblem && | |||
2932 | (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned()) | |||
2933 | if (ConstantInt *C1 = dyn_cast<ConstantInt>(B)) | |||
2934 | if (ConstantInt *C2 = dyn_cast<ConstantInt>(D)) { | |||
2935 | const APInt &AP1 = C1->getValue(); | |||
2936 | const APInt &AP2 = C2->getValue(); | |||
2937 | if (AP1.isNegative() == AP2.isNegative()) { | |||
2938 | APInt AP1Abs = C1->getValue().abs(); | |||
2939 | APInt AP2Abs = C2->getValue().abs(); | |||
2940 | if (AP1Abs.uge(AP2Abs)) { | |||
2941 | ConstantInt *C3 = Builder->getInt(AP1 - AP2); | |||
2942 | Value *NewAdd = Builder->CreateNSWAdd(A, C3); | |||
2943 | return new ICmpInst(Pred, NewAdd, C); | |||
2944 | } else { | |||
2945 | ConstantInt *C3 = Builder->getInt(AP2 - AP1); | |||
2946 | Value *NewAdd = Builder->CreateNSWAdd(C, C3); | |||
2947 | return new ICmpInst(Pred, A, NewAdd); | |||
2948 | } | |||
2949 | } | |||
2950 | } | |||
2951 | ||||
2952 | // Analyze the case when either Op0 or Op1 is a sub instruction. | |||
2953 | // Op0 = A - B (or A and B are null); Op1 = C - D (or C and D are null). | |||
2954 | A = nullptr; | |||
2955 | B = nullptr; | |||
2956 | C = nullptr; | |||
2957 | D = nullptr; | |||
2958 | if (BO0 && BO0->getOpcode() == Instruction::Sub) { | |||
2959 | A = BO0->getOperand(0); | |||
2960 | B = BO0->getOperand(1); | |||
2961 | } | |||
2962 | if (BO1 && BO1->getOpcode() == Instruction::Sub) { | |||
2963 | C = BO1->getOperand(0); | |||
2964 | D = BO1->getOperand(1); | |||
2965 | } | |||
2966 | ||||
2967 | // icmp (X-Y), X -> icmp 0, Y for equalities or if there is no overflow. | |||
2968 | if (A == Op1 && NoOp0WrapProblem) | |||
2969 | return new ICmpInst(Pred, Constant::getNullValue(Op1->getType()), B); | |||
2970 | ||||
2971 | // icmp X, (X-Y) -> icmp Y, 0 for equalities or if there is no overflow. | |||
2972 | if (C == Op0 && NoOp1WrapProblem) | |||
2973 | return new ICmpInst(Pred, D, Constant::getNullValue(Op0->getType())); | |||
2974 | ||||
2975 | // icmp (Y-X), (Z-X) -> icmp Y, Z for equalities or if there is no overflow. | |||
2976 | if (B && D && B == D && NoOp0WrapProblem && NoOp1WrapProblem && | |||
2977 | // Try not to increase register pressure. | |||
2978 | BO0->hasOneUse() && BO1->hasOneUse()) | |||
2979 | return new ICmpInst(Pred, A, C); | |||
2980 | ||||
2981 | // icmp (X-Y), (X-Z) -> icmp Z, Y for equalities or if there is no overflow. | |||
2982 | if (A && C && A == C && NoOp0WrapProblem && NoOp1WrapProblem && | |||
2983 | // Try not to increase register pressure. | |||
2984 | BO0->hasOneUse() && BO1->hasOneUse()) | |||
2985 | return new ICmpInst(Pred, D, B); | |||
2986 | ||||
2987 | // icmp (0-X) < cst --> x > -cst | |||
2988 | if (NoOp0WrapProblem && ICmpInst::isSigned(Pred)) { | |||
2989 | Value *X; | |||
2990 | if (match(BO0, m_Neg(m_Value(X)))) | |||
2991 | if (ConstantInt *RHSC = dyn_cast<ConstantInt>(Op1)) | |||
2992 | if (!RHSC->isMinValue(/*isSigned=*/true)) | |||
2993 | return new ICmpInst(I.getSwappedPredicate(), X, | |||
2994 | ConstantExpr::getNeg(RHSC)); | |||
2995 | } | |||
2996 | ||||
2997 | BinaryOperator *SRem = nullptr; | |||
2998 | // icmp (srem X, Y), Y | |||
2999 | if (BO0 && BO0->getOpcode() == Instruction::SRem && Op1 == BO0->getOperand(1)) | |||
3000 | SRem = BO0; | |||
3001 | // icmp Y, (srem X, Y) | |||
3002 | else if (BO1 && BO1->getOpcode() == Instruction::SRem && | |||
3003 | Op0 == BO1->getOperand(1)) | |||
3004 | SRem = BO1; | |||
3005 | if (SRem) { | |||
3006 | // We don't check hasOneUse to avoid increasing register pressure because | |||
3007 | // the value we use is the same value this instruction was already using. | |||
3008 | switch (SRem == BO0 ? ICmpInst::getSwappedPredicate(Pred) : Pred) { | |||
3009 | default: | |||
3010 | break; | |||
3011 | case ICmpInst::ICMP_EQ: | |||
3012 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | |||
3013 | case ICmpInst::ICMP_NE: | |||
3014 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | |||
3015 | case ICmpInst::ICMP_SGT: | |||
3016 | case ICmpInst::ICMP_SGE: | |||
3017 | return new ICmpInst(ICmpInst::ICMP_SGT, SRem->getOperand(1), | |||
3018 | Constant::getAllOnesValue(SRem->getType())); | |||
3019 | case ICmpInst::ICMP_SLT: | |||
3020 | case ICmpInst::ICMP_SLE: | |||
3021 | return new ICmpInst(ICmpInst::ICMP_SLT, SRem->getOperand(1), | |||
3022 | Constant::getNullValue(SRem->getType())); | |||
3023 | } | |||
3024 | } | |||
3025 | ||||
3026 | if (BO0 && BO1 && BO0->getOpcode() == BO1->getOpcode() && BO0->hasOneUse() && | |||
3027 | BO1->hasOneUse() && BO0->getOperand(1) == BO1->getOperand(1)) { | |||
3028 | switch (BO0->getOpcode()) { | |||
3029 | default: | |||
3030 | break; | |||
3031 | case Instruction::Add: | |||
3032 | case Instruction::Sub: | |||
3033 | case Instruction::Xor: | |||
3034 | if (I.isEquality()) // a+x icmp eq/ne b+x --> a icmp b | |||
3035 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | |||
3036 | // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b | |||
3037 | if (ConstantInt *CI = dyn_cast<ConstantInt>(BO0->getOperand(1))) { | |||
3038 | if (CI->getValue().isSignMask()) { | |||
3039 | ICmpInst::Predicate NewPred = | |||
3040 | I.isSigned() ? I.getUnsignedPredicate() : I.getSignedPredicate(); | |||
3041 | return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0)); | |||
3042 | } | |||
3043 | ||||
3044 | if (BO0->getOpcode() == Instruction::Xor && CI->isMaxValue(true)) { | |||
3045 | ICmpInst::Predicate NewPred = | |||
3046 | I.isSigned() ? I.getUnsignedPredicate() : I.getSignedPredicate(); | |||
3047 | NewPred = I.getSwappedPredicate(NewPred); | |||
3048 | return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0)); | |||
3049 | } | |||
3050 | } | |||
3051 | break; | |||
3052 | case Instruction::Mul: | |||
3053 | if (!I.isEquality()) | |||
3054 | break; | |||
3055 | ||||
3056 | if (ConstantInt *CI = dyn_cast<ConstantInt>(BO0->getOperand(1))) { | |||
3057 | // a * Cst icmp eq/ne b * Cst --> a & Mask icmp b & Mask | |||
3058 | // Mask = -1 >> count-trailing-zeros(Cst). | |||
3059 | if (!CI->isZero() && !CI->isOne()) { | |||
3060 | const APInt &AP = CI->getValue(); | |||
3061 | ConstantInt *Mask = ConstantInt::get( | |||
3062 | I.getContext(), | |||
3063 | APInt::getLowBitsSet(AP.getBitWidth(), | |||
3064 | AP.getBitWidth() - AP.countTrailingZeros())); | |||
3065 | Value *And1 = Builder->CreateAnd(BO0->getOperand(0), Mask); | |||
3066 | Value *And2 = Builder->CreateAnd(BO1->getOperand(0), Mask); | |||
3067 | return new ICmpInst(Pred, And1, And2); | |||
3068 | } | |||
3069 | } | |||
3070 | break; | |||
3071 | ||||
3072 | case Instruction::UDiv: | |||
3073 | case Instruction::LShr: | |||
3074 | if (I.isSigned() || !BO0->isExact() || !BO1->isExact()) | |||
3075 | break; | |||
3076 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | |||
3077 | ||||
3078 | case Instruction::SDiv: | |||
3079 | if (!I.isEquality() || !BO0->isExact() || !BO1->isExact()) | |||
3080 | break; | |||
3081 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | |||
3082 | ||||
3083 | case Instruction::AShr: | |||
3084 | if (!BO0->isExact() || !BO1->isExact()) | |||
3085 | break; | |||
3086 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | |||
3087 | ||||
3088 | case Instruction::Shl: { | |||
3089 | bool NUW = BO0->hasNoUnsignedWrap() && BO1->hasNoUnsignedWrap(); | |||
3090 | bool NSW = BO0->hasNoSignedWrap() && BO1->hasNoSignedWrap(); | |||
3091 | if (!NUW && !NSW) | |||
3092 | break; | |||
3093 | if (!NSW && I.isSigned()) | |||
3094 | break; | |||
3095 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | |||
3096 | } | |||
3097 | } | |||
3098 | } | |||
3099 | ||||
3100 | if (BO0) { | |||
3101 | // Transform A & (L - 1) `ult` L --> L != 0 | |||
3102 | auto LSubOne = m_Add(m_Specific(Op1), m_AllOnes()); | |||
3103 | auto BitwiseAnd = | |||
3104 | m_CombineOr(m_And(m_Value(), LSubOne), m_And(LSubOne, m_Value())); | |||
3105 | ||||
3106 | if (match(BO0, BitwiseAnd) && Pred == ICmpInst::ICMP_ULT) { | |||
3107 | auto *Zero = Constant::getNullValue(BO0->getType()); | |||
3108 | return new ICmpInst(ICmpInst::ICMP_NE, Op1, Zero); | |||
3109 | } | |||
3110 | } | |||
3111 | ||||
3112 | return nullptr; | |||
3113 | } | |||
3114 | ||||
3115 | /// Fold icmp Pred min|max(X, Y), X. | |||
3116 | static Instruction *foldICmpWithMinMax(ICmpInst &Cmp) { | |||
3117 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | |||
3118 | Value *Op0 = Cmp.getOperand(0); | |||
3119 | Value *X = Cmp.getOperand(1); | |||
3120 | ||||
3121 | // Canonicalize minimum or maximum operand to LHS of the icmp. | |||
3122 | if (match(X, m_c_SMin(m_Specific(Op0), m_Value())) || | |||
3123 | match(X, m_c_SMax(m_Specific(Op0), m_Value())) || | |||
3124 | match(X, m_c_UMin(m_Specific(Op0), m_Value())) || | |||
3125 | match(X, m_c_UMax(m_Specific(Op0), m_Value()))) { | |||
3126 | std::swap(Op0, X); | |||
3127 | Pred = Cmp.getSwappedPredicate(); | |||
3128 | } | |||
3129 | ||||
3130 | Value *Y; | |||
3131 | if (match(Op0, m_c_SMin(m_Specific(X), m_Value(Y)))) { | |||
3132 | // smin(X, Y) == X --> X s<= Y | |||
3133 | // smin(X, Y) s>= X --> X s<= Y | |||
3134 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SGE) | |||
3135 | return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); | |||
3136 | ||||
3137 | // smin(X, Y) != X --> X s> Y | |||
3138 | // smin(X, Y) s< X --> X s> Y | |||
3139 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SLT) | |||
3140 | return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); | |||
3141 | ||||
3142 | // These cases should be handled in InstSimplify: | |||
3143 | // smin(X, Y) s<= X --> true | |||
3144 | // smin(X, Y) s> X --> false | |||
3145 | return nullptr; | |||
3146 | } | |||
3147 | ||||
3148 | if (match(Op0, m_c_SMax(m_Specific(X), m_Value(Y)))) { | |||
3149 | // smax(X, Y) == X --> X s>= Y | |||
3150 | // smax(X, Y) s<= X --> X s>= Y | |||
3151 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SLE) | |||
3152 | return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); | |||
3153 | ||||
3154 | // smax(X, Y) != X --> X s< Y | |||
3155 | // smax(X, Y) s> X --> X s< Y | |||
3156 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SGT) | |||
3157 | return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); | |||
3158 | ||||
3159 | // These cases should be handled in InstSimplify: | |||
3160 | // smax(X, Y) s>= X --> true | |||
3161 | // smax(X, Y) s< X --> false | |||
3162 | return nullptr; | |||
3163 | } | |||
3164 | ||||
3165 | if (match(Op0, m_c_UMin(m_Specific(X), m_Value(Y)))) { | |||
3166 | // umin(X, Y) == X --> X u<= Y | |||
3167 | // umin(X, Y) u>= X --> X u<= Y | |||
3168 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_UGE) | |||
3169 | return new ICmpInst(ICmpInst::ICMP_ULE, X, Y); | |||
3170 | ||||
3171 | // umin(X, Y) != X --> X u> Y | |||
3172 | // umin(X, Y) u< X --> X u> Y | |||
3173 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_ULT) | |||
3174 | return new ICmpInst(ICmpInst::ICMP_UGT, X, Y); | |||
3175 | ||||
3176 | // These cases should be handled in InstSimplify: | |||
3177 | // umin(X, Y) u<= X --> true | |||
3178 | // umin(X, Y) u> X --> false | |||
3179 | return nullptr; | |||
3180 | } | |||
3181 | ||||
3182 | if (match(Op0, m_c_UMax(m_Specific(X), m_Value(Y)))) { | |||
3183 | // umax(X, Y) == X --> X u>= Y | |||
3184 | // umax(X, Y) u<= X --> X u>= Y | |||
3185 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_ULE) | |||
3186 | return new ICmpInst(ICmpInst::ICMP_UGE, X, Y); | |||
3187 | ||||
3188 | // umax(X, Y) != X --> X u< Y | |||
3189 | // umax(X, Y) u> X --> X u< Y | |||
3190 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_UGT) | |||
3191 | return new ICmpInst(ICmpInst::ICMP_ULT, X, Y); | |||
3192 | ||||
3193 | // These cases should be handled in InstSimplify: | |||
3194 | // umax(X, Y) u>= X --> true | |||
3195 | // umax(X, Y) u< X --> false | |||
3196 | return nullptr; | |||
3197 | } | |||
3198 | ||||
3199 | return nullptr; | |||
3200 | } | |||
3201 | ||||
3202 | Instruction *InstCombiner::foldICmpEquality(ICmpInst &I) { | |||
3203 | if (!I.isEquality()) | |||
3204 | return nullptr; | |||
3205 | ||||
3206 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
3207 | Value *A, *B, *C, *D; | |||
3208 | if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) { | |||
3209 | if (A == Op1 || B == Op1) { // (A^B) == A -> B == 0 | |||
3210 | Value *OtherVal = A == Op1 ? B : A; | |||
3211 | return new ICmpInst(I.getPredicate(), OtherVal, | |||
3212 | Constant::getNullValue(A->getType())); | |||
3213 | } | |||
3214 | ||||
3215 | if (match(Op1, m_Xor(m_Value(C), m_Value(D)))) { | |||
3216 | // A^c1 == C^c2 --> A == C^(c1^c2) | |||
3217 | ConstantInt *C1, *C2; | |||
3218 | if (match(B, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2)) && | |||
3219 | Op1->hasOneUse()) { | |||
3220 | Constant *NC = Builder->getInt(C1->getValue() ^ C2->getValue()); | |||
3221 | Value *Xor = Builder->CreateXor(C, NC); | |||
3222 | return new ICmpInst(I.getPredicate(), A, Xor); | |||
3223 | } | |||
3224 | ||||
3225 | // A^B == A^D -> B == D | |||
3226 | if (A == C) | |||
3227 | return new ICmpInst(I.getPredicate(), B, D); | |||
3228 | if (A == D) | |||
3229 | return new ICmpInst(I.getPredicate(), B, C); | |||
3230 | if (B == C) | |||
3231 | return new ICmpInst(I.getPredicate(), A, D); | |||
3232 | if (B == D) | |||
3233 | return new ICmpInst(I.getPredicate(), A, C); | |||
3234 | } | |||
3235 | } | |||
3236 | ||||
3237 | if (match(Op1, m_Xor(m_Value(A), m_Value(B))) && (A == Op0 || B == Op0)) { | |||
3238 | // A == (A^B) -> B == 0 | |||
3239 | Value *OtherVal = A == Op0 ? B : A; | |||
3240 | return new ICmpInst(I.getPredicate(), OtherVal, | |||
3241 | Constant::getNullValue(A->getType())); | |||
3242 | } | |||
3243 | ||||
3244 | // (X&Z) == (Y&Z) -> (X^Y) & Z == 0 | |||
3245 | if (match(Op0, m_OneUse(m_And(m_Value(A), m_Value(B)))) && | |||
3246 | match(Op1, m_OneUse(m_And(m_Value(C), m_Value(D))))) { | |||
3247 | Value *X = nullptr, *Y = nullptr, *Z = nullptr; | |||
3248 | ||||
3249 | if (A == C) { | |||
3250 | X = B; | |||
3251 | Y = D; | |||
3252 | Z = A; | |||
3253 | } else if (A == D) { | |||
3254 | X = B; | |||
3255 | Y = C; | |||
3256 | Z = A; | |||
3257 | } else if (B == C) { | |||
3258 | X = A; | |||
3259 | Y = D; | |||
3260 | Z = B; | |||
3261 | } else if (B == D) { | |||
3262 | X = A; | |||
3263 | Y = C; | |||
3264 | Z = B; | |||
3265 | } | |||
3266 | ||||
3267 | if (X) { // Build (X^Y) & Z | |||
3268 | Op1 = Builder->CreateXor(X, Y); | |||
3269 | Op1 = Builder->CreateAnd(Op1, Z); | |||
3270 | I.setOperand(0, Op1); | |||
3271 | I.setOperand(1, Constant::getNullValue(Op1->getType())); | |||
3272 | return &I; | |||
3273 | } | |||
3274 | } | |||
3275 | ||||
3276 | // Transform (zext A) == (B & (1<<X)-1) --> A == (trunc B) | |||
3277 | // and (B & (1<<X)-1) == (zext A) --> A == (trunc B) | |||
3278 | ConstantInt *Cst1; | |||
3279 | if ((Op0->hasOneUse() && match(Op0, m_ZExt(m_Value(A))) && | |||
3280 | match(Op1, m_And(m_Value(B), m_ConstantInt(Cst1)))) || | |||
3281 | (Op1->hasOneUse() && match(Op0, m_And(m_Value(B), m_ConstantInt(Cst1))) && | |||
3282 | match(Op1, m_ZExt(m_Value(A))))) { | |||
3283 | APInt Pow2 = Cst1->getValue() + 1; | |||
3284 | if (Pow2.isPowerOf2() && isa<IntegerType>(A->getType()) && | |||
3285 | Pow2.logBase2() == cast<IntegerType>(A->getType())->getBitWidth()) | |||
3286 | return new ICmpInst(I.getPredicate(), A, | |||
3287 | Builder->CreateTrunc(B, A->getType())); | |||
3288 | } | |||
3289 | ||||
3290 | // (A >> C) == (B >> C) --> (A^B) u< (1 << C) | |||
3291 | // For lshr and ashr pairs. | |||
3292 | if ((match(Op0, m_OneUse(m_LShr(m_Value(A), m_ConstantInt(Cst1)))) && | |||
3293 | match(Op1, m_OneUse(m_LShr(m_Value(B), m_Specific(Cst1))))) || | |||
3294 | (match(Op0, m_OneUse(m_AShr(m_Value(A), m_ConstantInt(Cst1)))) && | |||
3295 | match(Op1, m_OneUse(m_AShr(m_Value(B), m_Specific(Cst1)))))) { | |||
3296 | unsigned TypeBits = Cst1->getBitWidth(); | |||
3297 | unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits); | |||
3298 | if (ShAmt < TypeBits && ShAmt != 0) { | |||
3299 | ICmpInst::Predicate Pred = I.getPredicate() == ICmpInst::ICMP_NE | |||
3300 | ? ICmpInst::ICMP_UGE | |||
3301 | : ICmpInst::ICMP_ULT; | |||
3302 | Value *Xor = Builder->CreateXor(A, B, I.getName() + ".unshifted"); | |||
3303 | APInt CmpVal = APInt::getOneBitSet(TypeBits, ShAmt); | |||
3304 | return new ICmpInst(Pred, Xor, Builder->getInt(CmpVal)); | |||
3305 | } | |||
3306 | } | |||
3307 | ||||
3308 | // (A << C) == (B << C) --> ((A^B) & (~0U >> C)) == 0 | |||
3309 | if (match(Op0, m_OneUse(m_Shl(m_Value(A), m_ConstantInt(Cst1)))) && | |||
3310 | match(Op1, m_OneUse(m_Shl(m_Value(B), m_Specific(Cst1))))) { | |||
3311 | unsigned TypeBits = Cst1->getBitWidth(); | |||
3312 | unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits); | |||
3313 | if (ShAmt < TypeBits && ShAmt != 0) { | |||
3314 | Value *Xor = Builder->CreateXor(A, B, I.getName() + ".unshifted"); | |||
3315 | APInt AndVal = APInt::getLowBitsSet(TypeBits, TypeBits - ShAmt); | |||
3316 | Value *And = Builder->CreateAnd(Xor, Builder->getInt(AndVal), | |||
3317 | I.getName() + ".mask"); | |||
3318 | return new ICmpInst(I.getPredicate(), And, | |||
3319 | Constant::getNullValue(Cst1->getType())); | |||
3320 | } | |||
3321 | } | |||
3322 | ||||
3323 | // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to | |||
3324 | // "icmp (and X, mask), cst" | |||
3325 | uint64_t ShAmt = 0; | |||
3326 | if (Op0->hasOneUse() && | |||
3327 | match(Op0, m_Trunc(m_OneUse(m_LShr(m_Value(A), m_ConstantInt(ShAmt))))) && | |||
3328 | match(Op1, m_ConstantInt(Cst1)) && | |||
3329 | // Only do this when A has multiple uses. This is most important to do | |||
3330 | // when it exposes other optimizations. | |||
3331 | !A->hasOneUse()) { | |||
3332 | unsigned ASize = cast<IntegerType>(A->getType())->getPrimitiveSizeInBits(); | |||
3333 | ||||
3334 | if (ShAmt < ASize) { | |||
3335 | APInt MaskV = | |||
3336 | APInt::getLowBitsSet(ASize, Op0->getType()->getPrimitiveSizeInBits()); | |||
3337 | MaskV <<= ShAmt; | |||
3338 | ||||
3339 | APInt CmpV = Cst1->getValue().zext(ASize); | |||
3340 | CmpV <<= ShAmt; | |||
3341 | ||||
3342 | Value *Mask = Builder->CreateAnd(A, Builder->getInt(MaskV)); | |||
3343 | return new ICmpInst(I.getPredicate(), Mask, Builder->getInt(CmpV)); | |||
3344 | } | |||
3345 | } | |||
3346 | ||||
3347 | return nullptr; | |||
3348 | } | |||
3349 | ||||
3350 | /// Handle icmp (cast x to y), (cast/cst). We only handle extending casts so | |||
3351 | /// far. | |||
3352 | Instruction *InstCombiner::foldICmpWithCastAndCast(ICmpInst &ICmp) { | |||
3353 | const CastInst *LHSCI = cast<CastInst>(ICmp.getOperand(0)); | |||
3354 | Value *LHSCIOp = LHSCI->getOperand(0); | |||
3355 | Type *SrcTy = LHSCIOp->getType(); | |||
3356 | Type *DestTy = LHSCI->getType(); | |||
3357 | Value *RHSCIOp; | |||
3358 | ||||
3359 | // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the | |||
3360 | // integer type is the same size as the pointer type. | |||
3361 | if (LHSCI->getOpcode() == Instruction::PtrToInt && | |||
3362 | DL.getPointerTypeSizeInBits(SrcTy) == DestTy->getIntegerBitWidth()) { | |||
3363 | Value *RHSOp = nullptr; | |||
3364 | if (auto *RHSC = dyn_cast<PtrToIntOperator>(ICmp.getOperand(1))) { | |||
3365 | Value *RHSCIOp = RHSC->getOperand(0); | |||
3366 | if (RHSCIOp->getType()->getPointerAddressSpace() == | |||
3367 | LHSCIOp->getType()->getPointerAddressSpace()) { | |||
3368 | RHSOp = RHSC->getOperand(0); | |||
3369 | // If the pointer types don't match, insert a bitcast. | |||
3370 | if (LHSCIOp->getType() != RHSOp->getType()) | |||
3371 | RHSOp = Builder->CreateBitCast(RHSOp, LHSCIOp->getType()); | |||
3372 | } | |||
3373 | } else if (auto *RHSC = dyn_cast<Constant>(ICmp.getOperand(1))) { | |||
3374 | RHSOp = ConstantExpr::getIntToPtr(RHSC, SrcTy); | |||
3375 | } | |||
3376 | ||||
3377 | if (RHSOp) | |||
3378 | return new ICmpInst(ICmp.getPredicate(), LHSCIOp, RHSOp); | |||
3379 | } | |||
3380 | ||||
3381 | // The code below only handles extension cast instructions, so far. | |||
3382 | // Enforce this. | |||
3383 | if (LHSCI->getOpcode() != Instruction::ZExt && | |||
3384 | LHSCI->getOpcode() != Instruction::SExt) | |||
3385 | return nullptr; | |||
3386 | ||||
3387 | bool isSignedExt = LHSCI->getOpcode() == Instruction::SExt; | |||
3388 | bool isSignedCmp = ICmp.isSigned(); | |||
3389 | ||||
3390 | if (auto *CI = dyn_cast<CastInst>(ICmp.getOperand(1))) { | |||
3391 | // Not an extension from the same type? | |||
3392 | RHSCIOp = CI->getOperand(0); | |||
3393 | if (RHSCIOp->getType() != LHSCIOp->getType()) | |||
3394 | return nullptr; | |||
3395 | ||||
3396 | // If the signedness of the two casts doesn't agree (i.e. one is a sext | |||
3397 | // and the other is a zext), then we can't handle this. | |||
3398 | if (CI->getOpcode() != LHSCI->getOpcode()) | |||
3399 | return nullptr; | |||
3400 | ||||
3401 | // Deal with equality cases early. | |||
3402 | if (ICmp.isEquality()) | |||
3403 | return new ICmpInst(ICmp.getPredicate(), LHSCIOp, RHSCIOp); | |||
3404 | ||||
3405 | // A signed comparison of sign extended values simplifies into a | |||
3406 | // signed comparison. | |||
3407 | if (isSignedCmp && isSignedExt) | |||
3408 | return new ICmpInst(ICmp.getPredicate(), LHSCIOp, RHSCIOp); | |||
3409 | ||||
3410 | // The other three cases all fold into an unsigned comparison. | |||
3411 | return new ICmpInst(ICmp.getUnsignedPredicate(), LHSCIOp, RHSCIOp); | |||
3412 | } | |||
3413 | ||||
3414 | // If we aren't dealing with a constant on the RHS, exit early. | |||
3415 | auto *C = dyn_cast<Constant>(ICmp.getOperand(1)); | |||
3416 | if (!C) | |||
3417 | return nullptr; | |||
3418 | ||||
3419 | // Compute the constant that would happen if we truncated to SrcTy then | |||
3420 | // re-extended to DestTy. | |||
3421 | Constant *Res1 = ConstantExpr::getTrunc(C, SrcTy); | |||
3422 | Constant *Res2 = ConstantExpr::getCast(LHSCI->getOpcode(), Res1, DestTy); | |||
3423 | ||||
3424 | // If the re-extended constant didn't change... | |||
3425 | if (Res2 == C) { | |||
3426 | // Deal with equality cases early. | |||
3427 | if (ICmp.isEquality()) | |||
3428 | return new ICmpInst(ICmp.getPredicate(), LHSCIOp, Res1); | |||
3429 | ||||
3430 | // A signed comparison of sign extended values simplifies into a | |||
3431 | // signed comparison. | |||
3432 | if (isSignedExt && isSignedCmp) | |||
3433 | return new ICmpInst(ICmp.getPredicate(), LHSCIOp, Res1); | |||
3434 | ||||
3435 | // The other three cases all fold into an unsigned comparison. | |||
3436 | return new ICmpInst(ICmp.getUnsignedPredicate(), LHSCIOp, Res1); | |||
3437 | } | |||
3438 | ||||
3439 | // The re-extended constant changed, partly changed (in the case of a vector), | |||
3440 | // or could not be determined to be equal (in the case of a constant | |||
3441 | // expression), so the constant cannot be represented in the shorter type. | |||
3442 | // Consequently, we cannot emit a simple comparison. | |||
3443 | // All the cases that fold to true or false will have already been handled | |||
3444 | // by SimplifyICmpInst, so only deal with the tricky case. | |||
3445 | ||||
3446 | if (isSignedCmp || !isSignedExt || !isa<ConstantInt>(C)) | |||
3447 | return nullptr; | |||
3448 | ||||
3449 | // Evaluate the comparison for LT (we invert for GT below). LE and GE cases | |||
3450 | // should have been folded away previously and not enter in here. | |||
3451 | ||||
3452 | // We're performing an unsigned comp with a sign extended value. | |||
3453 | // This is true if the input is >= 0. [aka >s -1] | |||
3454 | Constant *NegOne = Constant::getAllOnesValue(SrcTy); | |||
3455 | Value *Result = Builder->CreateICmpSGT(LHSCIOp, NegOne, ICmp.getName()); | |||
3456 | ||||
3457 | // Finally, return the value computed. | |||
3458 | if (ICmp.getPredicate() == ICmpInst::ICMP_ULT) | |||
3459 | return replaceInstUsesWith(ICmp, Result); | |||
3460 | ||||
3461 | assert(ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!")((ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!" ) ? static_cast<void> (0) : __assert_fail ("ICmp.getPredicate() == ICmpInst::ICMP_UGT && \"ICmp should be folded!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3461, __PRETTY_FUNCTION__)); | |||
3462 | return BinaryOperator::CreateNot(Result); | |||
3463 | } | |||
3464 | ||||
3465 | bool InstCombiner::OptimizeOverflowCheck(OverflowCheckFlavor OCF, Value *LHS, | |||
3466 | Value *RHS, Instruction &OrigI, | |||
3467 | Value *&Result, Constant *&Overflow) { | |||
3468 | if (OrigI.isCommutative() && isa<Constant>(LHS) && !isa<Constant>(RHS)) | |||
3469 | std::swap(LHS, RHS); | |||
3470 | ||||
3471 | auto SetResult = [&](Value *OpResult, Constant *OverflowVal, bool ReuseName) { | |||
3472 | Result = OpResult; | |||
3473 | Overflow = OverflowVal; | |||
3474 | if (ReuseName) | |||
3475 | Result->takeName(&OrigI); | |||
3476 | return true; | |||
3477 | }; | |||
3478 | ||||
3479 | // If the overflow check was an add followed by a compare, the insertion point | |||
3480 | // may be pointing to the compare. We want to insert the new instructions | |||
3481 | // before the add in case there are uses of the add between the add and the | |||
3482 | // compare. | |||
3483 | Builder->SetInsertPoint(&OrigI); | |||
3484 | ||||
3485 | switch (OCF) { | |||
3486 | case OCF_INVALID: | |||
3487 | llvm_unreachable("bad overflow check kind!")::llvm::llvm_unreachable_internal("bad overflow check kind!", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3487); | |||
3488 | ||||
3489 | case OCF_UNSIGNED_ADD: { | |||
3490 | OverflowResult OR = computeOverflowForUnsignedAdd(LHS, RHS, &OrigI); | |||
3491 | if (OR == OverflowResult::NeverOverflows) | |||
3492 | return SetResult(Builder->CreateNUWAdd(LHS, RHS), Builder->getFalse(), | |||
3493 | true); | |||
3494 | ||||
3495 | if (OR == OverflowResult::AlwaysOverflows) | |||
3496 | return SetResult(Builder->CreateAdd(LHS, RHS), Builder->getTrue(), true); | |||
3497 | ||||
3498 | // Fall through uadd into sadd | |||
3499 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
3500 | } | |||
3501 | case OCF_SIGNED_ADD: { | |||
3502 | // X + 0 -> {X, false} | |||
3503 | if (match(RHS, m_Zero())) | |||
3504 | return SetResult(LHS, Builder->getFalse(), false); | |||
3505 | ||||
3506 | // We can strength reduce this signed add into a regular add if we can prove | |||
3507 | // that it will never overflow. | |||
3508 | if (OCF == OCF_SIGNED_ADD) | |||
3509 | if (WillNotOverflowSignedAdd(LHS, RHS, OrigI)) | |||
3510 | return SetResult(Builder->CreateNSWAdd(LHS, RHS), Builder->getFalse(), | |||
3511 | true); | |||
3512 | break; | |||
3513 | } | |||
3514 | ||||
3515 | case OCF_UNSIGNED_SUB: | |||
3516 | case OCF_SIGNED_SUB: { | |||
3517 | // X - 0 -> {X, false} | |||
3518 | if (match(RHS, m_Zero())) | |||
3519 | return SetResult(LHS, Builder->getFalse(), false); | |||
3520 | ||||
3521 | if (OCF == OCF_SIGNED_SUB) { | |||
3522 | if (WillNotOverflowSignedSub(LHS, RHS, OrigI)) | |||
3523 | return SetResult(Builder->CreateNSWSub(LHS, RHS), Builder->getFalse(), | |||
3524 | true); | |||
3525 | } else { | |||
3526 | if (WillNotOverflowUnsignedSub(LHS, RHS, OrigI)) | |||
3527 | return SetResult(Builder->CreateNUWSub(LHS, RHS), Builder->getFalse(), | |||
3528 | true); | |||
3529 | } | |||
3530 | break; | |||
3531 | } | |||
3532 | ||||
3533 | case OCF_UNSIGNED_MUL: { | |||
3534 | OverflowResult OR = computeOverflowForUnsignedMul(LHS, RHS, &OrigI); | |||
3535 | if (OR == OverflowResult::NeverOverflows) | |||
3536 | return SetResult(Builder->CreateNUWMul(LHS, RHS), Builder->getFalse(), | |||
3537 | true); | |||
3538 | if (OR == OverflowResult::AlwaysOverflows) | |||
3539 | return SetResult(Builder->CreateMul(LHS, RHS), Builder->getTrue(), true); | |||
3540 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
3541 | } | |||
3542 | case OCF_SIGNED_MUL: | |||
3543 | // X * undef -> undef | |||
3544 | if (isa<UndefValue>(RHS)) | |||
3545 | return SetResult(RHS, UndefValue::get(Builder->getInt1Ty()), false); | |||
3546 | ||||
3547 | // X * 0 -> {0, false} | |||
3548 | if (match(RHS, m_Zero())) | |||
3549 | return SetResult(RHS, Builder->getFalse(), false); | |||
3550 | ||||
3551 | // X * 1 -> {X, false} | |||
3552 | if (match(RHS, m_One())) | |||
3553 | return SetResult(LHS, Builder->getFalse(), false); | |||
3554 | ||||
3555 | if (OCF == OCF_SIGNED_MUL) | |||
3556 | if (WillNotOverflowSignedMul(LHS, RHS, OrigI)) | |||
3557 | return SetResult(Builder->CreateNSWMul(LHS, RHS), Builder->getFalse(), | |||
3558 | true); | |||
3559 | break; | |||
3560 | } | |||
3561 | ||||
3562 | return false; | |||
3563 | } | |||
3564 | ||||
3565 | /// \brief Recognize and process idiom involving test for multiplication | |||
3566 | /// overflow. | |||
3567 | /// | |||
3568 | /// The caller has matched a pattern of the form: | |||
3569 | /// I = cmp u (mul(zext A, zext B), V | |||
3570 | /// The function checks if this is a test for overflow and if so replaces | |||
3571 | /// multiplication with call to 'mul.with.overflow' intrinsic. | |||
3572 | /// | |||
3573 | /// \param I Compare instruction. | |||
3574 | /// \param MulVal Result of 'mult' instruction. It is one of the arguments of | |||
3575 | /// the compare instruction. Must be of integer type. | |||
3576 | /// \param OtherVal The other argument of compare instruction. | |||
3577 | /// \returns Instruction which must replace the compare instruction, NULL if no | |||
3578 | /// replacement required. | |||
3579 | static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal, | |||
3580 | Value *OtherVal, InstCombiner &IC) { | |||
3581 | // Don't bother doing this transformation for pointers, don't do it for | |||
3582 | // vectors. | |||
3583 | if (!isa<IntegerType>(MulVal->getType())) | |||
3584 | return nullptr; | |||
3585 | ||||
3586 | assert(I.getOperand(0) == MulVal || I.getOperand(1) == MulVal)((I.getOperand(0) == MulVal || I.getOperand(1) == MulVal) ? static_cast <void> (0) : __assert_fail ("I.getOperand(0) == MulVal || I.getOperand(1) == MulVal" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3586, __PRETTY_FUNCTION__)); | |||
3587 | assert(I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal)((I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal) ? static_cast<void> (0) : __assert_fail ("I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3587, __PRETTY_FUNCTION__)); | |||
3588 | auto *MulInstr = dyn_cast<Instruction>(MulVal); | |||
3589 | if (!MulInstr) | |||
3590 | return nullptr; | |||
3591 | assert(MulInstr->getOpcode() == Instruction::Mul)((MulInstr->getOpcode() == Instruction::Mul) ? static_cast <void> (0) : __assert_fail ("MulInstr->getOpcode() == Instruction::Mul" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3591, __PRETTY_FUNCTION__)); | |||
3592 | ||||
3593 | auto *LHS = cast<ZExtOperator>(MulInstr->getOperand(0)), | |||
3594 | *RHS = cast<ZExtOperator>(MulInstr->getOperand(1)); | |||
3595 | assert(LHS->getOpcode() == Instruction::ZExt)((LHS->getOpcode() == Instruction::ZExt) ? static_cast< void> (0) : __assert_fail ("LHS->getOpcode() == Instruction::ZExt" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3595, __PRETTY_FUNCTION__)); | |||
3596 | assert(RHS->getOpcode() == Instruction::ZExt)((RHS->getOpcode() == Instruction::ZExt) ? static_cast< void> (0) : __assert_fail ("RHS->getOpcode() == Instruction::ZExt" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3596, __PRETTY_FUNCTION__)); | |||
3597 | Value *A = LHS->getOperand(0), *B = RHS->getOperand(0); | |||
3598 | ||||
3599 | // Calculate type and width of the result produced by mul.with.overflow. | |||
3600 | Type *TyA = A->getType(), *TyB = B->getType(); | |||
3601 | unsigned WidthA = TyA->getPrimitiveSizeInBits(), | |||
3602 | WidthB = TyB->getPrimitiveSizeInBits(); | |||
3603 | unsigned MulWidth; | |||
3604 | Type *MulType; | |||
3605 | if (WidthB > WidthA) { | |||
3606 | MulWidth = WidthB; | |||
3607 | MulType = TyB; | |||
3608 | } else { | |||
3609 | MulWidth = WidthA; | |||
3610 | MulType = TyA; | |||
3611 | } | |||
3612 | ||||
3613 | // In order to replace the original mul with a narrower mul.with.overflow, | |||
3614 | // all uses must ignore upper bits of the product. The number of used low | |||
3615 | // bits must be not greater than the width of mul.with.overflow. | |||
3616 | if (MulVal->hasNUsesOrMore(2)) | |||
3617 | for (User *U : MulVal->users()) { | |||
3618 | if (U == &I) | |||
3619 | continue; | |||
3620 | if (TruncInst *TI = dyn_cast<TruncInst>(U)) { | |||
3621 | // Check if truncation ignores bits above MulWidth. | |||
3622 | unsigned TruncWidth = TI->getType()->getPrimitiveSizeInBits(); | |||
3623 | if (TruncWidth > MulWidth) | |||
3624 | return nullptr; | |||
3625 | } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) { | |||
3626 | // Check if AND ignores bits above MulWidth. | |||
3627 | if (BO->getOpcode() != Instruction::And) | |||
3628 | return nullptr; | |||
3629 | if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->getOperand(1))) { | |||
3630 | const APInt &CVal = CI->getValue(); | |||
3631 | if (CVal.getBitWidth() - CVal.countLeadingZeros() > MulWidth) | |||
3632 | return nullptr; | |||
3633 | } | |||
3634 | } else { | |||
3635 | // Other uses prohibit this transformation. | |||
3636 | return nullptr; | |||
3637 | } | |||
3638 | } | |||
3639 | ||||
3640 | // Recognize patterns | |||
3641 | switch (I.getPredicate()) { | |||
3642 | case ICmpInst::ICMP_EQ: | |||
3643 | case ICmpInst::ICMP_NE: | |||
3644 | // Recognize pattern: | |||
3645 | // mulval = mul(zext A, zext B) | |||
3646 | // cmp eq/neq mulval, zext trunc mulval | |||
3647 | if (ZExtInst *Zext = dyn_cast<ZExtInst>(OtherVal)) | |||
3648 | if (Zext->hasOneUse()) { | |||
3649 | Value *ZextArg = Zext->getOperand(0); | |||
3650 | if (TruncInst *Trunc = dyn_cast<TruncInst>(ZextArg)) | |||
3651 | if (Trunc->getType()->getPrimitiveSizeInBits() == MulWidth) | |||
3652 | break; //Recognized | |||
3653 | } | |||
3654 | ||||
3655 | // Recognize pattern: | |||
3656 | // mulval = mul(zext A, zext B) | |||
3657 | // cmp eq/neq mulval, and(mulval, mask), mask selects low MulWidth bits. | |||
3658 | ConstantInt *CI; | |||
3659 | Value *ValToMask; | |||
3660 | if (match(OtherVal, m_And(m_Value(ValToMask), m_ConstantInt(CI)))) { | |||
3661 | if (ValToMask != MulVal) | |||
3662 | return nullptr; | |||
3663 | const APInt &CVal = CI->getValue() + 1; | |||
3664 | if (CVal.isPowerOf2()) { | |||
3665 | unsigned MaskWidth = CVal.logBase2(); | |||
3666 | if (MaskWidth == MulWidth) | |||
3667 | break; // Recognized | |||
3668 | } | |||
3669 | } | |||
3670 | return nullptr; | |||
3671 | ||||
3672 | case ICmpInst::ICMP_UGT: | |||
3673 | // Recognize pattern: | |||
3674 | // mulval = mul(zext A, zext B) | |||
3675 | // cmp ugt mulval, max | |||
3676 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | |||
3677 | APInt MaxVal = APInt::getMaxValue(MulWidth); | |||
3678 | MaxVal = MaxVal.zext(CI->getBitWidth()); | |||
3679 | if (MaxVal.eq(CI->getValue())) | |||
3680 | break; // Recognized | |||
3681 | } | |||
3682 | return nullptr; | |||
3683 | ||||
3684 | case ICmpInst::ICMP_UGE: | |||
3685 | // Recognize pattern: | |||
3686 | // mulval = mul(zext A, zext B) | |||
3687 | // cmp uge mulval, max+1 | |||
3688 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | |||
3689 | APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth); | |||
3690 | if (MaxVal.eq(CI->getValue())) | |||
3691 | break; // Recognized | |||
3692 | } | |||
3693 | return nullptr; | |||
3694 | ||||
3695 | case ICmpInst::ICMP_ULE: | |||
3696 | // Recognize pattern: | |||
3697 | // mulval = mul(zext A, zext B) | |||
3698 | // cmp ule mulval, max | |||
3699 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | |||
3700 | APInt MaxVal = APInt::getMaxValue(MulWidth); | |||
3701 | MaxVal = MaxVal.zext(CI->getBitWidth()); | |||
3702 | if (MaxVal.eq(CI->getValue())) | |||
3703 | break; // Recognized | |||
3704 | } | |||
3705 | return nullptr; | |||
3706 | ||||
3707 | case ICmpInst::ICMP_ULT: | |||
3708 | // Recognize pattern: | |||
3709 | // mulval = mul(zext A, zext B) | |||
3710 | // cmp ule mulval, max + 1 | |||
3711 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | |||
3712 | APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth); | |||
3713 | if (MaxVal.eq(CI->getValue())) | |||
3714 | break; // Recognized | |||
3715 | } | |||
3716 | return nullptr; | |||
3717 | ||||
3718 | default: | |||
3719 | return nullptr; | |||
3720 | } | |||
3721 | ||||
3722 | InstCombiner::BuilderTy *Builder = IC.Builder; | |||
3723 | Builder->SetInsertPoint(MulInstr); | |||
3724 | ||||
3725 | // Replace: mul(zext A, zext B) --> mul.with.overflow(A, B) | |||
3726 | Value *MulA = A, *MulB = B; | |||
3727 | if (WidthA < MulWidth) | |||
3728 | MulA = Builder->CreateZExt(A, MulType); | |||
3729 | if (WidthB < MulWidth) | |||
3730 | MulB = Builder->CreateZExt(B, MulType); | |||
3731 | Value *F = Intrinsic::getDeclaration(I.getModule(), | |||
3732 | Intrinsic::umul_with_overflow, MulType); | |||
3733 | CallInst *Call = Builder->CreateCall(F, {MulA, MulB}, "umul"); | |||
3734 | IC.Worklist.Add(MulInstr); | |||
3735 | ||||
3736 | // If there are uses of mul result other than the comparison, we know that | |||
3737 | // they are truncation or binary AND. Change them to use result of | |||
3738 | // mul.with.overflow and adjust properly mask/size. | |||
3739 | if (MulVal->hasNUsesOrMore(2)) { | |||
3740 | Value *Mul = Builder->CreateExtractValue(Call, 0, "umul.value"); | |||
3741 | for (User *U : MulVal->users()) { | |||
3742 | if (U == &I || U == OtherVal) | |||
3743 | continue; | |||
3744 | if (TruncInst *TI = dyn_cast<TruncInst>(U)) { | |||
3745 | if (TI->getType()->getPrimitiveSizeInBits() == MulWidth) | |||
3746 | IC.replaceInstUsesWith(*TI, Mul); | |||
3747 | else | |||
3748 | TI->setOperand(0, Mul); | |||
3749 | } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) { | |||
3750 | assert(BO->getOpcode() == Instruction::And)((BO->getOpcode() == Instruction::And) ? static_cast<void > (0) : __assert_fail ("BO->getOpcode() == Instruction::And" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3750, __PRETTY_FUNCTION__)); | |||
3751 | // Replace (mul & mask) --> zext (mul.with.overflow & short_mask) | |||
3752 | ConstantInt *CI = cast<ConstantInt>(BO->getOperand(1)); | |||
3753 | APInt ShortMask = CI->getValue().trunc(MulWidth); | |||
3754 | Value *ShortAnd = Builder->CreateAnd(Mul, ShortMask); | |||
3755 | Instruction *Zext = | |||
3756 | cast<Instruction>(Builder->CreateZExt(ShortAnd, BO->getType())); | |||
3757 | IC.Worklist.Add(Zext); | |||
3758 | IC.replaceInstUsesWith(*BO, Zext); | |||
3759 | } else { | |||
3760 | llvm_unreachable("Unexpected Binary operation")::llvm::llvm_unreachable_internal("Unexpected Binary operation" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3760); | |||
3761 | } | |||
3762 | IC.Worklist.Add(cast<Instruction>(U)); | |||
3763 | } | |||
3764 | } | |||
3765 | if (isa<Instruction>(OtherVal)) | |||
3766 | IC.Worklist.Add(cast<Instruction>(OtherVal)); | |||
3767 | ||||
3768 | // The original icmp gets replaced with the overflow value, maybe inverted | |||
3769 | // depending on predicate. | |||
3770 | bool Inverse = false; | |||
3771 | switch (I.getPredicate()) { | |||
3772 | case ICmpInst::ICMP_NE: | |||
3773 | break; | |||
3774 | case ICmpInst::ICMP_EQ: | |||
3775 | Inverse = true; | |||
3776 | break; | |||
3777 | case ICmpInst::ICMP_UGT: | |||
3778 | case ICmpInst::ICMP_UGE: | |||
3779 | if (I.getOperand(0) == MulVal) | |||
3780 | break; | |||
3781 | Inverse = true; | |||
3782 | break; | |||
3783 | case ICmpInst::ICMP_ULT: | |||
3784 | case ICmpInst::ICMP_ULE: | |||
3785 | if (I.getOperand(1) == MulVal) | |||
3786 | break; | |||
3787 | Inverse = true; | |||
3788 | break; | |||
3789 | default: | |||
3790 | llvm_unreachable("Unexpected predicate")::llvm::llvm_unreachable_internal("Unexpected predicate", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3790); | |||
3791 | } | |||
3792 | if (Inverse) { | |||
3793 | Value *Res = Builder->CreateExtractValue(Call, 1); | |||
3794 | return BinaryOperator::CreateNot(Res); | |||
3795 | } | |||
3796 | ||||
3797 | return ExtractValueInst::Create(Call, 1); | |||
3798 | } | |||
3799 | ||||
3800 | /// When performing a comparison against a constant, it is possible that not all | |||
3801 | /// the bits in the LHS are demanded. This helper method computes the mask that | |||
3802 | /// IS demanded. | |||
3803 | static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth, | |||
3804 | bool isSignCheck) { | |||
3805 | if (isSignCheck) | |||
3806 | return APInt::getSignMask(BitWidth); | |||
3807 | ||||
3808 | ConstantInt *CI = dyn_cast<ConstantInt>(I.getOperand(1)); | |||
3809 | if (!CI) return APInt::getAllOnesValue(BitWidth); | |||
3810 | const APInt &RHS = CI->getValue(); | |||
3811 | ||||
3812 | switch (I.getPredicate()) { | |||
3813 | // For a UGT comparison, we don't care about any bits that | |||
3814 | // correspond to the trailing ones of the comparand. The value of these | |||
3815 | // bits doesn't impact the outcome of the comparison, because any value | |||
3816 | // greater than the RHS must differ in a bit higher than these due to carry. | |||
3817 | case ICmpInst::ICMP_UGT: { | |||
3818 | unsigned trailingOnes = RHS.countTrailingOnes(); | |||
3819 | return APInt::getBitsSetFrom(BitWidth, trailingOnes); | |||
3820 | } | |||
3821 | ||||
3822 | // Similarly, for a ULT comparison, we don't care about the trailing zeros. | |||
3823 | // Any value less than the RHS must differ in a higher bit because of carries. | |||
3824 | case ICmpInst::ICMP_ULT: { | |||
3825 | unsigned trailingZeros = RHS.countTrailingZeros(); | |||
3826 | return APInt::getBitsSetFrom(BitWidth, trailingZeros); | |||
3827 | } | |||
3828 | ||||
3829 | default: | |||
3830 | return APInt::getAllOnesValue(BitWidth); | |||
3831 | } | |||
3832 | } | |||
3833 | ||||
3834 | /// \brief Check if the order of \p Op0 and \p Op1 as operand in an ICmpInst | |||
3835 | /// should be swapped. | |||
3836 | /// The decision is based on how many times these two operands are reused | |||
3837 | /// as subtract operands and their positions in those instructions. | |||
3838 | /// The rational is that several architectures use the same instruction for | |||
3839 | /// both subtract and cmp, thus it is better if the order of those operands | |||
3840 | /// match. | |||
3841 | /// \return true if Op0 and Op1 should be swapped. | |||
3842 | static bool swapMayExposeCSEOpportunities(const Value * Op0, | |||
3843 | const Value * Op1) { | |||
3844 | // Filter out pointer value as those cannot appears directly in subtract. | |||
3845 | // FIXME: we may want to go through inttoptrs or bitcasts. | |||
3846 | if (Op0->getType()->isPointerTy()) | |||
3847 | return false; | |||
3848 | // Count every uses of both Op0 and Op1 in a subtract. | |||
3849 | // Each time Op0 is the first operand, count -1: swapping is bad, the | |||
3850 | // subtract has already the same layout as the compare. | |||
3851 | // Each time Op0 is the second operand, count +1: swapping is good, the | |||
3852 | // subtract has a different layout as the compare. | |||
3853 | // At the end, if the benefit is greater than 0, Op0 should come second to | |||
3854 | // expose more CSE opportunities. | |||
3855 | int GlobalSwapBenefits = 0; | |||
3856 | for (const User *U : Op0->users()) { | |||
3857 | const BinaryOperator *BinOp = dyn_cast<BinaryOperator>(U); | |||
3858 | if (!BinOp || BinOp->getOpcode() != Instruction::Sub) | |||
3859 | continue; | |||
3860 | // If Op0 is the first argument, this is not beneficial to swap the | |||
3861 | // arguments. | |||
3862 | int LocalSwapBenefits = -1; | |||
3863 | unsigned Op1Idx = 1; | |||
3864 | if (BinOp->getOperand(Op1Idx) == Op0) { | |||
3865 | Op1Idx = 0; | |||
3866 | LocalSwapBenefits = 1; | |||
3867 | } | |||
3868 | if (BinOp->getOperand(Op1Idx) != Op1) | |||
3869 | continue; | |||
3870 | GlobalSwapBenefits += LocalSwapBenefits; | |||
3871 | } | |||
3872 | return GlobalSwapBenefits > 0; | |||
3873 | } | |||
3874 | ||||
3875 | /// \brief Check that one use is in the same block as the definition and all | |||
3876 | /// other uses are in blocks dominated by a given block. | |||
3877 | /// | |||
3878 | /// \param DI Definition | |||
3879 | /// \param UI Use | |||
3880 | /// \param DB Block that must dominate all uses of \p DI outside | |||
3881 | /// the parent block | |||
3882 | /// \return true when \p UI is the only use of \p DI in the parent block | |||
3883 | /// and all other uses of \p DI are in blocks dominated by \p DB. | |||
3884 | /// | |||
3885 | bool InstCombiner::dominatesAllUses(const Instruction *DI, | |||
3886 | const Instruction *UI, | |||
3887 | const BasicBlock *DB) const { | |||
3888 | assert(DI && UI && "Instruction not defined\n")((DI && UI && "Instruction not defined\n") ? static_cast <void> (0) : __assert_fail ("DI && UI && \"Instruction not defined\\n\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3888, __PRETTY_FUNCTION__)); | |||
3889 | // Ignore incomplete definitions. | |||
3890 | if (!DI->getParent()) | |||
3891 | return false; | |||
3892 | // DI and UI must be in the same block. | |||
3893 | if (DI->getParent() != UI->getParent()) | |||
3894 | return false; | |||
3895 | // Protect from self-referencing blocks. | |||
3896 | if (DI->getParent() == DB) | |||
3897 | return false; | |||
3898 | for (const User *U : DI->users()) { | |||
3899 | auto *Usr = cast<Instruction>(U); | |||
3900 | if (Usr != UI && !DT.dominates(DB, Usr->getParent())) | |||
3901 | return false; | |||
3902 | } | |||
3903 | return true; | |||
3904 | } | |||
3905 | ||||
3906 | /// Return true when the instruction sequence within a block is select-cmp-br. | |||
3907 | static bool isChainSelectCmpBranch(const SelectInst *SI) { | |||
3908 | const BasicBlock *BB = SI->getParent(); | |||
3909 | if (!BB) | |||
3910 | return false; | |||
3911 | auto *BI = dyn_cast_or_null<BranchInst>(BB->getTerminator()); | |||
3912 | if (!BI || BI->getNumSuccessors() != 2) | |||
3913 | return false; | |||
3914 | auto *IC = dyn_cast<ICmpInst>(BI->getCondition()); | |||
3915 | if (!IC || (IC->getOperand(0) != SI && IC->getOperand(1) != SI)) | |||
3916 | return false; | |||
3917 | return true; | |||
3918 | } | |||
3919 | ||||
3920 | /// \brief True when a select result is replaced by one of its operands | |||
3921 | /// in select-icmp sequence. This will eventually result in the elimination | |||
3922 | /// of the select. | |||
3923 | /// | |||
3924 | /// \param SI Select instruction | |||
3925 | /// \param Icmp Compare instruction | |||
3926 | /// \param SIOpd Operand that replaces the select | |||
3927 | /// | |||
3928 | /// Notes: | |||
3929 | /// - The replacement is global and requires dominator information | |||
3930 | /// - The caller is responsible for the actual replacement | |||
3931 | /// | |||
3932 | /// Example: | |||
3933 | /// | |||
3934 | /// entry: | |||
3935 | /// %4 = select i1 %3, %C* %0, %C* null | |||
3936 | /// %5 = icmp eq %C* %4, null | |||
3937 | /// br i1 %5, label %9, label %7 | |||
3938 | /// ... | |||
3939 | /// ; <label>:7 ; preds = %entry | |||
3940 | /// %8 = getelementptr inbounds %C* %4, i64 0, i32 0 | |||
3941 | /// ... | |||
3942 | /// | |||
3943 | /// can be transformed to | |||
3944 | /// | |||
3945 | /// %5 = icmp eq %C* %0, null | |||
3946 | /// %6 = select i1 %3, i1 %5, i1 true | |||
3947 | /// br i1 %6, label %9, label %7 | |||
3948 | /// ... | |||
3949 | /// ; <label>:7 ; preds = %entry | |||
3950 | /// %8 = getelementptr inbounds %C* %0, i64 0, i32 0 // replace by %0! | |||
3951 | /// | |||
3952 | /// Similar when the first operand of the select is a constant or/and | |||
3953 | /// the compare is for not equal rather than equal. | |||
3954 | /// | |||
3955 | /// NOTE: The function is only called when the select and compare constants | |||
3956 | /// are equal, the optimization can work only for EQ predicates. This is not a | |||
3957 | /// major restriction since a NE compare should be 'normalized' to an equal | |||
3958 | /// compare, which usually happens in the combiner and test case | |||
3959 | /// select-cmp-br.ll checks for it. | |||
3960 | bool InstCombiner::replacedSelectWithOperand(SelectInst *SI, | |||
3961 | const ICmpInst *Icmp, | |||
3962 | const unsigned SIOpd) { | |||
3963 | assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!")(((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!" ) ? static_cast<void> (0) : __assert_fail ("(SIOpd == 1 || SIOpd == 2) && \"Invalid select operand!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3963, __PRETTY_FUNCTION__)); | |||
3964 | if (isChainSelectCmpBranch(SI) && Icmp->getPredicate() == ICmpInst::ICMP_EQ) { | |||
3965 | BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(1); | |||
3966 | // The check for the single predecessor is not the best that can be | |||
3967 | // done. But it protects efficiently against cases like when SI's | |||
3968 | // home block has two successors, Succ and Succ1, and Succ1 predecessor | |||
3969 | // of Succ. Then SI can't be replaced by SIOpd because the use that gets | |||
3970 | // replaced can be reached on either path. So the uniqueness check | |||
3971 | // guarantees that the path all uses of SI (outside SI's parent) are on | |||
3972 | // is disjoint from all other paths out of SI. But that information | |||
3973 | // is more expensive to compute, and the trade-off here is in favor | |||
3974 | // of compile-time. It should also be noticed that we check for a single | |||
3975 | // predecessor and not only uniqueness. This to handle the situation when | |||
3976 | // Succ and Succ1 points to the same basic block. | |||
3977 | if (Succ->getSinglePredecessor() && dominatesAllUses(SI, Icmp, Succ)) { | |||
3978 | NumSel++; | |||
3979 | SI->replaceUsesOutsideBlock(SI->getOperand(SIOpd), SI->getParent()); | |||
3980 | return true; | |||
3981 | } | |||
3982 | } | |||
3983 | return false; | |||
3984 | } | |||
3985 | ||||
3986 | /// Try to fold the comparison based on range information we can get by checking | |||
3987 | /// whether bits are known to be zero or one in the inputs. | |||
3988 | Instruction *InstCombiner::foldICmpUsingKnownBits(ICmpInst &I) { | |||
3989 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
3990 | Type *Ty = Op0->getType(); | |||
3991 | ICmpInst::Predicate Pred = I.getPredicate(); | |||
3992 | ||||
3993 | // Get scalar or pointer size. | |||
3994 | unsigned BitWidth = Ty->isIntOrIntVectorTy() | |||
3995 | ? Ty->getScalarSizeInBits() | |||
3996 | : DL.getTypeSizeInBits(Ty->getScalarType()); | |||
3997 | ||||
3998 | if (!BitWidth) | |||
3999 | return nullptr; | |||
4000 | ||||
4001 | // If this is a normal comparison, it demands all bits. If it is a sign bit | |||
4002 | // comparison, it only demands the sign bit. | |||
4003 | bool IsSignBit = false; | |||
4004 | const APInt *CmpC; | |||
4005 | if (match(Op1, m_APInt(CmpC))) { | |||
4006 | bool UnusedBit; | |||
4007 | IsSignBit = isSignBitCheck(Pred, *CmpC, UnusedBit); | |||
4008 | } | |||
4009 | ||||
4010 | KnownBits Op0Known(BitWidth); | |||
4011 | KnownBits Op1Known(BitWidth); | |||
4012 | ||||
4013 | if (SimplifyDemandedBits(&I, 0, | |||
4014 | getDemandedBitsLHSMask(I, BitWidth, IsSignBit), | |||
4015 | Op0Known, 0)) | |||
4016 | return &I; | |||
4017 | ||||
4018 | if (SimplifyDemandedBits(&I, 1, APInt::getAllOnesValue(BitWidth), | |||
4019 | Op1Known, 0)) | |||
4020 | return &I; | |||
4021 | ||||
4022 | // Given the known and unknown bits, compute a range that the LHS could be | |||
4023 | // in. Compute the Min, Max and RHS values based on the known bits. For the | |||
4024 | // EQ and NE we use unsigned values. | |||
4025 | APInt Op0Min(BitWidth, 0), Op0Max(BitWidth, 0); | |||
4026 | APInt Op1Min(BitWidth, 0), Op1Max(BitWidth, 0); | |||
4027 | if (I.isSigned()) { | |||
4028 | computeSignedMinMaxValuesFromKnownBits(Op0Known, Op0Min, Op0Max); | |||
4029 | computeSignedMinMaxValuesFromKnownBits(Op1Known, Op1Min, Op1Max); | |||
4030 | } else { | |||
4031 | computeUnsignedMinMaxValuesFromKnownBits(Op0Known, Op0Min, Op0Max); | |||
4032 | computeUnsignedMinMaxValuesFromKnownBits(Op1Known, Op1Min, Op1Max); | |||
4033 | } | |||
4034 | ||||
4035 | // If Min and Max are known to be the same, then SimplifyDemandedBits | |||
4036 | // figured out that the LHS is a constant. Constant fold this now, so that | |||
4037 | // code below can assume that Min != Max. | |||
4038 | if (!isa<Constant>(Op0) && Op0Min == Op0Max) | |||
4039 | return new ICmpInst(Pred, ConstantInt::get(Op0->getType(), Op0Min), Op1); | |||
4040 | if (!isa<Constant>(Op1) && Op1Min == Op1Max) | |||
4041 | return new ICmpInst(Pred, Op0, ConstantInt::get(Op1->getType(), Op1Min)); | |||
4042 | ||||
4043 | // Based on the range information we know about the LHS, see if we can | |||
4044 | // simplify this comparison. For example, (x&4) < 8 is always true. | |||
4045 | switch (Pred) { | |||
4046 | default: | |||
4047 | llvm_unreachable("Unknown icmp opcode!")::llvm::llvm_unreachable_internal("Unknown icmp opcode!", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4047); | |||
4048 | case ICmpInst::ICMP_EQ: | |||
4049 | case ICmpInst::ICMP_NE: { | |||
4050 | if (Op0Max.ult(Op1Min) || Op0Min.ugt(Op1Max)) { | |||
4051 | return Pred == CmpInst::ICMP_EQ | |||
4052 | ? replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())) | |||
4053 | : replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | |||
4054 | } | |||
4055 | ||||
4056 | // If all bits are known zero except for one, then we know at most one bit | |||
4057 | // is set. If the comparison is against zero, then this is a check to see if | |||
4058 | // *that* bit is set. | |||
4059 | APInt Op0KnownZeroInverted = ~Op0Known.Zero; | |||
4060 | if (Op1Known.isZero()) { | |||
4061 | // If the LHS is an AND with the same constant, look through it. | |||
4062 | Value *LHS = nullptr; | |||
4063 | const APInt *LHSC; | |||
4064 | if (!match(Op0, m_And(m_Value(LHS), m_APInt(LHSC))) || | |||
4065 | *LHSC != Op0KnownZeroInverted) | |||
4066 | LHS = Op0; | |||
4067 | ||||
4068 | Value *X; | |||
4069 | if (match(LHS, m_Shl(m_One(), m_Value(X)))) { | |||
4070 | APInt ValToCheck = Op0KnownZeroInverted; | |||
4071 | Type *XTy = X->getType(); | |||
4072 | if (ValToCheck.isPowerOf2()) { | |||
4073 | // ((1 << X) & 8) == 0 -> X != 3 | |||
4074 | // ((1 << X) & 8) != 0 -> X == 3 | |||
4075 | auto *CmpC = ConstantInt::get(XTy, ValToCheck.countTrailingZeros()); | |||
4076 | auto NewPred = ICmpInst::getInversePredicate(Pred); | |||
4077 | return new ICmpInst(NewPred, X, CmpC); | |||
4078 | } else if ((++ValToCheck).isPowerOf2()) { | |||
4079 | // ((1 << X) & 7) == 0 -> X >= 3 | |||
4080 | // ((1 << X) & 7) != 0 -> X < 3 | |||
4081 | auto *CmpC = ConstantInt::get(XTy, ValToCheck.countTrailingZeros()); | |||
4082 | auto NewPred = | |||
4083 | Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGE : CmpInst::ICMP_ULT; | |||
4084 | return new ICmpInst(NewPred, X, CmpC); | |||
4085 | } | |||
4086 | } | |||
4087 | ||||
4088 | // Check if the LHS is 8 >>u x and the result is a power of 2 like 1. | |||
4089 | const APInt *CI; | |||
4090 | if (Op0KnownZeroInverted == 1 && | |||
4091 | match(LHS, m_LShr(m_Power2(CI), m_Value(X)))) { | |||
4092 | // ((8 >>u X) & 1) == 0 -> X != 3 | |||
4093 | // ((8 >>u X) & 1) != 0 -> X == 3 | |||
4094 | unsigned CmpVal = CI->countTrailingZeros(); | |||
4095 | auto NewPred = ICmpInst::getInversePredicate(Pred); | |||
4096 | return new ICmpInst(NewPred, X, ConstantInt::get(X->getType(), CmpVal)); | |||
4097 | } | |||
4098 | } | |||
4099 | break; | |||
4100 | } | |||
4101 | case ICmpInst::ICMP_ULT: { | |||
4102 | if (Op0Max.ult(Op1Min)) // A <u B -> true if max(A) < min(B) | |||
4103 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | |||
4104 | if (Op0Min.uge(Op1Max)) // A <u B -> false if min(A) >= max(B) | |||
4105 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | |||
4106 | if (Op1Min == Op0Max) // A <u B -> A != B if max(A) == min(B) | |||
4107 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | |||
4108 | ||||
4109 | const APInt *CmpC; | |||
4110 | if (match(Op1, m_APInt(CmpC))) { | |||
4111 | // A <u C -> A == C-1 if min(A)+1 == C | |||
4112 | if (Op1Max == Op0Min + 1) { | |||
4113 | Constant *CMinus1 = ConstantInt::get(Op0->getType(), *CmpC - 1); | |||
4114 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, CMinus1); | |||
4115 | } | |||
4116 | } | |||
4117 | break; | |||
4118 | } | |||
4119 | case ICmpInst::ICMP_UGT: { | |||
4120 | if (Op0Min.ugt(Op1Max)) // A >u B -> true if min(A) > max(B) | |||
4121 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | |||
4122 | ||||
4123 | if (Op0Max.ule(Op1Min)) // A >u B -> false if max(A) <= max(B) | |||
4124 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | |||
4125 | ||||
4126 | if (Op1Max == Op0Min) // A >u B -> A != B if min(A) == max(B) | |||
4127 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | |||
4128 | ||||
4129 | const APInt *CmpC; | |||
4130 | if (match(Op1, m_APInt(CmpC))) { | |||
4131 | // A >u C -> A == C+1 if max(a)-1 == C | |||
4132 | if (*CmpC == Op0Max - 1) | |||
4133 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | |||
4134 | ConstantInt::get(Op1->getType(), *CmpC + 1)); | |||
4135 | } | |||
4136 | break; | |||
4137 | } | |||
4138 | case ICmpInst::ICMP_SLT: | |||
4139 | if (Op0Max.slt(Op1Min)) // A <s B -> true if max(A) < min(C) | |||
4140 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | |||
4141 | if (Op0Min.sge(Op1Max)) // A <s B -> false if min(A) >= max(C) | |||
4142 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | |||
4143 | if (Op1Min == Op0Max) // A <s B -> A != B if max(A) == min(B) | |||
4144 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | |||
4145 | if (ConstantInt *CI = dyn_cast<ConstantInt>(Op1)) { | |||
4146 | if (Op1Max == Op0Min + 1) // A <s C -> A == C-1 if min(A)+1 == C | |||
4147 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | |||
4148 | Builder->getInt(CI->getValue() - 1)); | |||
4149 | } | |||
4150 | break; | |||
4151 | case ICmpInst::ICMP_SGT: | |||
4152 | if (Op0Min.sgt(Op1Max)) // A >s B -> true if min(A) > max(B) | |||
4153 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | |||
4154 | if (Op0Max.sle(Op1Min)) // A >s B -> false if max(A) <= min(B) | |||
4155 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | |||
4156 | ||||
4157 | if (Op1Max == Op0Min) // A >s B -> A != B if min(A) == max(B) | |||
4158 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | |||
4159 | if (ConstantInt *CI = dyn_cast<ConstantInt>(Op1)) { | |||
4160 | if (Op1Min == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C | |||
4161 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | |||
4162 | Builder->getInt(CI->getValue() + 1)); | |||
4163 | } | |||
4164 | break; | |||
4165 | case ICmpInst::ICMP_SGE: | |||
4166 | assert(!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!")((!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!" ) ? static_cast<void> (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_SGE with ConstantInt not folded!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4166, __PRETTY_FUNCTION__)); | |||
4167 | if (Op0Min.sge(Op1Max)) // A >=s B -> true if min(A) >= max(B) | |||
4168 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | |||
4169 | if (Op0Max.slt(Op1Min)) // A >=s B -> false if max(A) < min(B) | |||
4170 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | |||
4171 | break; | |||
4172 | case ICmpInst::ICMP_SLE: | |||
4173 | assert(!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!")((!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!" ) ? static_cast<void> (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_SLE with ConstantInt not folded!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4173, __PRETTY_FUNCTION__)); | |||
4174 | if (Op0Max.sle(Op1Min)) // A <=s B -> true if max(A) <= min(B) | |||
4175 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | |||
4176 | if (Op0Min.sgt(Op1Max)) // A <=s B -> false if min(A) > max(B) | |||
4177 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | |||
4178 | break; | |||
4179 | case ICmpInst::ICMP_UGE: | |||
4180 | assert(!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!")((!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!" ) ? static_cast<void> (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_UGE with ConstantInt not folded!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4180, __PRETTY_FUNCTION__)); | |||
4181 | if (Op0Min.uge(Op1Max)) // A >=u B -> true if min(A) >= max(B) | |||
4182 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | |||
4183 | if (Op0Max.ult(Op1Min)) // A >=u B -> false if max(A) < min(B) | |||
4184 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | |||
4185 | break; | |||
4186 | case ICmpInst::ICMP_ULE: | |||
4187 | assert(!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!")((!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!" ) ? static_cast<void> (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_ULE with ConstantInt not folded!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4187, __PRETTY_FUNCTION__)); | |||
4188 | if (Op0Max.ule(Op1Min)) // A <=u B -> true if max(A) <= min(B) | |||
4189 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | |||
4190 | if (Op0Min.ugt(Op1Max)) // A <=u B -> false if min(A) > max(B) | |||
4191 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | |||
4192 | break; | |||
4193 | } | |||
4194 | ||||
4195 | // Turn a signed comparison into an unsigned one if both operands are known to | |||
4196 | // have the same sign. | |||
4197 | if (I.isSigned() && | |||
4198 | ((Op0Known.Zero.isNegative() && Op1Known.Zero.isNegative()) || | |||
4199 | (Op0Known.One.isNegative() && Op1Known.One.isNegative()))) | |||
4200 | return new ICmpInst(I.getUnsignedPredicate(), Op0, Op1); | |||
4201 | ||||
4202 | return nullptr; | |||
4203 | } | |||
4204 | ||||
4205 | /// If we have an icmp le or icmp ge instruction with a constant operand, turn | |||
4206 | /// it into the appropriate icmp lt or icmp gt instruction. This transform | |||
4207 | /// allows them to be folded in visitICmpInst. | |||
4208 | static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) { | |||
4209 | ICmpInst::Predicate Pred = I.getPredicate(); | |||
4210 | if (Pred != ICmpInst::ICMP_SLE && Pred != ICmpInst::ICMP_SGE && | |||
4211 | Pred != ICmpInst::ICMP_ULE && Pred != ICmpInst::ICMP_UGE) | |||
4212 | return nullptr; | |||
4213 | ||||
4214 | Value *Op0 = I.getOperand(0); | |||
4215 | Value *Op1 = I.getOperand(1); | |||
4216 | auto *Op1C = dyn_cast<Constant>(Op1); | |||
4217 | if (!Op1C) | |||
4218 | return nullptr; | |||
4219 | ||||
4220 | // Check if the constant operand can be safely incremented/decremented without | |||
4221 | // overflowing/underflowing. For scalars, SimplifyICmpInst has already handled | |||
4222 | // the edge cases for us, so we just assert on them. For vectors, we must | |||
4223 | // handle the edge cases. | |||
4224 | Type *Op1Type = Op1->getType(); | |||
4225 | bool IsSigned = I.isSigned(); | |||
4226 | bool IsLE = (Pred == ICmpInst::ICMP_SLE || Pred == ICmpInst::ICMP_ULE); | |||
4227 | auto *CI = dyn_cast<ConstantInt>(Op1C); | |||
4228 | if (CI) { | |||
4229 | // A <= MAX -> TRUE ; A >= MIN -> TRUE | |||
4230 | assert(IsLE ? !CI->isMaxValue(IsSigned) : !CI->isMinValue(IsSigned))((IsLE ? !CI->isMaxValue(IsSigned) : !CI->isMinValue(IsSigned )) ? static_cast<void> (0) : __assert_fail ("IsLE ? !CI->isMaxValue(IsSigned) : !CI->isMinValue(IsSigned)" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4230, __PRETTY_FUNCTION__)); | |||
4231 | } else if (Op1Type->isVectorTy()) { | |||
4232 | // TODO? If the edge cases for vectors were guaranteed to be handled as they | |||
4233 | // are for scalar, we could remove the min/max checks. However, to do that, | |||
4234 | // we would have to use insertelement/shufflevector to replace edge values. | |||
4235 | unsigned NumElts = Op1Type->getVectorNumElements(); | |||
4236 | for (unsigned i = 0; i != NumElts; ++i) { | |||
4237 | Constant *Elt = Op1C->getAggregateElement(i); | |||
4238 | if (!Elt) | |||
4239 | return nullptr; | |||
4240 | ||||
4241 | if (isa<UndefValue>(Elt)) | |||
4242 | continue; | |||
4243 | ||||
4244 | // Bail out if we can't determine if this constant is min/max or if we | |||
4245 | // know that this constant is min/max. | |||
4246 | auto *CI = dyn_cast<ConstantInt>(Elt); | |||
4247 | if (!CI || (IsLE ? CI->isMaxValue(IsSigned) : CI->isMinValue(IsSigned))) | |||
4248 | return nullptr; | |||
4249 | } | |||
4250 | } else { | |||
4251 | // ConstantExpr? | |||
4252 | return nullptr; | |||
4253 | } | |||
4254 | ||||
4255 | // Increment or decrement the constant and set the new comparison predicate: | |||
4256 | // ULE -> ULT ; UGE -> UGT ; SLE -> SLT ; SGE -> SGT | |||
4257 | Constant *OneOrNegOne = ConstantInt::get(Op1Type, IsLE ? 1 : -1, true); | |||
4258 | CmpInst::Predicate NewPred = IsLE ? ICmpInst::ICMP_ULT: ICmpInst::ICMP_UGT; | |||
4259 | NewPred = IsSigned ? ICmpInst::getSignedPredicate(NewPred) : NewPred; | |||
4260 | return new ICmpInst(NewPred, Op0, ConstantExpr::getAdd(Op1C, OneOrNegOne)); | |||
4261 | } | |||
4262 | ||||
4263 | /// Integer compare with boolean values can always be turned into bitwise ops. | |||
4264 | static Instruction *canonicalizeICmpBool(ICmpInst &I, | |||
4265 | InstCombiner::BuilderTy &Builder) { | |||
4266 | Value *A = I.getOperand(0), *B = I.getOperand(1); | |||
4267 | assert(A->getType()->getScalarType()->isIntegerTy(1) && "Bools only")((A->getType()->getScalarType()->isIntegerTy(1) && "Bools only") ? static_cast<void> (0) : __assert_fail ( "A->getType()->getScalarType()->isIntegerTy(1) && \"Bools only\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4267, __PRETTY_FUNCTION__)); | |||
4268 | ||||
4269 | // A boolean compared to true/false can be simplified to Op0/true/false in | |||
4270 | // 14 out of the 20 (10 predicates * 2 constants) possible combinations. | |||
4271 | // Cases not handled by InstSimplify are always 'not' of Op0. | |||
4272 | if (match(B, m_Zero())) { | |||
4273 | switch (I.getPredicate()) { | |||
4274 | case CmpInst::ICMP_EQ: // A == 0 -> !A | |||
4275 | case CmpInst::ICMP_ULE: // A <=u 0 -> !A | |||
4276 | case CmpInst::ICMP_SGE: // A >=s 0 -> !A | |||
4277 | return BinaryOperator::CreateNot(A); | |||
4278 | default: | |||
4279 | llvm_unreachable("ICmp i1 X, C not simplified as expected.")::llvm::llvm_unreachable_internal("ICmp i1 X, C not simplified as expected." , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4279); | |||
4280 | } | |||
4281 | } else if (match(B, m_One())) { | |||
4282 | switch (I.getPredicate()) { | |||
4283 | case CmpInst::ICMP_NE: // A != 1 -> !A | |||
4284 | case CmpInst::ICMP_ULT: // A <u 1 -> !A | |||
4285 | case CmpInst::ICMP_SGT: // A >s -1 -> !A | |||
4286 | return BinaryOperator::CreateNot(A); | |||
4287 | default: | |||
4288 | llvm_unreachable("ICmp i1 X, C not simplified as expected.")::llvm::llvm_unreachable_internal("ICmp i1 X, C not simplified as expected." , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4288); | |||
4289 | } | |||
4290 | } | |||
4291 | ||||
4292 | switch (I.getPredicate()) { | |||
4293 | default: | |||
4294 | llvm_unreachable("Invalid icmp instruction!")::llvm::llvm_unreachable_internal("Invalid icmp instruction!" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4294); | |||
4295 | case ICmpInst::ICMP_EQ: | |||
4296 | // icmp eq i1 A, B -> ~(A ^ B) | |||
4297 | return BinaryOperator::CreateNot(Builder.CreateXor(A, B)); | |||
4298 | ||||
4299 | case ICmpInst::ICMP_NE: | |||
4300 | // icmp ne i1 A, B -> A ^ B | |||
4301 | return BinaryOperator::CreateXor(A, B); | |||
4302 | ||||
4303 | case ICmpInst::ICMP_UGT: | |||
4304 | // icmp ugt -> icmp ult | |||
4305 | std::swap(A, B); | |||
4306 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
4307 | case ICmpInst::ICMP_ULT: | |||
4308 | // icmp ult i1 A, B -> ~A & B | |||
4309 | return BinaryOperator::CreateAnd(Builder.CreateNot(A), B); | |||
4310 | ||||
4311 | case ICmpInst::ICMP_SGT: | |||
4312 | // icmp sgt -> icmp slt | |||
4313 | std::swap(A, B); | |||
4314 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
4315 | case ICmpInst::ICMP_SLT: | |||
4316 | // icmp slt i1 A, B -> A & ~B | |||
4317 | return BinaryOperator::CreateAnd(Builder.CreateNot(B), A); | |||
4318 | ||||
4319 | case ICmpInst::ICMP_UGE: | |||
4320 | // icmp uge -> icmp ule | |||
4321 | std::swap(A, B); | |||
4322 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
4323 | case ICmpInst::ICMP_ULE: | |||
4324 | // icmp ule i1 A, B -> ~A | B | |||
4325 | return BinaryOperator::CreateOr(Builder.CreateNot(A), B); | |||
4326 | ||||
4327 | case ICmpInst::ICMP_SGE: | |||
4328 | // icmp sge -> icmp sle | |||
4329 | std::swap(A, B); | |||
4330 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
4331 | case ICmpInst::ICMP_SLE: | |||
4332 | // icmp sle i1 A, B -> A | ~B | |||
4333 | return BinaryOperator::CreateOr(Builder.CreateNot(B), A); | |||
4334 | } | |||
4335 | } | |||
4336 | ||||
4337 | Instruction *InstCombiner::visitICmpInst(ICmpInst &I) { | |||
4338 | bool Changed = false; | |||
4339 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
4340 | unsigned Op0Cplxity = getComplexity(Op0); | |||
4341 | unsigned Op1Cplxity = getComplexity(Op1); | |||
4342 | ||||
4343 | /// Orders the operands of the compare so that they are listed from most | |||
4344 | /// complex to least complex. This puts constants before unary operators, | |||
4345 | /// before binary operators. | |||
4346 | if (Op0Cplxity < Op1Cplxity || | |||
4347 | (Op0Cplxity == Op1Cplxity && swapMayExposeCSEOpportunities(Op0, Op1))) { | |||
4348 | I.swapOperands(); | |||
4349 | std::swap(Op0, Op1); | |||
4350 | Changed = true; | |||
4351 | } | |||
4352 | ||||
4353 | if (Value *V = SimplifyICmpInst(I.getPredicate(), Op0, Op1, | |||
4354 | SQ.getWithInstruction(&I))) | |||
4355 | return replaceInstUsesWith(I, V); | |||
4356 | ||||
4357 | // comparing -val or val with non-zero is the same as just comparing val | |||
4358 | // ie, abs(val) != 0 -> val != 0 | |||
4359 | if (I.getPredicate() == ICmpInst::ICMP_NE && match(Op1, m_Zero())) { | |||
4360 | Value *Cond, *SelectTrue, *SelectFalse; | |||
4361 | if (match(Op0, m_Select(m_Value(Cond), m_Value(SelectTrue), | |||
4362 | m_Value(SelectFalse)))) { | |||
4363 | if (Value *V = dyn_castNegVal(SelectTrue)) { | |||
4364 | if (V == SelectFalse) | |||
4365 | return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1); | |||
4366 | } | |||
4367 | else if (Value *V = dyn_castNegVal(SelectFalse)) { | |||
4368 | if (V == SelectTrue) | |||
4369 | return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1); | |||
4370 | } | |||
4371 | } | |||
4372 | } | |||
4373 | ||||
4374 | if (Op0->getType()->getScalarType()->isIntegerTy(1)) | |||
4375 | if (Instruction *Res = canonicalizeICmpBool(I, *Builder)) | |||
4376 | return Res; | |||
4377 | ||||
4378 | if (ICmpInst *NewICmp = canonicalizeCmpWithConstant(I)) | |||
4379 | return NewICmp; | |||
4380 | ||||
4381 | if (Instruction *Res = foldICmpWithConstant(I)) | |||
4382 | return Res; | |||
4383 | ||||
4384 | if (Instruction *Res = foldICmpUsingKnownBits(I)) | |||
4385 | return Res; | |||
4386 | ||||
4387 | // Test if the ICmpInst instruction is used exclusively by a select as | |||
4388 | // part of a minimum or maximum operation. If so, refrain from doing | |||
4389 | // any other folding. This helps out other analyses which understand | |||
4390 | // non-obfuscated minimum and maximum idioms, such as ScalarEvolution | |||
4391 | // and CodeGen. And in this case, at least one of the comparison | |||
4392 | // operands has at least one user besides the compare (the select), | |||
4393 | // which would often largely negate the benefit of folding anyway. | |||
4394 | if (I.hasOneUse()) | |||
4395 | if (SelectInst *SI = dyn_cast<SelectInst>(*I.user_begin())) | |||
4396 | if ((SI->getOperand(1) == Op0 && SI->getOperand(2) == Op1) || | |||
4397 | (SI->getOperand(2) == Op0 && SI->getOperand(1) == Op1)) | |||
4398 | return nullptr; | |||
4399 | ||||
4400 | // FIXME: We only do this after checking for min/max to prevent infinite | |||
4401 | // looping caused by a reverse canonicalization of these patterns for min/max. | |||
4402 | // FIXME: The organization of folds is a mess. These would naturally go into | |||
4403 | // canonicalizeCmpWithConstant(), but we can't move all of the above folds | |||
4404 | // down here after the min/max restriction. | |||
4405 | ICmpInst::Predicate Pred = I.getPredicate(); | |||
4406 | const APInt *C; | |||
4407 | if (match(Op1, m_APInt(C))) { | |||
4408 | // For i32: x >u 2147483647 -> x <s 0 -> true if sign bit set | |||
4409 | if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) { | |||
4410 | Constant *Zero = Constant::getNullValue(Op0->getType()); | |||
4411 | return new ICmpInst(ICmpInst::ICMP_SLT, Op0, Zero); | |||
4412 | } | |||
4413 | ||||
4414 | // For i32: x <u 2147483648 -> x >s -1 -> true if sign bit clear | |||
4415 | if (Pred == ICmpInst::ICMP_ULT && C->isMinSignedValue()) { | |||
4416 | Constant *AllOnes = Constant::getAllOnesValue(Op0->getType()); | |||
4417 | return new ICmpInst(ICmpInst::ICMP_SGT, Op0, AllOnes); | |||
4418 | } | |||
4419 | } | |||
4420 | ||||
4421 | if (Instruction *Res = foldICmpInstWithConstant(I)) | |||
4422 | return Res; | |||
4423 | ||||
4424 | if (Instruction *Res = foldICmpInstWithConstantNotInt(I)) | |||
4425 | return Res; | |||
4426 | ||||
4427 | // If we can optimize a 'icmp GEP, P' or 'icmp P, GEP', do so now. | |||
4428 | if (GEPOperator *GEP = dyn_cast<GEPOperator>(Op0)) | |||
4429 | if (Instruction *NI = foldGEPICmp(GEP, Op1, I.getPredicate(), I)) | |||
4430 | return NI; | |||
4431 | if (GEPOperator *GEP = dyn_cast<GEPOperator>(Op1)) | |||
4432 | if (Instruction *NI = foldGEPICmp(GEP, Op0, | |||
4433 | ICmpInst::getSwappedPredicate(I.getPredicate()), I)) | |||
4434 | return NI; | |||
4435 | ||||
4436 | // Try to optimize equality comparisons against alloca-based pointers. | |||
4437 | if (Op0->getType()->isPointerTy() && I.isEquality()) { | |||
4438 | assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?")((Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?" ) ? static_cast<void> (0) : __assert_fail ("Op1->getType()->isPointerTy() && \"Comparing pointer with non-pointer?\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4438, __PRETTY_FUNCTION__)); | |||
4439 | if (auto *Alloca = dyn_cast<AllocaInst>(GetUnderlyingObject(Op0, DL))) | |||
4440 | if (Instruction *New = foldAllocaCmp(I, Alloca, Op1)) | |||
4441 | return New; | |||
4442 | if (auto *Alloca = dyn_cast<AllocaInst>(GetUnderlyingObject(Op1, DL))) | |||
4443 | if (Instruction *New = foldAllocaCmp(I, Alloca, Op0)) | |||
4444 | return New; | |||
4445 | } | |||
4446 | ||||
4447 | // Test to see if the operands of the icmp are casted versions of other | |||
4448 | // values. If the ptr->ptr cast can be stripped off both arguments, we do so | |||
4449 | // now. | |||
4450 | if (BitCastInst *CI = dyn_cast<BitCastInst>(Op0)) { | |||
4451 | if (Op0->getType()->isPointerTy() && | |||
4452 | (isa<Constant>(Op1) || isa<BitCastInst>(Op1))) { | |||
4453 | // We keep moving the cast from the left operand over to the right | |||
4454 | // operand, where it can often be eliminated completely. | |||
4455 | Op0 = CI->getOperand(0); | |||
4456 | ||||
4457 | // If operand #1 is a bitcast instruction, it must also be a ptr->ptr cast | |||
4458 | // so eliminate it as well. | |||
4459 | if (BitCastInst *CI2 = dyn_cast<BitCastInst>(Op1)) | |||
4460 | Op1 = CI2->getOperand(0); | |||
4461 | ||||
4462 | // If Op1 is a constant, we can fold the cast into the constant. | |||
4463 | if (Op0->getType() != Op1->getType()) { | |||
4464 | if (Constant *Op1C = dyn_cast<Constant>(Op1)) { | |||
4465 | Op1 = ConstantExpr::getBitCast(Op1C, Op0->getType()); | |||
4466 | } else { | |||
4467 | // Otherwise, cast the RHS right before the icmp | |||
4468 | Op1 = Builder->CreateBitCast(Op1, Op0->getType()); | |||
4469 | } | |||
4470 | } | |||
4471 | return new ICmpInst(I.getPredicate(), Op0, Op1); | |||
4472 | } | |||
4473 | } | |||
4474 | ||||
4475 | if (isa<CastInst>(Op0)) { | |||
4476 | // Handle the special case of: icmp (cast bool to X), <cst> | |||
4477 | // This comes up when you have code like | |||
4478 | // int X = A < B; | |||
4479 | // if (X) ... | |||
4480 | // For generality, we handle any zero-extension of any operand comparison | |||
4481 | // with a constant or another cast from the same type. | |||
4482 | if (isa<Constant>(Op1) || isa<CastInst>(Op1)) | |||
4483 | if (Instruction *R = foldICmpWithCastAndCast(I)) | |||
4484 | return R; | |||
4485 | } | |||
4486 | ||||
4487 | if (Instruction *Res = foldICmpBinOp(I)) | |||
4488 | return Res; | |||
4489 | ||||
4490 | if (Instruction *Res = foldICmpWithMinMax(I)) | |||
4491 | return Res; | |||
4492 | ||||
4493 | { | |||
4494 | Value *A, *B; | |||
4495 | // Transform (A & ~B) == 0 --> (A & B) != 0 | |||
4496 | // and (A & ~B) != 0 --> (A & B) == 0 | |||
4497 | // if A is a power of 2. | |||
4498 | if (match(Op0, m_And(m_Value(A), m_Not(m_Value(B)))) && | |||
4499 | match(Op1, m_Zero()) && | |||
4500 | isKnownToBeAPowerOfTwo(A, DL, false, 0, &AC, &I, &DT) && I.isEquality()) | |||
4501 | return new ICmpInst(I.getInversePredicate(), | |||
4502 | Builder->CreateAnd(A, B), | |||
4503 | Op1); | |||
4504 | ||||
4505 | // ~x < ~y --> y < x | |||
4506 | // ~x < cst --> ~cst < x | |||
4507 | if (match(Op0, m_Not(m_Value(A)))) { | |||
4508 | if (match(Op1, m_Not(m_Value(B)))) | |||
4509 | return new ICmpInst(I.getPredicate(), B, A); | |||
4510 | if (ConstantInt *RHSC = dyn_cast<ConstantInt>(Op1)) | |||
4511 | return new ICmpInst(I.getPredicate(), ConstantExpr::getNot(RHSC), A); | |||
4512 | } | |||
4513 | ||||
4514 | Instruction *AddI = nullptr; | |||
4515 | if (match(&I, m_UAddWithOverflow(m_Value(A), m_Value(B), | |||
4516 | m_Instruction(AddI))) && | |||
4517 | isa<IntegerType>(A->getType())) { | |||
4518 | Value *Result; | |||
4519 | Constant *Overflow; | |||
4520 | if (OptimizeOverflowCheck(OCF_UNSIGNED_ADD, A, B, *AddI, Result, | |||
4521 | Overflow)) { | |||
4522 | replaceInstUsesWith(*AddI, Result); | |||
4523 | return replaceInstUsesWith(I, Overflow); | |||
4524 | } | |||
4525 | } | |||
4526 | ||||
4527 | // (zext a) * (zext b) --> llvm.umul.with.overflow. | |||
4528 | if (match(Op0, m_Mul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) { | |||
4529 | if (Instruction *R = processUMulZExtIdiom(I, Op0, Op1, *this)) | |||
4530 | return R; | |||
4531 | } | |||
4532 | if (match(Op1, m_Mul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) { | |||
4533 | if (Instruction *R = processUMulZExtIdiom(I, Op1, Op0, *this)) | |||
4534 | return R; | |||
4535 | } | |||
4536 | } | |||
4537 | ||||
4538 | if (Instruction *Res = foldICmpEquality(I)) | |||
4539 | return Res; | |||
4540 | ||||
4541 | // The 'cmpxchg' instruction returns an aggregate containing the old value and | |||
4542 | // an i1 which indicates whether or not we successfully did the swap. | |||
4543 | // | |||
4544 | // Replace comparisons between the old value and the expected value with the | |||
4545 | // indicator that 'cmpxchg' returns. | |||
4546 | // | |||
4547 | // N.B. This transform is only valid when the 'cmpxchg' is not permitted to | |||
4548 | // spuriously fail. In those cases, the old value may equal the expected | |||
4549 | // value but it is possible for the swap to not occur. | |||
4550 | if (I.getPredicate() == ICmpInst::ICMP_EQ) | |||
4551 | if (auto *EVI = dyn_cast<ExtractValueInst>(Op0)) | |||
4552 | if (auto *ACXI = dyn_cast<AtomicCmpXchgInst>(EVI->getAggregateOperand())) | |||
4553 | if (EVI->getIndices()[0] == 0 && ACXI->getCompareOperand() == Op1 && | |||
4554 | !ACXI->isWeak()) | |||
4555 | return ExtractValueInst::Create(ACXI, 1); | |||
4556 | ||||
4557 | { | |||
4558 | Value *X; ConstantInt *Cst; | |||
4559 | // icmp X+Cst, X | |||
4560 | if (match(Op0, m_Add(m_Value(X), m_ConstantInt(Cst))) && Op1 == X) | |||
4561 | return foldICmpAddOpConst(I, X, Cst, I.getPredicate()); | |||
4562 | ||||
4563 | // icmp X, X+Cst | |||
4564 | if (match(Op1, m_Add(m_Value(X), m_ConstantInt(Cst))) && Op0 == X) | |||
4565 | return foldICmpAddOpConst(I, X, Cst, I.getSwappedPredicate()); | |||
4566 | } | |||
4567 | return Changed ? &I : nullptr; | |||
4568 | } | |||
4569 | ||||
4570 | /// Fold fcmp ([us]itofp x, cst) if possible. | |||
4571 | Instruction *InstCombiner::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, | |||
4572 | Constant *RHSC) { | |||
4573 | if (!isa<ConstantFP>(RHSC)) return nullptr; | |||
4574 | const APFloat &RHS = cast<ConstantFP>(RHSC)->getValueAPF(); | |||
4575 | ||||
4576 | // Get the width of the mantissa. We don't want to hack on conversions that | |||
4577 | // might lose information from the integer, e.g. "i64 -> float" | |||
4578 | int MantissaWidth = LHSI->getType()->getFPMantissaWidth(); | |||
4579 | if (MantissaWidth == -1) return nullptr; // Unknown. | |||
4580 | ||||
4581 | IntegerType *IntTy = cast<IntegerType>(LHSI->getOperand(0)->getType()); | |||
4582 | ||||
4583 | bool LHSUnsigned = isa<UIToFPInst>(LHSI); | |||
4584 | ||||
4585 | if (I.isEquality()) { | |||
4586 | FCmpInst::Predicate P = I.getPredicate(); | |||
4587 | bool IsExact = false; | |||
4588 | APSInt RHSCvt(IntTy->getBitWidth(), LHSUnsigned); | |||
4589 | RHS.convertToInteger(RHSCvt, APFloat::rmNearestTiesToEven, &IsExact); | |||
4590 | ||||
4591 | // If the floating point constant isn't an integer value, we know if we will | |||
4592 | // ever compare equal / not equal to it. | |||
4593 | if (!IsExact) { | |||
4594 | // TODO: Can never be -0.0 and other non-representable values | |||
4595 | APFloat RHSRoundInt(RHS); | |||
4596 | RHSRoundInt.roundToIntegral(APFloat::rmNearestTiesToEven); | |||
4597 | if (RHS.compare(RHSRoundInt) != APFloat::cmpEqual) { | |||
4598 | if (P == FCmpInst::FCMP_OEQ || P == FCmpInst::FCMP_UEQ) | |||
4599 | return replaceInstUsesWith(I, Builder->getFalse()); | |||
4600 | ||||
4601 | assert(P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE)((P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE) ? static_cast <void> (0) : __assert_fail ("P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4601, __PRETTY_FUNCTION__)); | |||
4602 | return replaceInstUsesWith(I, Builder->getTrue()); | |||
4603 | } | |||
4604 | } | |||
4605 | ||||
4606 | // TODO: If the constant is exactly representable, is it always OK to do | |||
4607 | // equality compares as integer? | |||
4608 | } | |||
4609 | ||||
4610 | // Check to see that the input is converted from an integer type that is small | |||
4611 | // enough that preserves all bits. TODO: check here for "known" sign bits. | |||
4612 | // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e. | |||
4613 | unsigned InputSize = IntTy->getScalarSizeInBits(); | |||
4614 | ||||
4615 | // Following test does NOT adjust InputSize downwards for signed inputs, | |||
4616 | // because the most negative value still requires all the mantissa bits | |||
4617 | // to distinguish it from one less than that value. | |||
4618 | if ((int)InputSize > MantissaWidth) { | |||
4619 | // Conversion would lose accuracy. Check if loss can impact comparison. | |||
4620 | int Exp = ilogb(RHS); | |||
4621 | if (Exp == APFloat::IEK_Inf) { | |||
4622 | int MaxExponent = ilogb(APFloat::getLargest(RHS.getSemantics())); | |||
4623 | if (MaxExponent < (int)InputSize - !LHSUnsigned) | |||
4624 | // Conversion could create infinity. | |||
4625 | return nullptr; | |||
4626 | } else { | |||
4627 | // Note that if RHS is zero or NaN, then Exp is negative | |||
4628 | // and first condition is trivially false. | |||
4629 | if (MantissaWidth <= Exp && Exp <= (int)InputSize - !LHSUnsigned) | |||
4630 | // Conversion could affect comparison. | |||
4631 | return nullptr; | |||
4632 | } | |||
4633 | } | |||
4634 | ||||
4635 | // Otherwise, we can potentially simplify the comparison. We know that it | |||
4636 | // will always come through as an integer value and we know the constant is | |||
4637 | // not a NAN (it would have been previously simplified). | |||
4638 | assert(!RHS.isNaN() && "NaN comparison not already folded!")((!RHS.isNaN() && "NaN comparison not already folded!" ) ? static_cast<void> (0) : __assert_fail ("!RHS.isNaN() && \"NaN comparison not already folded!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4638, __PRETTY_FUNCTION__)); | |||
4639 | ||||
4640 | ICmpInst::Predicate Pred; | |||
4641 | switch (I.getPredicate()) { | |||
4642 | default: llvm_unreachable("Unexpected predicate!")::llvm::llvm_unreachable_internal("Unexpected predicate!", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4642); | |||
4643 | case FCmpInst::FCMP_UEQ: | |||
4644 | case FCmpInst::FCMP_OEQ: | |||
4645 | Pred = ICmpInst::ICMP_EQ; | |||
4646 | break; | |||
4647 | case FCmpInst::FCMP_UGT: | |||
4648 | case FCmpInst::FCMP_OGT: | |||
4649 | Pred = LHSUnsigned ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_SGT; | |||
4650 | break; | |||
4651 | case FCmpInst::FCMP_UGE: | |||
4652 | case FCmpInst::FCMP_OGE: | |||
4653 | Pred = LHSUnsigned ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_SGE; | |||
4654 | break; | |||
4655 | case FCmpInst::FCMP_ULT: | |||
4656 | case FCmpInst::FCMP_OLT: | |||
4657 | Pred = LHSUnsigned ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_SLT; | |||
4658 | break; | |||
4659 | case FCmpInst::FCMP_ULE: | |||
4660 | case FCmpInst::FCMP_OLE: | |||
4661 | Pred = LHSUnsigned ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_SLE; | |||
4662 | break; | |||
4663 | case FCmpInst::FCMP_UNE: | |||
4664 | case FCmpInst::FCMP_ONE: | |||
4665 | Pred = ICmpInst::ICMP_NE; | |||
4666 | break; | |||
4667 | case FCmpInst::FCMP_ORD: | |||
4668 | return replaceInstUsesWith(I, Builder->getTrue()); | |||
4669 | case FCmpInst::FCMP_UNO: | |||
4670 | return replaceInstUsesWith(I, Builder->getFalse()); | |||
4671 | } | |||
4672 | ||||
4673 | // Now we know that the APFloat is a normal number, zero or inf. | |||
4674 | ||||
4675 | // See if the FP constant is too large for the integer. For example, | |||
4676 | // comparing an i8 to 300.0. | |||
4677 | unsigned IntWidth = IntTy->getScalarSizeInBits(); | |||
4678 | ||||
4679 | if (!LHSUnsigned) { | |||
4680 | // If the RHS value is > SignedMax, fold the comparison. This handles +INF | |||
4681 | // and large values. | |||
4682 | APFloat SMax(RHS.getSemantics()); | |||
4683 | SMax.convertFromAPInt(APInt::getSignedMaxValue(IntWidth), true, | |||
4684 | APFloat::rmNearestTiesToEven); | |||
4685 | if (SMax.compare(RHS) == APFloat::cmpLessThan) { // smax < 13123.0 | |||
4686 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT || | |||
4687 | Pred == ICmpInst::ICMP_SLE) | |||
4688 | return replaceInstUsesWith(I, Builder->getTrue()); | |||
4689 | return replaceInstUsesWith(I, Builder->getFalse()); | |||
4690 | } | |||
4691 | } else { | |||
4692 | // If the RHS value is > UnsignedMax, fold the comparison. This handles | |||
4693 | // +INF and large values. | |||
4694 | APFloat UMax(RHS.getSemantics()); | |||
4695 | UMax.convertFromAPInt(APInt::getMaxValue(IntWidth), false, | |||
4696 | APFloat::rmNearestTiesToEven); | |||
4697 | if (UMax.compare(RHS) == APFloat::cmpLessThan) { // umax < 13123.0 | |||
4698 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT || | |||
4699 | Pred == ICmpInst::ICMP_ULE) | |||
4700 | return replaceInstUsesWith(I, Builder->getTrue()); | |||
4701 | return replaceInstUsesWith(I, Builder->getFalse()); | |||
4702 | } | |||
4703 | } | |||
4704 | ||||
4705 | if (!LHSUnsigned) { | |||
4706 | // See if the RHS value is < SignedMin. | |||
4707 | APFloat SMin(RHS.getSemantics()); | |||
4708 | SMin.convertFromAPInt(APInt::getSignedMinValue(IntWidth), true, | |||
4709 | APFloat::rmNearestTiesToEven); | |||
4710 | if (SMin.compare(RHS) == APFloat::cmpGreaterThan) { // smin > 12312.0 | |||
4711 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT || | |||
4712 | Pred == ICmpInst::ICMP_SGE) | |||
4713 | return replaceInstUsesWith(I, Builder->getTrue()); | |||
4714 | return replaceInstUsesWith(I, Builder->getFalse()); | |||
4715 | } | |||
4716 | } else { | |||
4717 | // See if the RHS value is < UnsignedMin. | |||
4718 | APFloat SMin(RHS.getSemantics()); | |||
4719 | SMin.convertFromAPInt(APInt::getMinValue(IntWidth), true, | |||
4720 | APFloat::rmNearestTiesToEven); | |||
4721 | if (SMin.compare(RHS) == APFloat::cmpGreaterThan) { // umin > 12312.0 | |||
4722 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT || | |||
4723 | Pred == ICmpInst::ICMP_UGE) | |||
4724 | return replaceInstUsesWith(I, Builder->getTrue()); | |||
4725 | return replaceInstUsesWith(I, Builder->getFalse()); | |||
4726 | } | |||
4727 | } | |||
4728 | ||||
4729 | // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or | |||
4730 | // [0, UMAX], but it may still be fractional. See if it is fractional by | |||
4731 | // casting the FP value to the integer value and back, checking for equality. | |||
4732 | // Don't do this for zero, because -0.0 is not fractional. | |||
4733 | Constant *RHSInt = LHSUnsigned | |||
4734 | ? ConstantExpr::getFPToUI(RHSC, IntTy) | |||
4735 | : ConstantExpr::getFPToSI(RHSC, IntTy); | |||
4736 | if (!RHS.isZero()) { | |||
4737 | bool Equal = LHSUnsigned | |||
4738 | ? ConstantExpr::getUIToFP(RHSInt, RHSC->getType()) == RHSC | |||
4739 | : ConstantExpr::getSIToFP(RHSInt, RHSC->getType()) == RHSC; | |||
4740 | if (!Equal) { | |||
4741 | // If we had a comparison against a fractional value, we have to adjust | |||
4742 | // the compare predicate and sometimes the value. RHSC is rounded towards | |||
4743 | // zero at this point. | |||
4744 | switch (Pred) { | |||
4745 | default: llvm_unreachable("Unexpected integer comparison!")::llvm::llvm_unreachable_internal("Unexpected integer comparison!" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4745); | |||
4746 | case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true | |||
4747 | return replaceInstUsesWith(I, Builder->getTrue()); | |||
4748 | case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false | |||
4749 | return replaceInstUsesWith(I, Builder->getFalse()); | |||
4750 | case ICmpInst::ICMP_ULE: | |||
4751 | // (float)int <= 4.4 --> int <= 4 | |||
4752 | // (float)int <= -4.4 --> false | |||
4753 | if (RHS.isNegative()) | |||
4754 | return replaceInstUsesWith(I, Builder->getFalse()); | |||
4755 | break; | |||
4756 | case ICmpInst::ICMP_SLE: | |||
4757 | // (float)int <= 4.4 --> int <= 4 | |||
4758 | // (float)int <= -4.4 --> int < -4 | |||
4759 | if (RHS.isNegative()) | |||
4760 | Pred = ICmpInst::ICMP_SLT; | |||
4761 | break; | |||
4762 | case ICmpInst::ICMP_ULT: | |||
4763 | // (float)int < -4.4 --> false | |||
4764 | // (float)int < 4.4 --> int <= 4 | |||
4765 | if (RHS.isNegative()) | |||
4766 | return replaceInstUsesWith(I, Builder->getFalse()); | |||
4767 | Pred = ICmpInst::ICMP_ULE; | |||
4768 | break; | |||
4769 | case ICmpInst::ICMP_SLT: | |||
4770 | // (float)int < -4.4 --> int < -4 | |||
4771 | // (float)int < 4.4 --> int <= 4 | |||
4772 | if (!RHS.isNegative()) | |||
4773 | Pred = ICmpInst::ICMP_SLE; | |||
4774 | break; | |||
4775 | case ICmpInst::ICMP_UGT: | |||
4776 | // (float)int > 4.4 --> int > 4 | |||
4777 | // (float)int > -4.4 --> true | |||
4778 | if (RHS.isNegative()) | |||
4779 | return replaceInstUsesWith(I, Builder->getTrue()); | |||
4780 | break; | |||
4781 | case ICmpInst::ICMP_SGT: | |||
4782 | // (float)int > 4.4 --> int > 4 | |||
4783 | // (float)int > -4.4 --> int >= -4 | |||
4784 | if (RHS.isNegative()) | |||
4785 | Pred = ICmpInst::ICMP_SGE; | |||
4786 | break; | |||
4787 | case ICmpInst::ICMP_UGE: | |||
4788 | // (float)int >= -4.4 --> true | |||
4789 | // (float)int >= 4.4 --> int > 4 | |||
4790 | if (RHS.isNegative()) | |||
4791 | return replaceInstUsesWith(I, Builder->getTrue()); | |||
4792 | Pred = ICmpInst::ICMP_UGT; | |||
4793 | break; | |||
4794 | case ICmpInst::ICMP_SGE: | |||
4795 | // (float)int >= -4.4 --> int >= -4 | |||
4796 | // (float)int >= 4.4 --> int > 4 | |||
4797 | if (!RHS.isNegative()) | |||
4798 | Pred = ICmpInst::ICMP_SGT; | |||
4799 | break; | |||
4800 | } | |||
4801 | } | |||
4802 | } | |||
4803 | ||||
4804 | // Lower this FP comparison into an appropriate integer version of the | |||
4805 | // comparison. | |||
4806 | return new ICmpInst(Pred, LHSI->getOperand(0), RHSInt); | |||
4807 | } | |||
4808 | ||||
4809 | Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) { | |||
4810 | bool Changed = false; | |||
4811 | ||||
4812 | /// Orders the operands of the compare so that they are listed from most | |||
4813 | /// complex to least complex. This puts constants before unary operators, | |||
4814 | /// before binary operators. | |||
4815 | if (getComplexity(I.getOperand(0)) < getComplexity(I.getOperand(1))) { | |||
4816 | I.swapOperands(); | |||
4817 | Changed = true; | |||
4818 | } | |||
4819 | ||||
4820 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
4821 | ||||
4822 | if (Value *V = | |||
4823 | SimplifyFCmpInst(I.getPredicate(), Op0, Op1, I.getFastMathFlags(), | |||
4824 | SQ.getWithInstruction(&I))) | |||
4825 | return replaceInstUsesWith(I, V); | |||
4826 | ||||
4827 | // Simplify 'fcmp pred X, X' | |||
4828 | if (Op0 == Op1) { | |||
4829 | switch (I.getPredicate()) { | |||
4830 | default: llvm_unreachable("Unknown predicate!")::llvm::llvm_unreachable_internal("Unknown predicate!", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4830); | |||
4831 | case FCmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y) | |||
4832 | case FCmpInst::FCMP_ULT: // True if unordered or less than | |||
4833 | case FCmpInst::FCMP_UGT: // True if unordered or greater than | |||
4834 | case FCmpInst::FCMP_UNE: // True if unordered or not equal | |||
4835 | // Canonicalize these to be 'fcmp uno %X, 0.0'. | |||
4836 | I.setPredicate(FCmpInst::FCMP_UNO); | |||
4837 | I.setOperand(1, Constant::getNullValue(Op0->getType())); | |||
4838 | return &I; | |||
4839 | ||||
4840 | case FCmpInst::FCMP_ORD: // True if ordered (no nans) | |||
4841 | case FCmpInst::FCMP_OEQ: // True if ordered and equal | |||
4842 | case FCmpInst::FCMP_OGE: // True if ordered and greater than or equal | |||
4843 | case FCmpInst::FCMP_OLE: // True if ordered and less than or equal | |||
4844 | // Canonicalize these to be 'fcmp ord %X, 0.0'. | |||
4845 | I.setPredicate(FCmpInst::FCMP_ORD); | |||
4846 | I.setOperand(1, Constant::getNullValue(Op0->getType())); | |||
4847 | return &I; | |||
4848 | } | |||
4849 | } | |||
4850 | ||||
4851 | // Test if the FCmpInst instruction is used exclusively by a select as | |||
4852 | // part of a minimum or maximum operation. If so, refrain from doing | |||
4853 | // any other folding. This helps out other analyses which understand | |||
4854 | // non-obfuscated minimum and maximum idioms, such as ScalarEvolution | |||
4855 | // and CodeGen. And in this case, at least one of the comparison | |||
4856 | // operands has at least one user besides the compare (the select), | |||
4857 | // which would often largely negate the benefit of folding anyway. | |||
4858 | if (I.hasOneUse()) | |||
4859 | if (SelectInst *SI = dyn_cast<SelectInst>(*I.user_begin())) | |||
4860 | if ((SI->getOperand(1) == Op0 && SI->getOperand(2) == Op1) || | |||
4861 | (SI->getOperand(2) == Op0 && SI->getOperand(1) == Op1)) | |||
4862 | return nullptr; | |||
4863 | ||||
4864 | // Handle fcmp with constant RHS | |||
4865 | if (Constant *RHSC = dyn_cast<Constant>(Op1)) { | |||
4866 | if (Instruction *LHSI = dyn_cast<Instruction>(Op0)) | |||
4867 | switch (LHSI->getOpcode()) { | |||
4868 | case Instruction::FPExt: { | |||
4869 | // fcmp (fpext x), C -> fcmp x, (fptrunc C) if fptrunc is lossless | |||
4870 | FPExtInst *LHSExt = cast<FPExtInst>(LHSI); | |||
4871 | ConstantFP *RHSF = dyn_cast<ConstantFP>(RHSC); | |||
4872 | if (!RHSF) | |||
4873 | break; | |||
4874 | ||||
4875 | const fltSemantics *Sem; | |||
4876 | // FIXME: This shouldn't be here. | |||
4877 | if (LHSExt->getSrcTy()->isHalfTy()) | |||
4878 | Sem = &APFloat::IEEEhalf(); | |||
4879 | else if (LHSExt->getSrcTy()->isFloatTy()) | |||
4880 | Sem = &APFloat::IEEEsingle(); | |||
4881 | else if (LHSExt->getSrcTy()->isDoubleTy()) | |||
4882 | Sem = &APFloat::IEEEdouble(); | |||
4883 | else if (LHSExt->getSrcTy()->isFP128Ty()) | |||
4884 | Sem = &APFloat::IEEEquad(); | |||
4885 | else if (LHSExt->getSrcTy()->isX86_FP80Ty()) | |||
4886 | Sem = &APFloat::x87DoubleExtended(); | |||
4887 | else if (LHSExt->getSrcTy()->isPPC_FP128Ty()) | |||
4888 | Sem = &APFloat::PPCDoubleDouble(); | |||
4889 | else | |||
4890 | break; | |||
4891 | ||||
4892 | bool Lossy; | |||
4893 | APFloat F = RHSF->getValueAPF(); | |||
4894 | F.convert(*Sem, APFloat::rmNearestTiesToEven, &Lossy); | |||
4895 | ||||
4896 | // Avoid lossy conversions and denormals. Zero is a special case | |||
4897 | // that's OK to convert. | |||
4898 | APFloat Fabs = F; | |||
4899 | Fabs.clearSign(); | |||
4900 | if (!Lossy && | |||
4901 | ((Fabs.compare(APFloat::getSmallestNormalized(*Sem)) != | |||
4902 | APFloat::cmpLessThan) || Fabs.isZero())) | |||
4903 | ||||
4904 | return new FCmpInst(I.getPredicate(), LHSExt->getOperand(0), | |||
4905 | ConstantFP::get(RHSC->getContext(), F)); | |||
4906 | break; | |||
4907 | } | |||
4908 | case Instruction::PHI: | |||
4909 | // Only fold fcmp into the PHI if the phi and fcmp are in the same | |||
4910 | // block. If in the same block, we're encouraging jump threading. If | |||
4911 | // not, we are just pessimizing the code by making an i1 phi. | |||
4912 | if (LHSI->getParent() == I.getParent()) | |||
4913 | if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI))) | |||
4914 | return NV; | |||
4915 | break; | |||
4916 | case Instruction::SIToFP: | |||
4917 | case Instruction::UIToFP: | |||
4918 | if (Instruction *NV = foldFCmpIntToFPConst(I, LHSI, RHSC)) | |||
4919 | return NV; | |||
4920 | break; | |||
4921 | case Instruction::FSub: { | |||
4922 | // fcmp pred (fneg x), C -> fcmp swap(pred) x, -C | |||
4923 | Value *Op; | |||
4924 | if (match(LHSI, m_FNeg(m_Value(Op)))) | |||
4925 | return new FCmpInst(I.getSwappedPredicate(), Op, | |||
4926 | ConstantExpr::getFNeg(RHSC)); | |||
4927 | break; | |||
4928 | } | |||
4929 | case Instruction::Load: | |||
4930 | if (GetElementPtrInst *GEP = | |||
4931 | dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) { | |||
4932 | if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) | |||
4933 | if (GV->isConstant() && GV->hasDefinitiveInitializer() && | |||
4934 | !cast<LoadInst>(LHSI)->isVolatile()) | |||
4935 | if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, I)) | |||
4936 | return Res; | |||
4937 | } | |||
4938 | break; | |||
4939 | case Instruction::Call: { | |||
4940 | if (!RHSC->isNullValue()) | |||
4941 | break; | |||
4942 | ||||
4943 | CallInst *CI = cast<CallInst>(LHSI); | |||
4944 | Intrinsic::ID IID = getIntrinsicForCallSite(CI, &TLI); | |||
4945 | if (IID != Intrinsic::fabs) | |||
4946 | break; | |||
4947 | ||||
4948 | // Various optimization for fabs compared with zero. | |||
4949 | switch (I.getPredicate()) { | |||
4950 | default: | |||
4951 | break; | |||
4952 | // fabs(x) < 0 --> false | |||
4953 | case FCmpInst::FCMP_OLT: | |||
4954 | llvm_unreachable("handled by SimplifyFCmpInst")::llvm::llvm_unreachable_internal("handled by SimplifyFCmpInst" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4954); | |||
4955 | // fabs(x) > 0 --> x != 0 | |||
4956 | case FCmpInst::FCMP_OGT: | |||
4957 | return new FCmpInst(FCmpInst::FCMP_ONE, CI->getArgOperand(0), RHSC); | |||
4958 | // fabs(x) <= 0 --> x == 0 | |||
4959 | case FCmpInst::FCMP_OLE: | |||
4960 | return new FCmpInst(FCmpInst::FCMP_OEQ, CI->getArgOperand(0), RHSC); | |||
4961 | // fabs(x) >= 0 --> !isnan(x) | |||
4962 | case FCmpInst::FCMP_OGE: | |||
4963 | return new FCmpInst(FCmpInst::FCMP_ORD, CI->getArgOperand(0), RHSC); | |||
4964 | // fabs(x) == 0 --> x == 0 | |||
4965 | // fabs(x) != 0 --> x != 0 | |||
4966 | case FCmpInst::FCMP_OEQ: | |||
4967 | case FCmpInst::FCMP_UEQ: | |||
4968 | case FCmpInst::FCMP_ONE: | |||
4969 | case FCmpInst::FCMP_UNE: | |||
4970 | return new FCmpInst(I.getPredicate(), CI->getArgOperand(0), RHSC); | |||
4971 | } | |||
4972 | } | |||
4973 | } | |||
4974 | } | |||
4975 | ||||
4976 | // fcmp pred (fneg x), (fneg y) -> fcmp swap(pred) x, y | |||
4977 | Value *X, *Y; | |||
4978 | if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_FNeg(m_Value(Y)))) | |||
4979 | return new FCmpInst(I.getSwappedPredicate(), X, Y); | |||
4980 | ||||
4981 | // fcmp (fpext x), (fpext y) -> fcmp x, y | |||
4982 | if (FPExtInst *LHSExt = dyn_cast<FPExtInst>(Op0)) | |||
4983 | if (FPExtInst *RHSExt = dyn_cast<FPExtInst>(Op1)) | |||
4984 | if (LHSExt->getSrcTy() == RHSExt->getSrcTy()) | |||
4985 | return new FCmpInst(I.getPredicate(), LHSExt->getOperand(0), | |||
4986 | RHSExt->getOperand(0)); | |||
4987 | ||||
4988 | return Changed ? &I : nullptr; | |||
4989 | } |