File: | build/source/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp |
Warning: | line 865, column 24 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- InstCombineMulDivRem.cpp -------------------------------------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This file implements the visit functions for mul, fmul, sdiv, udiv, fdiv, | |||
10 | // srem, urem, frem. | |||
11 | // | |||
12 | //===----------------------------------------------------------------------===// | |||
13 | ||||
14 | #include "InstCombineInternal.h" | |||
15 | #include "llvm/ADT/APInt.h" | |||
16 | #include "llvm/ADT/SmallVector.h" | |||
17 | #include "llvm/Analysis/InstructionSimplify.h" | |||
18 | #include "llvm/Analysis/ValueTracking.h" | |||
19 | #include "llvm/IR/BasicBlock.h" | |||
20 | #include "llvm/IR/Constant.h" | |||
21 | #include "llvm/IR/Constants.h" | |||
22 | #include "llvm/IR/InstrTypes.h" | |||
23 | #include "llvm/IR/Instruction.h" | |||
24 | #include "llvm/IR/Instructions.h" | |||
25 | #include "llvm/IR/IntrinsicInst.h" | |||
26 | #include "llvm/IR/Intrinsics.h" | |||
27 | #include "llvm/IR/Operator.h" | |||
28 | #include "llvm/IR/PatternMatch.h" | |||
29 | #include "llvm/IR/Type.h" | |||
30 | #include "llvm/IR/Value.h" | |||
31 | #include "llvm/Support/Casting.h" | |||
32 | #include "llvm/Support/ErrorHandling.h" | |||
33 | #include "llvm/Transforms/InstCombine/InstCombiner.h" | |||
34 | #include "llvm/Transforms/Utils/BuildLibCalls.h" | |||
35 | #include <cassert> | |||
36 | ||||
37 | #define DEBUG_TYPE"instcombine" "instcombine" | |||
38 | #include "llvm/Transforms/Utils/InstructionWorklist.h" | |||
39 | ||||
40 | using namespace llvm; | |||
41 | using namespace PatternMatch; | |||
42 | ||||
43 | /// The specific integer value is used in a context where it is known to be | |||
44 | /// non-zero. If this allows us to simplify the computation, do so and return | |||
45 | /// the new operand, otherwise return null. | |||
46 | static Value *simplifyValueKnownNonZero(Value *V, InstCombinerImpl &IC, | |||
47 | Instruction &CxtI) { | |||
48 | // If V has multiple uses, then we would have to do more analysis to determine | |||
49 | // if this is safe. For example, the use could be in dynamically unreached | |||
50 | // code. | |||
51 | if (!V->hasOneUse()) return nullptr; | |||
52 | ||||
53 | bool MadeChange = false; | |||
54 | ||||
55 | // ((1 << A) >>u B) --> (1 << (A-B)) | |||
56 | // Because V cannot be zero, we know that B is less than A. | |||
57 | Value *A = nullptr, *B = nullptr, *One = nullptr; | |||
58 | if (match(V, m_LShr(m_OneUse(m_Shl(m_Value(One), m_Value(A))), m_Value(B))) && | |||
59 | match(One, m_One())) { | |||
60 | A = IC.Builder.CreateSub(A, B); | |||
61 | return IC.Builder.CreateShl(One, A); | |||
62 | } | |||
63 | ||||
64 | // (PowerOfTwo >>u B) --> isExact since shifting out the result would make it | |||
65 | // inexact. Similarly for <<. | |||
66 | BinaryOperator *I = dyn_cast<BinaryOperator>(V); | |||
67 | if (I && I->isLogicalShift() && | |||
68 | IC.isKnownToBeAPowerOfTwo(I->getOperand(0), false, 0, &CxtI)) { | |||
69 | // We know that this is an exact/nuw shift and that the input is a | |||
70 | // non-zero context as well. | |||
71 | if (Value *V2 = simplifyValueKnownNonZero(I->getOperand(0), IC, CxtI)) { | |||
72 | IC.replaceOperand(*I, 0, V2); | |||
73 | MadeChange = true; | |||
74 | } | |||
75 | ||||
76 | if (I->getOpcode() == Instruction::LShr && !I->isExact()) { | |||
77 | I->setIsExact(); | |||
78 | MadeChange = true; | |||
79 | } | |||
80 | ||||
81 | if (I->getOpcode() == Instruction::Shl && !I->hasNoUnsignedWrap()) { | |||
82 | I->setHasNoUnsignedWrap(); | |||
83 | MadeChange = true; | |||
84 | } | |||
85 | } | |||
86 | ||||
87 | // TODO: Lots more we could do here: | |||
88 | // If V is a phi node, we can call this on each of its operands. | |||
89 | // "select cond, X, 0" can simplify to "X". | |||
90 | ||||
91 | return MadeChange ? V : nullptr; | |||
92 | } | |||
93 | ||||
94 | // TODO: This is a specific form of a much more general pattern. | |||
95 | // We could detect a select with any binop identity constant, or we | |||
96 | // could use SimplifyBinOp to see if either arm of the select reduces. | |||
97 | // But that needs to be done carefully and/or while removing potential | |||
98 | // reverse canonicalizations as in InstCombiner::foldSelectIntoOp(). | |||
99 | static Value *foldMulSelectToNegate(BinaryOperator &I, | |||
100 | InstCombiner::BuilderTy &Builder) { | |||
101 | Value *Cond, *OtherOp; | |||
102 | ||||
103 | // mul (select Cond, 1, -1), OtherOp --> select Cond, OtherOp, -OtherOp | |||
104 | // mul OtherOp, (select Cond, 1, -1) --> select Cond, OtherOp, -OtherOp | |||
105 | if (match(&I, m_c_Mul(m_OneUse(m_Select(m_Value(Cond), m_One(), m_AllOnes())), | |||
106 | m_Value(OtherOp)))) { | |||
107 | bool HasAnyNoWrap = I.hasNoSignedWrap() || I.hasNoUnsignedWrap(); | |||
108 | Value *Neg = Builder.CreateNeg(OtherOp, "", false, HasAnyNoWrap); | |||
109 | return Builder.CreateSelect(Cond, OtherOp, Neg); | |||
110 | } | |||
111 | // mul (select Cond, -1, 1), OtherOp --> select Cond, -OtherOp, OtherOp | |||
112 | // mul OtherOp, (select Cond, -1, 1) --> select Cond, -OtherOp, OtherOp | |||
113 | if (match(&I, m_c_Mul(m_OneUse(m_Select(m_Value(Cond), m_AllOnes(), m_One())), | |||
114 | m_Value(OtherOp)))) { | |||
115 | bool HasAnyNoWrap = I.hasNoSignedWrap() || I.hasNoUnsignedWrap(); | |||
116 | Value *Neg = Builder.CreateNeg(OtherOp, "", false, HasAnyNoWrap); | |||
117 | return Builder.CreateSelect(Cond, Neg, OtherOp); | |||
118 | } | |||
119 | ||||
120 | // fmul (select Cond, 1.0, -1.0), OtherOp --> select Cond, OtherOp, -OtherOp | |||
121 | // fmul OtherOp, (select Cond, 1.0, -1.0) --> select Cond, OtherOp, -OtherOp | |||
122 | if (match(&I, m_c_FMul(m_OneUse(m_Select(m_Value(Cond), m_SpecificFP(1.0), | |||
123 | m_SpecificFP(-1.0))), | |||
124 | m_Value(OtherOp)))) { | |||
125 | IRBuilder<>::FastMathFlagGuard FMFGuard(Builder); | |||
126 | Builder.setFastMathFlags(I.getFastMathFlags()); | |||
127 | return Builder.CreateSelect(Cond, OtherOp, Builder.CreateFNeg(OtherOp)); | |||
128 | } | |||
129 | ||||
130 | // fmul (select Cond, -1.0, 1.0), OtherOp --> select Cond, -OtherOp, OtherOp | |||
131 | // fmul OtherOp, (select Cond, -1.0, 1.0) --> select Cond, -OtherOp, OtherOp | |||
132 | if (match(&I, m_c_FMul(m_OneUse(m_Select(m_Value(Cond), m_SpecificFP(-1.0), | |||
133 | m_SpecificFP(1.0))), | |||
134 | m_Value(OtherOp)))) { | |||
135 | IRBuilder<>::FastMathFlagGuard FMFGuard(Builder); | |||
136 | Builder.setFastMathFlags(I.getFastMathFlags()); | |||
137 | return Builder.CreateSelect(Cond, Builder.CreateFNeg(OtherOp), OtherOp); | |||
138 | } | |||
139 | ||||
140 | return nullptr; | |||
141 | } | |||
142 | ||||
143 | /// Reduce integer multiplication patterns that contain a (+/-1 << Z) factor. | |||
144 | /// Callers are expected to call this twice to handle commuted patterns. | |||
145 | static Value *foldMulShl1(BinaryOperator &Mul, bool CommuteOperands, | |||
146 | InstCombiner::BuilderTy &Builder) { | |||
147 | Value *X = Mul.getOperand(0), *Y = Mul.getOperand(1); | |||
148 | if (CommuteOperands) | |||
149 | std::swap(X, Y); | |||
150 | ||||
151 | const bool HasNSW = Mul.hasNoSignedWrap(); | |||
152 | const bool HasNUW = Mul.hasNoUnsignedWrap(); | |||
153 | ||||
154 | // X * (1 << Z) --> X << Z | |||
155 | Value *Z; | |||
156 | if (match(Y, m_Shl(m_One(), m_Value(Z)))) { | |||
157 | bool PropagateNSW = HasNSW && cast<ShlOperator>(Y)->hasNoSignedWrap(); | |||
158 | return Builder.CreateShl(X, Z, Mul.getName(), HasNUW, PropagateNSW); | |||
159 | } | |||
160 | ||||
161 | // Similar to above, but an increment of the shifted value becomes an add: | |||
162 | // X * ((1 << Z) + 1) --> (X * (1 << Z)) + X --> (X << Z) + X | |||
163 | // This increases uses of X, so it may require a freeze, but that is still | |||
164 | // expected to be an improvement because it removes the multiply. | |||
165 | BinaryOperator *Shift; | |||
166 | if (match(Y, m_OneUse(m_Add(m_BinOp(Shift), m_One()))) && | |||
167 | match(Shift, m_OneUse(m_Shl(m_One(), m_Value(Z))))) { | |||
168 | bool PropagateNSW = HasNSW && Shift->hasNoSignedWrap(); | |||
169 | Value *FrX = Builder.CreateFreeze(X, X->getName() + ".fr"); | |||
170 | Value *Shl = Builder.CreateShl(FrX, Z, "mulshl", HasNUW, PropagateNSW); | |||
171 | return Builder.CreateAdd(Shl, FrX, Mul.getName(), HasNUW, PropagateNSW); | |||
172 | } | |||
173 | ||||
174 | // Similar to above, but a decrement of the shifted value is disguised as | |||
175 | // 'not' and becomes a sub: | |||
176 | // X * (~(-1 << Z)) --> X * ((1 << Z) - 1) --> (X << Z) - X | |||
177 | // This increases uses of X, so it may require a freeze, but that is still | |||
178 | // expected to be an improvement because it removes the multiply. | |||
179 | if (match(Y, m_OneUse(m_Not(m_OneUse(m_Shl(m_AllOnes(), m_Value(Z))))))) { | |||
180 | Value *FrX = Builder.CreateFreeze(X, X->getName() + ".fr"); | |||
181 | Value *Shl = Builder.CreateShl(FrX, Z, "mulshl"); | |||
182 | return Builder.CreateSub(Shl, FrX, Mul.getName()); | |||
183 | } | |||
184 | ||||
185 | return nullptr; | |||
186 | } | |||
187 | ||||
188 | static Value *takeLog2(IRBuilderBase &Builder, Value *Op, unsigned Depth, | |||
189 | bool AssumeNonZero, bool DoFold); | |||
190 | ||||
191 | Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) { | |||
192 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
193 | if (Value *V = | |||
194 | simplifyMulInst(Op0, Op1, I.hasNoSignedWrap(), I.hasNoUnsignedWrap(), | |||
195 | SQ.getWithInstruction(&I))) | |||
196 | return replaceInstUsesWith(I, V); | |||
197 | ||||
198 | if (SimplifyAssociativeOrCommutative(I)) | |||
199 | return &I; | |||
200 | ||||
201 | if (Instruction *X = foldVectorBinop(I)) | |||
202 | return X; | |||
203 | ||||
204 | if (Instruction *Phi = foldBinopWithPhiOperands(I)) | |||
205 | return Phi; | |||
206 | ||||
207 | if (Value *V = foldUsingDistributiveLaws(I)) | |||
208 | return replaceInstUsesWith(I, V); | |||
209 | ||||
210 | Type *Ty = I.getType(); | |||
211 | const unsigned BitWidth = Ty->getScalarSizeInBits(); | |||
212 | const bool HasNSW = I.hasNoSignedWrap(); | |||
213 | const bool HasNUW = I.hasNoUnsignedWrap(); | |||
214 | ||||
215 | // X * -1 --> 0 - X | |||
216 | if (match(Op1, m_AllOnes())) { | |||
217 | return HasNSW ? BinaryOperator::CreateNSWNeg(Op0) | |||
218 | : BinaryOperator::CreateNeg(Op0); | |||
219 | } | |||
220 | ||||
221 | // Also allow combining multiply instructions on vectors. | |||
222 | { | |||
223 | Value *NewOp; | |||
224 | Constant *C1, *C2; | |||
225 | const APInt *IVal; | |||
226 | if (match(&I, m_Mul(m_Shl(m_Value(NewOp), m_Constant(C2)), | |||
227 | m_Constant(C1))) && | |||
228 | match(C1, m_APInt(IVal))) { | |||
229 | // ((X << C2)*C1) == (X * (C1 << C2)) | |||
230 | Constant *Shl = ConstantExpr::getShl(C1, C2); | |||
231 | BinaryOperator *Mul = cast<BinaryOperator>(I.getOperand(0)); | |||
232 | BinaryOperator *BO = BinaryOperator::CreateMul(NewOp, Shl); | |||
233 | if (HasNUW && Mul->hasNoUnsignedWrap()) | |||
234 | BO->setHasNoUnsignedWrap(); | |||
235 | if (HasNSW && Mul->hasNoSignedWrap() && Shl->isNotMinSignedValue()) | |||
236 | BO->setHasNoSignedWrap(); | |||
237 | return BO; | |||
238 | } | |||
239 | ||||
240 | if (match(&I, m_Mul(m_Value(NewOp), m_Constant(C1)))) { | |||
241 | // Replace X*(2^C) with X << C, where C is either a scalar or a vector. | |||
242 | if (Constant *NewCst = ConstantExpr::getExactLogBase2(C1)) { | |||
243 | BinaryOperator *Shl = BinaryOperator::CreateShl(NewOp, NewCst); | |||
244 | ||||
245 | if (HasNUW) | |||
246 | Shl->setHasNoUnsignedWrap(); | |||
247 | if (HasNSW) { | |||
248 | const APInt *V; | |||
249 | if (match(NewCst, m_APInt(V)) && *V != V->getBitWidth() - 1) | |||
250 | Shl->setHasNoSignedWrap(); | |||
251 | } | |||
252 | ||||
253 | return Shl; | |||
254 | } | |||
255 | } | |||
256 | } | |||
257 | ||||
258 | if (Op0->hasOneUse() && match(Op1, m_NegatedPower2())) { | |||
259 | // Interpret X * (-1<<C) as (-X) * (1<<C) and try to sink the negation. | |||
260 | // The "* (1<<C)" thus becomes a potential shifting opportunity. | |||
261 | if (Value *NegOp0 = Negator::Negate(/*IsNegation*/ true, Op0, *this)) | |||
262 | return BinaryOperator::CreateMul( | |||
263 | NegOp0, ConstantExpr::getNeg(cast<Constant>(Op1)), I.getName()); | |||
264 | ||||
265 | // Try to convert multiply of extended operand to narrow negate and shift | |||
266 | // for better analysis. | |||
267 | // This is valid if the shift amount (trailing zeros in the multiplier | |||
268 | // constant) clears more high bits than the bitwidth difference between | |||
269 | // source and destination types: | |||
270 | // ({z/s}ext X) * (-1<<C) --> (zext (-X)) << C | |||
271 | const APInt *NegPow2C; | |||
272 | Value *X; | |||
273 | if (match(Op0, m_ZExtOrSExt(m_Value(X))) && | |||
274 | match(Op1, m_APIntAllowUndef(NegPow2C))) { | |||
275 | unsigned SrcWidth = X->getType()->getScalarSizeInBits(); | |||
276 | unsigned ShiftAmt = NegPow2C->countr_zero(); | |||
277 | if (ShiftAmt >= BitWidth - SrcWidth) { | |||
278 | Value *N = Builder.CreateNeg(X, X->getName() + ".neg"); | |||
279 | Value *Z = Builder.CreateZExt(N, Ty, N->getName() + ".z"); | |||
280 | return BinaryOperator::CreateShl(Z, ConstantInt::get(Ty, ShiftAmt)); | |||
281 | } | |||
282 | } | |||
283 | } | |||
284 | ||||
285 | if (Instruction *FoldedMul = foldBinOpIntoSelectOrPhi(I)) | |||
286 | return FoldedMul; | |||
287 | ||||
288 | if (Value *FoldedMul = foldMulSelectToNegate(I, Builder)) | |||
289 | return replaceInstUsesWith(I, FoldedMul); | |||
290 | ||||
291 | // Simplify mul instructions with a constant RHS. | |||
292 | Constant *MulC; | |||
293 | if (match(Op1, m_ImmConstant(MulC))) { | |||
294 | // Canonicalize (X+C1)*MulC -> X*MulC+C1*MulC. | |||
295 | // Canonicalize (X|C1)*MulC -> X*MulC+C1*MulC. | |||
296 | Value *X; | |||
297 | Constant *C1; | |||
298 | if ((match(Op0, m_OneUse(m_Add(m_Value(X), m_ImmConstant(C1))))) || | |||
299 | (match(Op0, m_OneUse(m_Or(m_Value(X), m_ImmConstant(C1)))) && | |||
300 | haveNoCommonBitsSet(X, C1, DL, &AC, &I, &DT))) { | |||
301 | // C1*MulC simplifies to a tidier constant. | |||
302 | Value *NewC = Builder.CreateMul(C1, MulC); | |||
303 | auto *BOp0 = cast<BinaryOperator>(Op0); | |||
304 | bool Op0NUW = | |||
305 | (BOp0->getOpcode() == Instruction::Or || BOp0->hasNoUnsignedWrap()); | |||
306 | Value *NewMul = Builder.CreateMul(X, MulC); | |||
307 | auto *BO = BinaryOperator::CreateAdd(NewMul, NewC); | |||
308 | if (HasNUW && Op0NUW) { | |||
309 | // If NewMulBO is constant we also can set BO to nuw. | |||
310 | if (auto *NewMulBO = dyn_cast<BinaryOperator>(NewMul)) | |||
311 | NewMulBO->setHasNoUnsignedWrap(); | |||
312 | BO->setHasNoUnsignedWrap(); | |||
313 | } | |||
314 | return BO; | |||
315 | } | |||
316 | } | |||
317 | ||||
318 | // abs(X) * abs(X) -> X * X | |||
319 | // nabs(X) * nabs(X) -> X * X | |||
320 | if (Op0 == Op1) { | |||
321 | Value *X, *Y; | |||
322 | SelectPatternFlavor SPF = matchSelectPattern(Op0, X, Y).Flavor; | |||
323 | if (SPF == SPF_ABS || SPF == SPF_NABS) | |||
324 | return BinaryOperator::CreateMul(X, X); | |||
325 | ||||
326 | if (match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(X)))) | |||
327 | return BinaryOperator::CreateMul(X, X); | |||
328 | } | |||
329 | ||||
330 | // -X * C --> X * -C | |||
331 | Value *X, *Y; | |||
332 | Constant *Op1C; | |||
333 | if (match(Op0, m_Neg(m_Value(X))) && match(Op1, m_Constant(Op1C))) | |||
334 | return BinaryOperator::CreateMul(X, ConstantExpr::getNeg(Op1C)); | |||
335 | ||||
336 | // -X * -Y --> X * Y | |||
337 | if (match(Op0, m_Neg(m_Value(X))) && match(Op1, m_Neg(m_Value(Y)))) { | |||
338 | auto *NewMul = BinaryOperator::CreateMul(X, Y); | |||
339 | if (HasNSW && cast<OverflowingBinaryOperator>(Op0)->hasNoSignedWrap() && | |||
340 | cast<OverflowingBinaryOperator>(Op1)->hasNoSignedWrap()) | |||
341 | NewMul->setHasNoSignedWrap(); | |||
342 | return NewMul; | |||
343 | } | |||
344 | ||||
345 | // -X * Y --> -(X * Y) | |||
346 | // X * -Y --> -(X * Y) | |||
347 | if (match(&I, m_c_Mul(m_OneUse(m_Neg(m_Value(X))), m_Value(Y)))) | |||
348 | return BinaryOperator::CreateNeg(Builder.CreateMul(X, Y)); | |||
349 | ||||
350 | // (X / Y) * Y = X - (X % Y) | |||
351 | // (X / Y) * -Y = (X % Y) - X | |||
352 | { | |||
353 | Value *Y = Op1; | |||
354 | BinaryOperator *Div = dyn_cast<BinaryOperator>(Op0); | |||
355 | if (!Div || (Div->getOpcode() != Instruction::UDiv && | |||
356 | Div->getOpcode() != Instruction::SDiv)) { | |||
357 | Y = Op0; | |||
358 | Div = dyn_cast<BinaryOperator>(Op1); | |||
359 | } | |||
360 | Value *Neg = dyn_castNegVal(Y); | |||
361 | if (Div && Div->hasOneUse() && | |||
362 | (Div->getOperand(1) == Y || Div->getOperand(1) == Neg) && | |||
363 | (Div->getOpcode() == Instruction::UDiv || | |||
364 | Div->getOpcode() == Instruction::SDiv)) { | |||
365 | Value *X = Div->getOperand(0), *DivOp1 = Div->getOperand(1); | |||
366 | ||||
367 | // If the division is exact, X % Y is zero, so we end up with X or -X. | |||
368 | if (Div->isExact()) { | |||
369 | if (DivOp1 == Y) | |||
370 | return replaceInstUsesWith(I, X); | |||
371 | return BinaryOperator::CreateNeg(X); | |||
372 | } | |||
373 | ||||
374 | auto RemOpc = Div->getOpcode() == Instruction::UDiv ? Instruction::URem | |||
375 | : Instruction::SRem; | |||
376 | // X must be frozen because we are increasing its number of uses. | |||
377 | Value *XFreeze = Builder.CreateFreeze(X, X->getName() + ".fr"); | |||
378 | Value *Rem = Builder.CreateBinOp(RemOpc, XFreeze, DivOp1); | |||
379 | if (DivOp1 == Y) | |||
380 | return BinaryOperator::CreateSub(XFreeze, Rem); | |||
381 | return BinaryOperator::CreateSub(Rem, XFreeze); | |||
382 | } | |||
383 | } | |||
384 | ||||
385 | // Fold the following two scenarios: | |||
386 | // 1) i1 mul -> i1 and. | |||
387 | // 2) X * Y --> X & Y, iff X, Y can be only {0,1}. | |||
388 | // Note: We could use known bits to generalize this and related patterns with | |||
389 | // shifts/truncs | |||
390 | if (Ty->isIntOrIntVectorTy(1) || | |||
391 | (match(Op0, m_And(m_Value(), m_One())) && | |||
392 | match(Op1, m_And(m_Value(), m_One())))) | |||
393 | return BinaryOperator::CreateAnd(Op0, Op1); | |||
394 | ||||
395 | if (Value *R = foldMulShl1(I, /* CommuteOperands */ false, Builder)) | |||
396 | return replaceInstUsesWith(I, R); | |||
397 | if (Value *R = foldMulShl1(I, /* CommuteOperands */ true, Builder)) | |||
398 | return replaceInstUsesWith(I, R); | |||
399 | ||||
400 | // (zext bool X) * (zext bool Y) --> zext (and X, Y) | |||
401 | // (sext bool X) * (sext bool Y) --> zext (and X, Y) | |||
402 | // Note: -1 * -1 == 1 * 1 == 1 (if the extends match, the result is the same) | |||
403 | if (((match(Op0, m_ZExt(m_Value(X))) && match(Op1, m_ZExt(m_Value(Y)))) || | |||
404 | (match(Op0, m_SExt(m_Value(X))) && match(Op1, m_SExt(m_Value(Y))))) && | |||
405 | X->getType()->isIntOrIntVectorTy(1) && X->getType() == Y->getType() && | |||
406 | (Op0->hasOneUse() || Op1->hasOneUse() || X == Y)) { | |||
407 | Value *And = Builder.CreateAnd(X, Y, "mulbool"); | |||
408 | return CastInst::Create(Instruction::ZExt, And, Ty); | |||
409 | } | |||
410 | // (sext bool X) * (zext bool Y) --> sext (and X, Y) | |||
411 | // (zext bool X) * (sext bool Y) --> sext (and X, Y) | |||
412 | // Note: -1 * 1 == 1 * -1 == -1 | |||
413 | if (((match(Op0, m_SExt(m_Value(X))) && match(Op1, m_ZExt(m_Value(Y)))) || | |||
414 | (match(Op0, m_ZExt(m_Value(X))) && match(Op1, m_SExt(m_Value(Y))))) && | |||
415 | X->getType()->isIntOrIntVectorTy(1) && X->getType() == Y->getType() && | |||
416 | (Op0->hasOneUse() || Op1->hasOneUse())) { | |||
417 | Value *And = Builder.CreateAnd(X, Y, "mulbool"); | |||
418 | return CastInst::Create(Instruction::SExt, And, Ty); | |||
419 | } | |||
420 | ||||
421 | // (zext bool X) * Y --> X ? Y : 0 | |||
422 | // Y * (zext bool X) --> X ? Y : 0 | |||
423 | if (match(Op0, m_ZExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) | |||
424 | return SelectInst::Create(X, Op1, ConstantInt::getNullValue(Ty)); | |||
425 | if (match(Op1, m_ZExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) | |||
426 | return SelectInst::Create(X, Op0, ConstantInt::getNullValue(Ty)); | |||
427 | ||||
428 | Constant *ImmC; | |||
429 | if (match(Op1, m_ImmConstant(ImmC))) { | |||
430 | // (sext bool X) * C --> X ? -C : 0 | |||
431 | if (match(Op0, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) { | |||
432 | Constant *NegC = ConstantExpr::getNeg(ImmC); | |||
433 | return SelectInst::Create(X, NegC, ConstantInt::getNullValue(Ty)); | |||
434 | } | |||
435 | ||||
436 | // (ashr i32 X, 31) * C --> (X < 0) ? -C : 0 | |||
437 | const APInt *C; | |||
438 | if (match(Op0, m_OneUse(m_AShr(m_Value(X), m_APInt(C)))) && | |||
439 | *C == C->getBitWidth() - 1) { | |||
440 | Constant *NegC = ConstantExpr::getNeg(ImmC); | |||
441 | Value *IsNeg = Builder.CreateIsNeg(X, "isneg"); | |||
442 | return SelectInst::Create(IsNeg, NegC, ConstantInt::getNullValue(Ty)); | |||
443 | } | |||
444 | } | |||
445 | ||||
446 | // (lshr X, 31) * Y --> (X < 0) ? Y : 0 | |||
447 | // TODO: We are not checking one-use because the elimination of the multiply | |||
448 | // is better for analysis? | |||
449 | const APInt *C; | |||
450 | if (match(&I, m_c_BinOp(m_LShr(m_Value(X), m_APInt(C)), m_Value(Y))) && | |||
451 | *C == C->getBitWidth() - 1) { | |||
452 | Value *IsNeg = Builder.CreateIsNeg(X, "isneg"); | |||
453 | return SelectInst::Create(IsNeg, Y, ConstantInt::getNullValue(Ty)); | |||
454 | } | |||
455 | ||||
456 | // (and X, 1) * Y --> (trunc X) ? Y : 0 | |||
457 | if (match(&I, m_c_BinOp(m_OneUse(m_And(m_Value(X), m_One())), m_Value(Y)))) { | |||
458 | Value *Tr = Builder.CreateTrunc(X, CmpInst::makeCmpResultType(Ty)); | |||
459 | return SelectInst::Create(Tr, Y, ConstantInt::getNullValue(Ty)); | |||
460 | } | |||
461 | ||||
462 | // ((ashr X, 31) | 1) * X --> abs(X) | |||
463 | // X * ((ashr X, 31) | 1) --> abs(X) | |||
464 | if (match(&I, m_c_BinOp(m_Or(m_AShr(m_Value(X), | |||
465 | m_SpecificIntAllowUndef(BitWidth - 1)), | |||
466 | m_One()), | |||
467 | m_Deferred(X)))) { | |||
468 | Value *Abs = Builder.CreateBinaryIntrinsic( | |||
469 | Intrinsic::abs, X, ConstantInt::getBool(I.getContext(), HasNSW)); | |||
470 | Abs->takeName(&I); | |||
471 | return replaceInstUsesWith(I, Abs); | |||
472 | } | |||
473 | ||||
474 | if (Instruction *Ext = narrowMathIfNoOverflow(I)) | |||
475 | return Ext; | |||
476 | ||||
477 | // min(X, Y) * max(X, Y) => X * Y. | |||
478 | if (match(&I, m_CombineOr(m_c_Mul(m_SMax(m_Value(X), m_Value(Y)), | |||
479 | m_c_SMin(m_Deferred(X), m_Deferred(Y))), | |||
480 | m_c_Mul(m_UMax(m_Value(X), m_Value(Y)), | |||
481 | m_c_UMin(m_Deferred(X), m_Deferred(Y)))))) | |||
482 | return BinaryOperator::CreateWithCopiedFlags(Instruction::Mul, X, Y, &I); | |||
483 | ||||
484 | // (mul Op0 Op1): | |||
485 | // if Log2(Op0) folds away -> | |||
486 | // (shl Op1, Log2(Op0)) | |||
487 | // if Log2(Op1) folds away -> | |||
488 | // (shl Op0, Log2(Op1)) | |||
489 | if (takeLog2(Builder, Op0, /*Depth*/ 0, /*AssumeNonZero*/ false, | |||
490 | /*DoFold*/ false)) { | |||
491 | Value *Res = takeLog2(Builder, Op0, /*Depth*/ 0, /*AssumeNonZero*/ false, | |||
492 | /*DoFold*/ true); | |||
493 | BinaryOperator *Shl = BinaryOperator::CreateShl(Op1, Res); | |||
494 | // We can only propegate nuw flag. | |||
495 | Shl->setHasNoUnsignedWrap(HasNUW); | |||
496 | return Shl; | |||
497 | } | |||
498 | if (takeLog2(Builder, Op1, /*Depth*/ 0, /*AssumeNonZero*/ false, | |||
499 | /*DoFold*/ false)) { | |||
500 | Value *Res = takeLog2(Builder, Op1, /*Depth*/ 0, /*AssumeNonZero*/ false, | |||
501 | /*DoFold*/ true); | |||
502 | BinaryOperator *Shl = BinaryOperator::CreateShl(Op0, Res); | |||
503 | // We can only propegate nuw flag. | |||
504 | Shl->setHasNoUnsignedWrap(HasNUW); | |||
505 | return Shl; | |||
506 | } | |||
507 | ||||
508 | bool Changed = false; | |||
509 | if (!HasNSW && willNotOverflowSignedMul(Op0, Op1, I)) { | |||
510 | Changed = true; | |||
511 | I.setHasNoSignedWrap(true); | |||
512 | } | |||
513 | ||||
514 | if (!HasNUW && willNotOverflowUnsignedMul(Op0, Op1, I)) { | |||
515 | Changed = true; | |||
516 | I.setHasNoUnsignedWrap(true); | |||
517 | } | |||
518 | ||||
519 | return Changed ? &I : nullptr; | |||
520 | } | |||
521 | ||||
522 | Instruction *InstCombinerImpl::foldFPSignBitOps(BinaryOperator &I) { | |||
523 | BinaryOperator::BinaryOps Opcode = I.getOpcode(); | |||
524 | assert((Opcode == Instruction::FMul || Opcode == Instruction::FDiv) &&(static_cast <bool> ((Opcode == Instruction::FMul || Opcode == Instruction::FDiv) && "Expected fmul or fdiv") ? void (0) : __assert_fail ("(Opcode == Instruction::FMul || Opcode == Instruction::FDiv) && \"Expected fmul or fdiv\"" , "llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp", 525, __extension__ __PRETTY_FUNCTION__)) | |||
525 | "Expected fmul or fdiv")(static_cast <bool> ((Opcode == Instruction::FMul || Opcode == Instruction::FDiv) && "Expected fmul or fdiv") ? void (0) : __assert_fail ("(Opcode == Instruction::FMul || Opcode == Instruction::FDiv) && \"Expected fmul or fdiv\"" , "llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp", 525, __extension__ __PRETTY_FUNCTION__)); | |||
526 | ||||
527 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
528 | Value *X, *Y; | |||
529 | ||||
530 | // -X * -Y --> X * Y | |||
531 | // -X / -Y --> X / Y | |||
532 | if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_FNeg(m_Value(Y)))) | |||
533 | return BinaryOperator::CreateWithCopiedFlags(Opcode, X, Y, &I); | |||
534 | ||||
535 | // fabs(X) * fabs(X) -> X * X | |||
536 | // fabs(X) / fabs(X) -> X / X | |||
537 | if (Op0 == Op1 && match(Op0, m_FAbs(m_Value(X)))) | |||
538 | return BinaryOperator::CreateWithCopiedFlags(Opcode, X, X, &I); | |||
539 | ||||
540 | // fabs(X) * fabs(Y) --> fabs(X * Y) | |||
541 | // fabs(X) / fabs(Y) --> fabs(X / Y) | |||
542 | if (match(Op0, m_FAbs(m_Value(X))) && match(Op1, m_FAbs(m_Value(Y))) && | |||
543 | (Op0->hasOneUse() || Op1->hasOneUse())) { | |||
544 | IRBuilder<>::FastMathFlagGuard FMFGuard(Builder); | |||
545 | Builder.setFastMathFlags(I.getFastMathFlags()); | |||
546 | Value *XY = Builder.CreateBinOp(Opcode, X, Y); | |||
547 | Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, XY); | |||
548 | Fabs->takeName(&I); | |||
549 | return replaceInstUsesWith(I, Fabs); | |||
550 | } | |||
551 | ||||
552 | return nullptr; | |||
553 | } | |||
554 | ||||
555 | Instruction *InstCombinerImpl::visitFMul(BinaryOperator &I) { | |||
556 | if (Value *V = simplifyFMulInst(I.getOperand(0), I.getOperand(1), | |||
557 | I.getFastMathFlags(), | |||
558 | SQ.getWithInstruction(&I))) | |||
559 | return replaceInstUsesWith(I, V); | |||
560 | ||||
561 | if (SimplifyAssociativeOrCommutative(I)) | |||
562 | return &I; | |||
563 | ||||
564 | if (Instruction *X = foldVectorBinop(I)) | |||
565 | return X; | |||
566 | ||||
567 | if (Instruction *Phi = foldBinopWithPhiOperands(I)) | |||
568 | return Phi; | |||
569 | ||||
570 | if (Instruction *FoldedMul = foldBinOpIntoSelectOrPhi(I)) | |||
571 | return FoldedMul; | |||
572 | ||||
573 | if (Value *FoldedMul = foldMulSelectToNegate(I, Builder)) | |||
574 | return replaceInstUsesWith(I, FoldedMul); | |||
575 | ||||
576 | if (Instruction *R = foldFPSignBitOps(I)) | |||
577 | return R; | |||
578 | ||||
579 | // X * -1.0 --> -X | |||
580 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
581 | if (match(Op1, m_SpecificFP(-1.0))) | |||
582 | return UnaryOperator::CreateFNegFMF(Op0, &I); | |||
583 | ||||
584 | // With no-nans: X * 0.0 --> copysign(0.0, X) | |||
585 | if (I.hasNoNaNs() && match(Op1, m_PosZeroFP())) { | |||
586 | CallInst *CopySign = Builder.CreateIntrinsic(Intrinsic::copysign, | |||
587 | {I.getType()}, {Op1, Op0}, &I); | |||
588 | return replaceInstUsesWith(I, CopySign); | |||
589 | } | |||
590 | ||||
591 | // -X * C --> X * -C | |||
592 | Value *X, *Y; | |||
593 | Constant *C; | |||
594 | if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_Constant(C))) | |||
595 | if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL)) | |||
596 | return BinaryOperator::CreateFMulFMF(X, NegC, &I); | |||
597 | ||||
598 | // (select A, B, C) * (select A, D, E) --> select A, (B*D), (C*E) | |||
599 | if (Value *V = SimplifySelectsFeedingBinaryOp(I, Op0, Op1)) | |||
600 | return replaceInstUsesWith(I, V); | |||
601 | ||||
602 | if (I.hasAllowReassoc()) { | |||
603 | // Reassociate constant RHS with another constant to form constant | |||
604 | // expression. | |||
605 | if (match(Op1, m_Constant(C)) && C->isFiniteNonZeroFP()) { | |||
606 | Constant *C1; | |||
607 | if (match(Op0, m_OneUse(m_FDiv(m_Constant(C1), m_Value(X))))) { | |||
608 | // (C1 / X) * C --> (C * C1) / X | |||
609 | Constant *CC1 = | |||
610 | ConstantFoldBinaryOpOperands(Instruction::FMul, C, C1, DL); | |||
611 | if (CC1 && CC1->isNormalFP()) | |||
612 | return BinaryOperator::CreateFDivFMF(CC1, X, &I); | |||
613 | } | |||
614 | if (match(Op0, m_FDiv(m_Value(X), m_Constant(C1)))) { | |||
615 | // (X / C1) * C --> X * (C / C1) | |||
616 | Constant *CDivC1 = | |||
617 | ConstantFoldBinaryOpOperands(Instruction::FDiv, C, C1, DL); | |||
618 | if (CDivC1 && CDivC1->isNormalFP()) | |||
619 | return BinaryOperator::CreateFMulFMF(X, CDivC1, &I); | |||
620 | ||||
621 | // If the constant was a denormal, try reassociating differently. | |||
622 | // (X / C1) * C --> X / (C1 / C) | |||
623 | Constant *C1DivC = | |||
624 | ConstantFoldBinaryOpOperands(Instruction::FDiv, C1, C, DL); | |||
625 | if (C1DivC && Op0->hasOneUse() && C1DivC->isNormalFP()) | |||
626 | return BinaryOperator::CreateFDivFMF(X, C1DivC, &I); | |||
627 | } | |||
628 | ||||
629 | // We do not need to match 'fadd C, X' and 'fsub X, C' because they are | |||
630 | // canonicalized to 'fadd X, C'. Distributing the multiply may allow | |||
631 | // further folds and (X * C) + C2 is 'fma'. | |||
632 | if (match(Op0, m_OneUse(m_FAdd(m_Value(X), m_Constant(C1))))) { | |||
633 | // (X + C1) * C --> (X * C) + (C * C1) | |||
634 | if (Constant *CC1 = ConstantFoldBinaryOpOperands( | |||
635 | Instruction::FMul, C, C1, DL)) { | |||
636 | Value *XC = Builder.CreateFMulFMF(X, C, &I); | |||
637 | return BinaryOperator::CreateFAddFMF(XC, CC1, &I); | |||
638 | } | |||
639 | } | |||
640 | if (match(Op0, m_OneUse(m_FSub(m_Constant(C1), m_Value(X))))) { | |||
641 | // (C1 - X) * C --> (C * C1) - (X * C) | |||
642 | if (Constant *CC1 = ConstantFoldBinaryOpOperands( | |||
643 | Instruction::FMul, C, C1, DL)) { | |||
644 | Value *XC = Builder.CreateFMulFMF(X, C, &I); | |||
645 | return BinaryOperator::CreateFSubFMF(CC1, XC, &I); | |||
646 | } | |||
647 | } | |||
648 | } | |||
649 | ||||
650 | Value *Z; | |||
651 | if (match(&I, m_c_FMul(m_OneUse(m_FDiv(m_Value(X), m_Value(Y))), | |||
652 | m_Value(Z)))) { | |||
653 | // Sink division: (X / Y) * Z --> (X * Z) / Y | |||
654 | Value *NewFMul = Builder.CreateFMulFMF(X, Z, &I); | |||
655 | return BinaryOperator::CreateFDivFMF(NewFMul, Y, &I); | |||
656 | } | |||
657 | ||||
658 | // sqrt(X) * sqrt(Y) -> sqrt(X * Y) | |||
659 | // nnan disallows the possibility of returning a number if both operands are | |||
660 | // negative (in that case, we should return NaN). | |||
661 | if (I.hasNoNaNs() && match(Op0, m_OneUse(m_Sqrt(m_Value(X)))) && | |||
662 | match(Op1, m_OneUse(m_Sqrt(m_Value(Y))))) { | |||
663 | Value *XY = Builder.CreateFMulFMF(X, Y, &I); | |||
664 | Value *Sqrt = Builder.CreateUnaryIntrinsic(Intrinsic::sqrt, XY, &I); | |||
665 | return replaceInstUsesWith(I, Sqrt); | |||
666 | } | |||
667 | ||||
668 | // The following transforms are done irrespective of the number of uses | |||
669 | // for the expression "1.0/sqrt(X)". | |||
670 | // 1) 1.0/sqrt(X) * X -> X/sqrt(X) | |||
671 | // 2) X * 1.0/sqrt(X) -> X/sqrt(X) | |||
672 | // We always expect the backend to reduce X/sqrt(X) to sqrt(X), if it | |||
673 | // has the necessary (reassoc) fast-math-flags. | |||
674 | if (I.hasNoSignedZeros() && | |||
675 | match(Op0, (m_FDiv(m_SpecificFP(1.0), m_Value(Y)))) && | |||
676 | match(Y, m_Sqrt(m_Value(X))) && Op1 == X) | |||
677 | return BinaryOperator::CreateFDivFMF(X, Y, &I); | |||
678 | if (I.hasNoSignedZeros() && | |||
679 | match(Op1, (m_FDiv(m_SpecificFP(1.0), m_Value(Y)))) && | |||
680 | match(Y, m_Sqrt(m_Value(X))) && Op0 == X) | |||
681 | return BinaryOperator::CreateFDivFMF(X, Y, &I); | |||
682 | ||||
683 | // Like the similar transform in instsimplify, this requires 'nsz' because | |||
684 | // sqrt(-0.0) = -0.0, and -0.0 * -0.0 does not simplify to -0.0. | |||
685 | if (I.hasNoNaNs() && I.hasNoSignedZeros() && Op0 == Op1 && | |||
686 | Op0->hasNUses(2)) { | |||
687 | // Peek through fdiv to find squaring of square root: | |||
688 | // (X / sqrt(Y)) * (X / sqrt(Y)) --> (X * X) / Y | |||
689 | if (match(Op0, m_FDiv(m_Value(X), m_Sqrt(m_Value(Y))))) { | |||
690 | Value *XX = Builder.CreateFMulFMF(X, X, &I); | |||
691 | return BinaryOperator::CreateFDivFMF(XX, Y, &I); | |||
692 | } | |||
693 | // (sqrt(Y) / X) * (sqrt(Y) / X) --> Y / (X * X) | |||
694 | if (match(Op0, m_FDiv(m_Sqrt(m_Value(Y)), m_Value(X)))) { | |||
695 | Value *XX = Builder.CreateFMulFMF(X, X, &I); | |||
696 | return BinaryOperator::CreateFDivFMF(Y, XX, &I); | |||
697 | } | |||
698 | } | |||
699 | ||||
700 | // pow(X, Y) * X --> pow(X, Y+1) | |||
701 | // X * pow(X, Y) --> pow(X, Y+1) | |||
702 | if (match(&I, m_c_FMul(m_OneUse(m_Intrinsic<Intrinsic::pow>(m_Value(X), | |||
703 | m_Value(Y))), | |||
704 | m_Deferred(X)))) { | |||
705 | Value *Y1 = | |||
706 | Builder.CreateFAddFMF(Y, ConstantFP::get(I.getType(), 1.0), &I); | |||
707 | Value *Pow = Builder.CreateBinaryIntrinsic(Intrinsic::pow, X, Y1, &I); | |||
708 | return replaceInstUsesWith(I, Pow); | |||
709 | } | |||
710 | ||||
711 | if (I.isOnlyUserOfAnyOperand()) { | |||
712 | // pow(X, Y) * pow(X, Z) -> pow(X, Y + Z) | |||
713 | if (match(Op0, m_Intrinsic<Intrinsic::pow>(m_Value(X), m_Value(Y))) && | |||
714 | match(Op1, m_Intrinsic<Intrinsic::pow>(m_Specific(X), m_Value(Z)))) { | |||
715 | auto *YZ = Builder.CreateFAddFMF(Y, Z, &I); | |||
716 | auto *NewPow = Builder.CreateBinaryIntrinsic(Intrinsic::pow, X, YZ, &I); | |||
717 | return replaceInstUsesWith(I, NewPow); | |||
718 | } | |||
719 | // pow(X, Y) * pow(Z, Y) -> pow(X * Z, Y) | |||
720 | if (match(Op0, m_Intrinsic<Intrinsic::pow>(m_Value(X), m_Value(Y))) && | |||
721 | match(Op1, m_Intrinsic<Intrinsic::pow>(m_Value(Z), m_Specific(Y)))) { | |||
722 | auto *XZ = Builder.CreateFMulFMF(X, Z, &I); | |||
723 | auto *NewPow = Builder.CreateBinaryIntrinsic(Intrinsic::pow, XZ, Y, &I); | |||
724 | return replaceInstUsesWith(I, NewPow); | |||
725 | } | |||
726 | ||||
727 | // powi(x, y) * powi(x, z) -> powi(x, y + z) | |||
728 | if (match(Op0, m_Intrinsic<Intrinsic::powi>(m_Value(X), m_Value(Y))) && | |||
729 | match(Op1, m_Intrinsic<Intrinsic::powi>(m_Specific(X), m_Value(Z))) && | |||
730 | Y->getType() == Z->getType()) { | |||
731 | auto *YZ = Builder.CreateAdd(Y, Z); | |||
732 | auto *NewPow = Builder.CreateIntrinsic( | |||
733 | Intrinsic::powi, {X->getType(), YZ->getType()}, {X, YZ}, &I); | |||
734 | return replaceInstUsesWith(I, NewPow); | |||
735 | } | |||
736 | ||||
737 | // exp(X) * exp(Y) -> exp(X + Y) | |||
738 | if (match(Op0, m_Intrinsic<Intrinsic::exp>(m_Value(X))) && | |||
739 | match(Op1, m_Intrinsic<Intrinsic::exp>(m_Value(Y)))) { | |||
740 | Value *XY = Builder.CreateFAddFMF(X, Y, &I); | |||
741 | Value *Exp = Builder.CreateUnaryIntrinsic(Intrinsic::exp, XY, &I); | |||
742 | return replaceInstUsesWith(I, Exp); | |||
743 | } | |||
744 | ||||
745 | // exp2(X) * exp2(Y) -> exp2(X + Y) | |||
746 | if (match(Op0, m_Intrinsic<Intrinsic::exp2>(m_Value(X))) && | |||
747 | match(Op1, m_Intrinsic<Intrinsic::exp2>(m_Value(Y)))) { | |||
748 | Value *XY = Builder.CreateFAddFMF(X, Y, &I); | |||
749 | Value *Exp2 = Builder.CreateUnaryIntrinsic(Intrinsic::exp2, XY, &I); | |||
750 | return replaceInstUsesWith(I, Exp2); | |||
751 | } | |||
752 | } | |||
753 | ||||
754 | // (X*Y) * X => (X*X) * Y where Y != X | |||
755 | // The purpose is two-fold: | |||
756 | // 1) to form a power expression (of X). | |||
757 | // 2) potentially shorten the critical path: After transformation, the | |||
758 | // latency of the instruction Y is amortized by the expression of X*X, | |||
759 | // and therefore Y is in a "less critical" position compared to what it | |||
760 | // was before the transformation. | |||
761 | if (match(Op0, m_OneUse(m_c_FMul(m_Specific(Op1), m_Value(Y)))) && | |||
762 | Op1 != Y) { | |||
763 | Value *XX = Builder.CreateFMulFMF(Op1, Op1, &I); | |||
764 | return BinaryOperator::CreateFMulFMF(XX, Y, &I); | |||
765 | } | |||
766 | if (match(Op1, m_OneUse(m_c_FMul(m_Specific(Op0), m_Value(Y)))) && | |||
767 | Op0 != Y) { | |||
768 | Value *XX = Builder.CreateFMulFMF(Op0, Op0, &I); | |||
769 | return BinaryOperator::CreateFMulFMF(XX, Y, &I); | |||
770 | } | |||
771 | } | |||
772 | ||||
773 | // log2(X * 0.5) * Y = log2(X) * Y - Y | |||
774 | if (I.isFast()) { | |||
775 | IntrinsicInst *Log2 = nullptr; | |||
776 | if (match(Op0, m_OneUse(m_Intrinsic<Intrinsic::log2>( | |||
777 | m_OneUse(m_FMul(m_Value(X), m_SpecificFP(0.5))))))) { | |||
778 | Log2 = cast<IntrinsicInst>(Op0); | |||
779 | Y = Op1; | |||
780 | } | |||
781 | if (match(Op1, m_OneUse(m_Intrinsic<Intrinsic::log2>( | |||
782 | m_OneUse(m_FMul(m_Value(X), m_SpecificFP(0.5))))))) { | |||
783 | Log2 = cast<IntrinsicInst>(Op1); | |||
784 | Y = Op0; | |||
785 | } | |||
786 | if (Log2) { | |||
787 | Value *Log2 = Builder.CreateUnaryIntrinsic(Intrinsic::log2, X, &I); | |||
788 | Value *LogXTimesY = Builder.CreateFMulFMF(Log2, Y, &I); | |||
789 | return BinaryOperator::CreateFSubFMF(LogXTimesY, Y, &I); | |||
790 | } | |||
791 | } | |||
792 | ||||
793 | // Simplify FMUL recurrences starting with 0.0 to 0.0 if nnan and nsz are set. | |||
794 | // Given a phi node with entry value as 0 and it used in fmul operation, | |||
795 | // we can replace fmul with 0 safely and eleminate loop operation. | |||
796 | PHINode *PN = nullptr; | |||
797 | Value *Start = nullptr, *Step = nullptr; | |||
798 | if (matchSimpleRecurrence(&I, PN, Start, Step) && I.hasNoNaNs() && | |||
799 | I.hasNoSignedZeros() && match(Start, m_Zero())) | |||
800 | return replaceInstUsesWith(I, Start); | |||
801 | ||||
802 | // minimun(X, Y) * maximum(X, Y) => X * Y. | |||
803 | if (match(&I, | |||
804 | m_c_FMul(m_Intrinsic<Intrinsic::maximum>(m_Value(X), m_Value(Y)), | |||
805 | m_c_Intrinsic<Intrinsic::minimum>(m_Deferred(X), | |||
806 | m_Deferred(Y))))) { | |||
807 | BinaryOperator *Result = BinaryOperator::CreateFMulFMF(X, Y, &I); | |||
808 | // We cannot preserve ninf if nnan flag is not set. | |||
809 | // If X is NaN and Y is Inf then in original program we had NaN * NaN, | |||
810 | // while in optimized version NaN * Inf and this is a poison with ninf flag. | |||
811 | if (!Result->hasNoNaNs()) | |||
812 | Result->setHasNoInfs(false); | |||
813 | return Result; | |||
814 | } | |||
815 | ||||
816 | return nullptr; | |||
817 | } | |||
818 | ||||
819 | /// Fold a divide or remainder with a select instruction divisor when one of the | |||
820 | /// select operands is zero. In that case, we can use the other select operand | |||
821 | /// because div/rem by zero is undefined. | |||
822 | bool InstCombinerImpl::simplifyDivRemOfSelectWithZeroOp(BinaryOperator &I) { | |||
823 | SelectInst *SI = dyn_cast<SelectInst>(I.getOperand(1)); | |||
824 | if (!SI) | |||
825 | return false; | |||
826 | ||||
827 | int NonNullOperand; | |||
828 | if (match(SI->getTrueValue(), m_Zero())) | |||
829 | // div/rem X, (Cond ? 0 : Y) -> div/rem X, Y | |||
830 | NonNullOperand = 2; | |||
831 | else if (match(SI->getFalseValue(), m_Zero())) | |||
832 | // div/rem X, (Cond ? Y : 0) -> div/rem X, Y | |||
833 | NonNullOperand = 1; | |||
834 | else | |||
835 | return false; | |||
836 | ||||
837 | // Change the div/rem to use 'Y' instead of the select. | |||
838 | replaceOperand(I, 1, SI->getOperand(NonNullOperand)); | |||
839 | ||||
840 | // Okay, we know we replace the operand of the div/rem with 'Y' with no | |||
841 | // problem. However, the select, or the condition of the select may have | |||
842 | // multiple uses. Based on our knowledge that the operand must be non-zero, | |||
843 | // propagate the known value for the select into other uses of it, and | |||
844 | // propagate a known value of the condition into its other users. | |||
845 | ||||
846 | // If the select and condition only have a single use, don't bother with this, | |||
847 | // early exit. | |||
848 | Value *SelectCond = SI->getCondition(); | |||
849 | if (SI->use_empty() && SelectCond->hasOneUse()) | |||
850 | return true; | |||
851 | ||||
852 | // Scan the current block backward, looking for other uses of SI. | |||
853 | BasicBlock::iterator BBI = I.getIterator(), BBFront = I.getParent()->begin(); | |||
854 | Type *CondTy = SelectCond->getType(); | |||
855 | while (BBI != BBFront) { | |||
856 | --BBI; | |||
857 | // If we found an instruction that we can't assume will return, so | |||
858 | // information from below it cannot be propagated above it. | |||
859 | if (!isGuaranteedToTransferExecutionToSuccessor(&*BBI)) | |||
860 | break; | |||
861 | ||||
862 | // Replace uses of the select or its condition with the known values. | |||
863 | for (Use &Op : BBI->operands()) { | |||
864 | if (Op == SI) { | |||
865 | replaceUse(Op, SI->getOperand(NonNullOperand)); | |||
| ||||
866 | Worklist.push(&*BBI); | |||
867 | } else if (Op == SelectCond) { | |||
868 | replaceUse(Op, NonNullOperand == 1 ? ConstantInt::getTrue(CondTy) | |||
869 | : ConstantInt::getFalse(CondTy)); | |||
870 | Worklist.push(&*BBI); | |||
871 | } | |||
872 | } | |||
873 | ||||
874 | // If we past the instruction, quit looking for it. | |||
875 | if (&*BBI == SI) | |||
876 | SI = nullptr; | |||
877 | if (&*BBI == SelectCond) | |||
878 | SelectCond = nullptr; | |||
879 | ||||
880 | // If we ran out of things to eliminate, break out of the loop. | |||
881 | if (!SelectCond
| |||
882 | break; | |||
883 | ||||
884 | } | |||
885 | return true; | |||
886 | } | |||
887 | ||||
888 | /// True if the multiply can not be expressed in an int this size. | |||
889 | static bool multiplyOverflows(const APInt &C1, const APInt &C2, APInt &Product, | |||
890 | bool IsSigned) { | |||
891 | bool Overflow; | |||
892 | Product = IsSigned ? C1.smul_ov(C2, Overflow) : C1.umul_ov(C2, Overflow); | |||
893 | return Overflow; | |||
894 | } | |||
895 | ||||
896 | /// True if C1 is a multiple of C2. Quotient contains C1/C2. | |||
897 | static bool isMultiple(const APInt &C1, const APInt &C2, APInt &Quotient, | |||
898 | bool IsSigned) { | |||
899 | assert(C1.getBitWidth() == C2.getBitWidth() && "Constant widths not equal")(static_cast <bool> (C1.getBitWidth() == C2.getBitWidth () && "Constant widths not equal") ? void (0) : __assert_fail ("C1.getBitWidth() == C2.getBitWidth() && \"Constant widths not equal\"" , "llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp", 899, __extension__ __PRETTY_FUNCTION__)); | |||
900 | ||||
901 | // Bail if we will divide by zero. | |||
902 | if (C2.isZero()) | |||
903 | return false; | |||
904 | ||||
905 | // Bail if we would divide INT_MIN by -1. | |||
906 | if (IsSigned && C1.isMinSignedValue() && C2.isAllOnes()) | |||
907 | return false; | |||
908 | ||||
909 | APInt Remainder(C1.getBitWidth(), /*val=*/0ULL, IsSigned); | |||
910 | if (IsSigned) | |||
911 | APInt::sdivrem(C1, C2, Quotient, Remainder); | |||
912 | else | |||
913 | APInt::udivrem(C1, C2, Quotient, Remainder); | |||
914 | ||||
915 | return Remainder.isMinValue(); | |||
916 | } | |||
917 | ||||
918 | static Instruction *foldIDivShl(BinaryOperator &I, | |||
919 | InstCombiner::BuilderTy &Builder) { | |||
920 | assert((I.getOpcode() == Instruction::SDiv ||(static_cast <bool> ((I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::UDiv) && "Expected integer divide" ) ? void (0) : __assert_fail ("(I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::UDiv) && \"Expected integer divide\"" , "llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp", 922, __extension__ __PRETTY_FUNCTION__)) | |||
921 | I.getOpcode() == Instruction::UDiv) &&(static_cast <bool> ((I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::UDiv) && "Expected integer divide" ) ? void (0) : __assert_fail ("(I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::UDiv) && \"Expected integer divide\"" , "llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp", 922, __extension__ __PRETTY_FUNCTION__)) | |||
922 | "Expected integer divide")(static_cast <bool> ((I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::UDiv) && "Expected integer divide" ) ? void (0) : __assert_fail ("(I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::UDiv) && \"Expected integer divide\"" , "llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp", 922, __extension__ __PRETTY_FUNCTION__)); | |||
923 | ||||
924 | bool IsSigned = I.getOpcode() == Instruction::SDiv; | |||
925 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
926 | Type *Ty = I.getType(); | |||
927 | ||||
928 | Instruction *Ret = nullptr; | |||
929 | Value *X, *Y, *Z; | |||
930 | ||||
931 | // With appropriate no-wrap constraints, remove a common factor in the | |||
932 | // dividend and divisor that is disguised as a left-shifted value. | |||
933 | if (match(Op1, m_Shl(m_Value(X), m_Value(Z))) && | |||
934 | match(Op0, m_c_Mul(m_Specific(X), m_Value(Y)))) { | |||
935 | // Both operands must have the matching no-wrap for this kind of division. | |||
936 | auto *Mul = cast<OverflowingBinaryOperator>(Op0); | |||
937 | auto *Shl = cast<OverflowingBinaryOperator>(Op1); | |||
938 | bool HasNUW = Mul->hasNoUnsignedWrap() && Shl->hasNoUnsignedWrap(); | |||
939 | bool HasNSW = Mul->hasNoSignedWrap() && Shl->hasNoSignedWrap(); | |||
940 | ||||
941 | // (X * Y) u/ (X << Z) --> Y u>> Z | |||
942 | if (!IsSigned && HasNUW) | |||
943 | Ret = BinaryOperator::CreateLShr(Y, Z); | |||
944 | ||||
945 | // (X * Y) s/ (X << Z) --> Y s/ (1 << Z) | |||
946 | if (IsSigned && HasNSW && (Op0->hasOneUse() || Op1->hasOneUse())) { | |||
947 | Value *Shl = Builder.CreateShl(ConstantInt::get(Ty, 1), Z); | |||
948 | Ret = BinaryOperator::CreateSDiv(Y, Shl); | |||
949 | } | |||
950 | } | |||
951 | ||||
952 | // With appropriate no-wrap constraints, remove a common factor in the | |||
953 | // dividend and divisor that is disguised as a left-shift amount. | |||
954 | if (match(Op0, m_Shl(m_Value(X), m_Value(Z))) && | |||
955 | match(Op1, m_Shl(m_Value(Y), m_Specific(Z)))) { | |||
956 | auto *Shl0 = cast<OverflowingBinaryOperator>(Op0); | |||
957 | auto *Shl1 = cast<OverflowingBinaryOperator>(Op1); | |||
958 | ||||
959 | // For unsigned div, we need 'nuw' on both shifts or | |||
960 | // 'nsw' on both shifts + 'nuw' on the dividend. | |||
961 | // (X << Z) / (Y << Z) --> X / Y | |||
962 | if (!IsSigned && | |||
963 | ((Shl0->hasNoUnsignedWrap() && Shl1->hasNoUnsignedWrap()) || | |||
964 | (Shl0->hasNoUnsignedWrap() && Shl0->hasNoSignedWrap() && | |||
965 | Shl1->hasNoSignedWrap()))) | |||
966 | Ret = BinaryOperator::CreateUDiv(X, Y); | |||
967 | ||||
968 | // For signed div, we need 'nsw' on both shifts + 'nuw' on the divisor. | |||
969 | // (X << Z) / (Y << Z) --> X / Y | |||
970 | if (IsSigned && Shl0->hasNoSignedWrap() && Shl1->hasNoSignedWrap() && | |||
971 | Shl1->hasNoUnsignedWrap()) | |||
972 | Ret = BinaryOperator::CreateSDiv(X, Y); | |||
973 | } | |||
974 | ||||
975 | if (!Ret) | |||
976 | return nullptr; | |||
977 | ||||
978 | Ret->setIsExact(I.isExact()); | |||
979 | return Ret; | |||
980 | } | |||
981 | ||||
982 | /// This function implements the transforms common to both integer division | |||
983 | /// instructions (udiv and sdiv). It is called by the visitors to those integer | |||
984 | /// division instructions. | |||
985 | /// Common integer divide transforms | |||
986 | Instruction *InstCombinerImpl::commonIDivTransforms(BinaryOperator &I) { | |||
987 | if (Instruction *Phi = foldBinopWithPhiOperands(I)) | |||
| ||||
988 | return Phi; | |||
989 | ||||
990 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
991 | bool IsSigned = I.getOpcode() == Instruction::SDiv; | |||
992 | Type *Ty = I.getType(); | |||
993 | ||||
994 | // The RHS is known non-zero. | |||
995 | if (Value *V = simplifyValueKnownNonZero(I.getOperand(1), *this, I)) | |||
996 | return replaceOperand(I, 1, V); | |||
997 | ||||
998 | // Handle cases involving: [su]div X, (select Cond, Y, Z) | |||
999 | // This does not apply for fdiv. | |||
1000 | if (simplifyDivRemOfSelectWithZeroOp(I)) | |||
1001 | return &I; | |||
1002 | ||||
1003 | // If the divisor is a select-of-constants, try to constant fold all div ops: | |||
1004 | // C / (select Cond, TrueC, FalseC) --> select Cond, (C / TrueC), (C / FalseC) | |||
1005 | // TODO: Adapt simplifyDivRemOfSelectWithZeroOp to allow this and other folds. | |||
1006 | if (match(Op0, m_ImmConstant()) && | |||
1007 | match(Op1, m_Select(m_Value(), m_ImmConstant(), m_ImmConstant()))) { | |||
1008 | if (Instruction *R = FoldOpIntoSelect(I, cast<SelectInst>(Op1), | |||
1009 | /*FoldWithMultiUse*/ true)) | |||
1010 | return R; | |||
1011 | } | |||
1012 | ||||
1013 | const APInt *C2; | |||
1014 | if (match(Op1, m_APInt(C2))) { | |||
1015 | Value *X; | |||
1016 | const APInt *C1; | |||
1017 | ||||
1018 | // (X / C1) / C2 -> X / (C1*C2) | |||
1019 | if ((IsSigned && match(Op0, m_SDiv(m_Value(X), m_APInt(C1)))) || | |||
1020 | (!IsSigned && match(Op0, m_UDiv(m_Value(X), m_APInt(C1))))) { | |||
1021 | APInt Product(C1->getBitWidth(), /*val=*/0ULL, IsSigned); | |||
1022 | if (!multiplyOverflows(*C1, *C2, Product, IsSigned)) | |||
1023 | return BinaryOperator::Create(I.getOpcode(), X, | |||
1024 | ConstantInt::get(Ty, Product)); | |||
1025 | } | |||
1026 | ||||
1027 | APInt Quotient(C2->getBitWidth(), /*val=*/0ULL, IsSigned); | |||
1028 | if ((IsSigned && match(Op0, m_NSWMul(m_Value(X), m_APInt(C1)))) || | |||
1029 | (!IsSigned && match(Op0, m_NUWMul(m_Value(X), m_APInt(C1))))) { | |||
1030 | ||||
1031 | // (X * C1) / C2 -> X / (C2 / C1) if C2 is a multiple of C1. | |||
1032 | if (isMultiple(*C2, *C1, Quotient, IsSigned)) { | |||
1033 | auto *NewDiv = BinaryOperator::Create(I.getOpcode(), X, | |||
1034 | ConstantInt::get(Ty, Quotient)); | |||
1035 | NewDiv->setIsExact(I.isExact()); | |||
1036 | return NewDiv; | |||
1037 | } | |||
1038 | ||||
1039 | // (X * C1) / C2 -> X * (C1 / C2) if C1 is a multiple of C2. | |||
1040 | if (isMultiple(*C1, *C2, Quotient, IsSigned)) { | |||
1041 | auto *Mul = BinaryOperator::Create(Instruction::Mul, X, | |||
1042 | ConstantInt::get(Ty, Quotient)); | |||
1043 | auto *OBO = cast<OverflowingBinaryOperator>(Op0); | |||
1044 | Mul->setHasNoUnsignedWrap(!IsSigned && OBO->hasNoUnsignedWrap()); | |||
1045 | Mul->setHasNoSignedWrap(OBO->hasNoSignedWrap()); | |||
1046 | return Mul; | |||
1047 | } | |||
1048 | } | |||
1049 | ||||
1050 | if ((IsSigned && match(Op0, m_NSWShl(m_Value(X), m_APInt(C1))) && | |||
1051 | C1->ult(C1->getBitWidth() - 1)) || | |||
1052 | (!IsSigned && match(Op0, m_NUWShl(m_Value(X), m_APInt(C1))) && | |||
1053 | C1->ult(C1->getBitWidth()))) { | |||
1054 | APInt C1Shifted = APInt::getOneBitSet( | |||
1055 | C1->getBitWidth(), static_cast<unsigned>(C1->getZExtValue())); | |||
1056 | ||||
1057 | // (X << C1) / C2 -> X / (C2 >> C1) if C2 is a multiple of 1 << C1. | |||
1058 | if (isMultiple(*C2, C1Shifted, Quotient, IsSigned)) { | |||
1059 | auto *BO = BinaryOperator::Create(I.getOpcode(), X, | |||
1060 | ConstantInt::get(Ty, Quotient)); | |||
1061 | BO->setIsExact(I.isExact()); | |||
1062 | return BO; | |||
1063 | } | |||
1064 | ||||
1065 | // (X << C1) / C2 -> X * ((1 << C1) / C2) if 1 << C1 is a multiple of C2. | |||
1066 | if (isMultiple(C1Shifted, *C2, Quotient, IsSigned)) { | |||
1067 | auto *Mul = BinaryOperator::Create(Instruction::Mul, X, | |||
1068 | ConstantInt::get(Ty, Quotient)); | |||
1069 | auto *OBO = cast<OverflowingBinaryOperator>(Op0); | |||
1070 | Mul->setHasNoUnsignedWrap(!IsSigned && OBO->hasNoUnsignedWrap()); | |||
1071 | Mul->setHasNoSignedWrap(OBO->hasNoSignedWrap()); | |||
1072 | return Mul; | |||
1073 | } | |||
1074 | } | |||
1075 | ||||
1076 | // Distribute div over add to eliminate a matching div/mul pair: | |||
1077 | // ((X * C2) + C1) / C2 --> X + C1/C2 | |||
1078 | // We need a multiple of the divisor for a signed add constant, but | |||
1079 | // unsigned is fine with any constant pair. | |||
1080 | if (IsSigned && | |||
1081 | match(Op0, m_NSWAdd(m_NSWMul(m_Value(X), m_SpecificInt(*C2)), | |||
1082 | m_APInt(C1))) && | |||
1083 | isMultiple(*C1, *C2, Quotient, IsSigned)) { | |||
1084 | return BinaryOperator::CreateNSWAdd(X, ConstantInt::get(Ty, Quotient)); | |||
1085 | } | |||
1086 | if (!IsSigned && | |||
1087 | match(Op0, m_NUWAdd(m_NUWMul(m_Value(X), m_SpecificInt(*C2)), | |||
1088 | m_APInt(C1)))) { | |||
1089 | return BinaryOperator::CreateNUWAdd(X, | |||
1090 | ConstantInt::get(Ty, C1->udiv(*C2))); | |||
1091 | } | |||
1092 | ||||
1093 | if (!C2->isZero()) // avoid X udiv 0 | |||
1094 | if (Instruction *FoldedDiv = foldBinOpIntoSelectOrPhi(I)) | |||
1095 | return FoldedDiv; | |||
1096 | } | |||
1097 | ||||
1098 | if (match(Op0, m_One())) { | |||
1099 | assert(!Ty->isIntOrIntVectorTy(1) && "i1 divide not removed?")(static_cast <bool> (!Ty->isIntOrIntVectorTy(1) && "i1 divide not removed?") ? void (0) : __assert_fail ("!Ty->isIntOrIntVectorTy(1) && \"i1 divide not removed?\"" , "llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp", 1099, __extension__ __PRETTY_FUNCTION__)); | |||
1100 | if (IsSigned) { | |||
1101 | // 1 / 0 --> undef ; 1 / 1 --> 1 ; 1 / -1 --> -1 ; 1 / anything else --> 0 | |||
1102 | // (Op1 + 1) u< 3 ? Op1 : 0 | |||
1103 | // Op1 must be frozen because we are increasing its number of uses. | |||
1104 | Value *F1 = Builder.CreateFreeze(Op1, Op1->getName() + ".fr"); | |||
1105 | Value *Inc = Builder.CreateAdd(F1, Op0); | |||
1106 | Value *Cmp = Builder.CreateICmpULT(Inc, ConstantInt::get(Ty, 3)); | |||
1107 | return SelectInst::Create(Cmp, F1, ConstantInt::get(Ty, 0)); | |||
1108 | } else { | |||
1109 | // If Op1 is 0 then it's undefined behaviour. If Op1 is 1 then the | |||
1110 | // result is one, otherwise it's zero. | |||
1111 | return new ZExtInst(Builder.CreateICmpEQ(Op1, Op0), Ty); | |||
1112 | } | |||
1113 | } | |||
1114 | ||||
1115 | // See if we can fold away this div instruction. | |||
1116 | if (SimplifyDemandedInstructionBits(I)) | |||
1117 | return &I; | |||
1118 | ||||
1119 | // (X - (X rem Y)) / Y -> X / Y; usually originates as ((X / Y) * Y) / Y | |||
1120 | Value *X, *Z; | |||
1121 | if (match(Op0, m_Sub(m_Value(X), m_Value(Z)))) // (X - Z) / Y; Y = Op1 | |||
1122 | if ((IsSigned && match(Z, m_SRem(m_Specific(X), m_Specific(Op1)))) || | |||
1123 | (!IsSigned && match(Z, m_URem(m_Specific(X), m_Specific(Op1))))) | |||
1124 | return BinaryOperator::Create(I.getOpcode(), X, Op1); | |||
1125 | ||||
1126 | // (X << Y) / X -> 1 << Y | |||
1127 | Value *Y; | |||
1128 | if (IsSigned && match(Op0, m_NSWShl(m_Specific(Op1), m_Value(Y)))) | |||
1129 | return BinaryOperator::CreateNSWShl(ConstantInt::get(Ty, 1), Y); | |||
1130 | if (!IsSigned && match(Op0, m_NUWShl(m_Specific(Op1), m_Value(Y)))) | |||
1131 | return BinaryOperator::CreateNUWShl(ConstantInt::get(Ty, 1), Y); | |||
1132 | ||||
1133 | // X / (X * Y) -> 1 / Y if the multiplication does not overflow. | |||
1134 | if (match(Op1, m_c_Mul(m_Specific(Op0), m_Value(Y)))) { | |||
1135 | bool HasNSW = cast<OverflowingBinaryOperator>(Op1)->hasNoSignedWrap(); | |||
1136 | bool HasNUW = cast<OverflowingBinaryOperator>(Op1)->hasNoUnsignedWrap(); | |||
1137 | if ((IsSigned && HasNSW) || (!IsSigned && HasNUW)) { | |||
1138 | replaceOperand(I, 0, ConstantInt::get(Ty, 1)); | |||
1139 | replaceOperand(I, 1, Y); | |||
1140 | return &I; | |||
1141 | } | |||
1142 | } | |||
1143 | ||||
1144 | // (X << Z) / (X * Y) -> (1 << Z) / Y | |||
1145 | // TODO: Handle sdiv. | |||
1146 | if (!IsSigned && Op1->hasOneUse() && | |||
1147 | match(Op0, m_NUWShl(m_Value(X), m_Value(Z))) && | |||
1148 | match(Op1, m_c_Mul(m_Specific(X), m_Value(Y)))) | |||
1149 | if (cast<OverflowingBinaryOperator>(Op1)->hasNoUnsignedWrap()) { | |||
1150 | Instruction *NewDiv = BinaryOperator::CreateUDiv( | |||
1151 | Builder.CreateShl(ConstantInt::get(Ty, 1), Z, "", /*NUW*/ true), Y); | |||
1152 | NewDiv->setIsExact(I.isExact()); | |||
1153 | return NewDiv; | |||
1154 | } | |||
1155 | ||||
1156 | if (Instruction *R = foldIDivShl(I, Builder)) | |||
1157 | return R; | |||
1158 | ||||
1159 | // With the appropriate no-wrap constraint, remove a multiply by the divisor | |||
1160 | // after peeking through another divide: | |||
1161 | // ((Op1 * X) / Y) / Op1 --> X / Y | |||
1162 | if (match(Op0, m_BinOp(I.getOpcode(), m_c_Mul(m_Specific(Op1), m_Value(X)), | |||
1163 | m_Value(Y)))) { | |||
1164 | auto *InnerDiv = cast<PossiblyExactOperator>(Op0); | |||
1165 | auto *Mul = cast<OverflowingBinaryOperator>(InnerDiv->getOperand(0)); | |||
1166 | Instruction *NewDiv = nullptr; | |||
1167 | if (!IsSigned && Mul->hasNoUnsignedWrap()) | |||
1168 | NewDiv = BinaryOperator::CreateUDiv(X, Y); | |||
1169 | else if (IsSigned && Mul->hasNoSignedWrap()) | |||
1170 | NewDiv = BinaryOperator::CreateSDiv(X, Y); | |||
1171 | ||||
1172 | // Exact propagates only if both of the original divides are exact. | |||
1173 | if (NewDiv) { | |||
1174 | NewDiv->setIsExact(I.isExact() && InnerDiv->isExact()); | |||
1175 | return NewDiv; | |||
1176 | } | |||
1177 | } | |||
1178 | ||||
1179 | return nullptr; | |||
1180 | } | |||
1181 | ||||
1182 | static const unsigned MaxDepth = 6; | |||
1183 | ||||
1184 | // Take the exact integer log2 of the value. If DoFold is true, create the | |||
1185 | // actual instructions, otherwise return a non-null dummy value. Return nullptr | |||
1186 | // on failure. | |||
1187 | static Value *takeLog2(IRBuilderBase &Builder, Value *Op, unsigned Depth, | |||
1188 | bool AssumeNonZero, bool DoFold) { | |||
1189 | auto IfFold = [DoFold](function_ref<Value *()> Fn) { | |||
1190 | if (!DoFold) | |||
1191 | return reinterpret_cast<Value *>(-1); | |||
1192 | return Fn(); | |||
1193 | }; | |||
1194 | ||||
1195 | // FIXME: assert that Op1 isn't/doesn't contain undef. | |||
1196 | ||||
1197 | // log2(2^C) -> C | |||
1198 | if (match(Op, m_Power2())) | |||
1199 | return IfFold([&]() { | |||
1200 | Constant *C = ConstantExpr::getExactLogBase2(cast<Constant>(Op)); | |||
1201 | if (!C) | |||
1202 | llvm_unreachable("Failed to constant fold udiv -> logbase2")::llvm::llvm_unreachable_internal("Failed to constant fold udiv -> logbase2" , "llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp", 1202); | |||
1203 | return C; | |||
1204 | }); | |||
1205 | ||||
1206 | // The remaining tests are all recursive, so bail out if we hit the limit. | |||
1207 | if (Depth++ == MaxDepth) | |||
1208 | return nullptr; | |||
1209 | ||||
1210 | // log2(zext X) -> zext log2(X) | |||
1211 | // FIXME: Require one use? | |||
1212 | Value *X, *Y; | |||
1213 | if (match(Op, m_ZExt(m_Value(X)))) | |||
1214 | if (Value *LogX = takeLog2(Builder, X, Depth, AssumeNonZero, DoFold)) | |||
1215 | return IfFold([&]() { return Builder.CreateZExt(LogX, Op->getType()); }); | |||
1216 | ||||
1217 | // log2(X << Y) -> log2(X) + Y | |||
1218 | // FIXME: Require one use unless X is 1? | |||
1219 | if (match(Op, m_Shl(m_Value(X), m_Value(Y)))) { | |||
1220 | auto *BO = cast<OverflowingBinaryOperator>(Op); | |||
1221 | // nuw will be set if the `shl` is trivially non-zero. | |||
1222 | if (AssumeNonZero || BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap()) | |||
1223 | if (Value *LogX = takeLog2(Builder, X, Depth, AssumeNonZero, DoFold)) | |||
1224 | return IfFold([&]() { return Builder.CreateAdd(LogX, Y); }); | |||
1225 | } | |||
1226 | ||||
1227 | // log2(Cond ? X : Y) -> Cond ? log2(X) : log2(Y) | |||
1228 | // FIXME: missed optimization: if one of the hands of select is/contains | |||
1229 | // undef, just directly pick the other one. | |||
1230 | // FIXME: can both hands contain undef? | |||
1231 | // FIXME: Require one use? | |||
1232 | if (SelectInst *SI = dyn_cast<SelectInst>(Op)) | |||
1233 | if (Value *LogX = takeLog2(Builder, SI->getOperand(1), Depth, | |||
1234 | AssumeNonZero, DoFold)) | |||
1235 | if (Value *LogY = takeLog2(Builder, SI->getOperand(2), Depth, | |||
1236 | AssumeNonZero, DoFold)) | |||
1237 | return IfFold([&]() { | |||
1238 | return Builder.CreateSelect(SI->getOperand(0), LogX, LogY); | |||
1239 | }); | |||
1240 | ||||
1241 | // log2(umin(X, Y)) -> umin(log2(X), log2(Y)) | |||
1242 | // log2(umax(X, Y)) -> umax(log2(X), log2(Y)) | |||
1243 | auto *MinMax = dyn_cast<MinMaxIntrinsic>(Op); | |||
1244 | if (MinMax && MinMax->hasOneUse() && !MinMax->isSigned()) { | |||
1245 | // Use AssumeNonZero as false here. Otherwise we can hit case where | |||
1246 | // log2(umax(X, Y)) != umax(log2(X), log2(Y)) (because overflow). | |||
1247 | if (Value *LogX = takeLog2(Builder, MinMax->getLHS(), Depth, | |||
1248 | /*AssumeNonZero*/ false, DoFold)) | |||
1249 | if (Value *LogY = takeLog2(Builder, MinMax->getRHS(), Depth, | |||
1250 | /*AssumeNonZero*/ false, DoFold)) | |||
1251 | return IfFold([&]() { | |||
1252 | return Builder.CreateBinaryIntrinsic(MinMax->getIntrinsicID(), LogX, | |||
1253 | LogY); | |||
1254 | }); | |||
1255 | } | |||
1256 | ||||
1257 | return nullptr; | |||
1258 | } | |||
1259 | ||||
1260 | /// If we have zero-extended operands of an unsigned div or rem, we may be able | |||
1261 | /// to narrow the operation (sink the zext below the math). | |||
1262 | static Instruction *narrowUDivURem(BinaryOperator &I, | |||
1263 | InstCombiner::BuilderTy &Builder) { | |||
1264 | Instruction::BinaryOps Opcode = I.getOpcode(); | |||
1265 | Value *N = I.getOperand(0); | |||
1266 | Value *D = I.getOperand(1); | |||
1267 | Type *Ty = I.getType(); | |||
1268 | Value *X, *Y; | |||
1269 | if (match(N, m_ZExt(m_Value(X))) && match(D, m_ZExt(m_Value(Y))) && | |||
1270 | X->getType() == Y->getType() && (N->hasOneUse() || D->hasOneUse())) { | |||
1271 | // udiv (zext X), (zext Y) --> zext (udiv X, Y) | |||
1272 | // urem (zext X), (zext Y) --> zext (urem X, Y) | |||
1273 | Value *NarrowOp = Builder.CreateBinOp(Opcode, X, Y); | |||
1274 | return new ZExtInst(NarrowOp, Ty); | |||
1275 | } | |||
1276 | ||||
1277 | Constant *C; | |||
1278 | if (isa<Instruction>(N) && match(N, m_OneUse(m_ZExt(m_Value(X)))) && | |||
1279 | match(D, m_Constant(C))) { | |||
1280 | // If the constant is the same in the smaller type, use the narrow version. | |||
1281 | Constant *TruncC = ConstantExpr::getTrunc(C, X->getType()); | |||
1282 | if (ConstantExpr::getZExt(TruncC, Ty) != C) | |||
1283 | return nullptr; | |||
1284 | ||||
1285 | // udiv (zext X), C --> zext (udiv X, C') | |||
1286 | // urem (zext X), C --> zext (urem X, C') | |||
1287 | return new ZExtInst(Builder.CreateBinOp(Opcode, X, TruncC), Ty); | |||
1288 | } | |||
1289 | if (isa<Instruction>(D) && match(D, m_OneUse(m_ZExt(m_Value(X)))) && | |||
1290 | match(N, m_Constant(C))) { | |||
1291 | // If the constant is the same in the smaller type, use the narrow version. | |||
1292 | Constant *TruncC = ConstantExpr::getTrunc(C, X->getType()); | |||
1293 | if (ConstantExpr::getZExt(TruncC, Ty) != C) | |||
1294 | return nullptr; | |||
1295 | ||||
1296 | // udiv C, (zext X) --> zext (udiv C', X) | |||
1297 | // urem C, (zext X) --> zext (urem C', X) | |||
1298 | return new ZExtInst(Builder.CreateBinOp(Opcode, TruncC, X), Ty); | |||
1299 | } | |||
1300 | ||||
1301 | return nullptr; | |||
1302 | } | |||
1303 | ||||
1304 | Instruction *InstCombinerImpl::visitUDiv(BinaryOperator &I) { | |||
1305 | if (Value *V = simplifyUDivInst(I.getOperand(0), I.getOperand(1), I.isExact(), | |||
1306 | SQ.getWithInstruction(&I))) | |||
1307 | return replaceInstUsesWith(I, V); | |||
1308 | ||||
1309 | if (Instruction *X = foldVectorBinop(I)) | |||
1310 | return X; | |||
1311 | ||||
1312 | // Handle the integer div common cases | |||
1313 | if (Instruction *Common = commonIDivTransforms(I)) | |||
1314 | return Common; | |||
1315 | ||||
1316 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
1317 | Value *X; | |||
1318 | const APInt *C1, *C2; | |||
1319 | if (match(Op0, m_LShr(m_Value(X), m_APInt(C1))) && match(Op1, m_APInt(C2))) { | |||
1320 | // (X lshr C1) udiv C2 --> X udiv (C2 << C1) | |||
1321 | bool Overflow; | |||
1322 | APInt C2ShlC1 = C2->ushl_ov(*C1, Overflow); | |||
1323 | if (!Overflow) { | |||
1324 | bool IsExact = I.isExact() && match(Op0, m_Exact(m_Value())); | |||
1325 | BinaryOperator *BO = BinaryOperator::CreateUDiv( | |||
1326 | X, ConstantInt::get(X->getType(), C2ShlC1)); | |||
1327 | if (IsExact) | |||
1328 | BO->setIsExact(); | |||
1329 | return BO; | |||
1330 | } | |||
1331 | } | |||
1332 | ||||
1333 | // Op0 / C where C is large (negative) --> zext (Op0 >= C) | |||
1334 | // TODO: Could use isKnownNegative() to handle non-constant values. | |||
1335 | Type *Ty = I.getType(); | |||
1336 | if (match(Op1, m_Negative())) { | |||
1337 | Value *Cmp = Builder.CreateICmpUGE(Op0, Op1); | |||
1338 | return CastInst::CreateZExtOrBitCast(Cmp, Ty); | |||
1339 | } | |||
1340 | // Op0 / (sext i1 X) --> zext (Op0 == -1) (if X is 0, the div is undefined) | |||
1341 | if (match(Op1, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) { | |||
1342 | Value *Cmp = Builder.CreateICmpEQ(Op0, ConstantInt::getAllOnesValue(Ty)); | |||
1343 | return CastInst::CreateZExtOrBitCast(Cmp, Ty); | |||
1344 | } | |||
1345 | ||||
1346 | if (Instruction *NarrowDiv = narrowUDivURem(I, Builder)) | |||
1347 | return NarrowDiv; | |||
1348 | ||||
1349 | // If the udiv operands are non-overflowing multiplies with a common operand, | |||
1350 | // then eliminate the common factor: | |||
1351 | // (A * B) / (A * X) --> B / X (and commuted variants) | |||
1352 | // TODO: The code would be reduced if we had m_c_NUWMul pattern matching. | |||
1353 | // TODO: If -reassociation handled this generally, we could remove this. | |||
1354 | Value *A, *B; | |||
1355 | if (match(Op0, m_NUWMul(m_Value(A), m_Value(B)))) { | |||
1356 | if (match(Op1, m_NUWMul(m_Specific(A), m_Value(X))) || | |||
1357 | match(Op1, m_NUWMul(m_Value(X), m_Specific(A)))) | |||
1358 | return BinaryOperator::CreateUDiv(B, X); | |||
1359 | if (match(Op1, m_NUWMul(m_Specific(B), m_Value(X))) || | |||
1360 | match(Op1, m_NUWMul(m_Value(X), m_Specific(B)))) | |||
1361 | return BinaryOperator::CreateUDiv(A, X); | |||
1362 | } | |||
1363 | ||||
1364 | // Look through a right-shift to find the common factor: | |||
1365 | // ((Op1 *nuw A) >> B) / Op1 --> A >> B | |||
1366 | if (match(Op0, m_LShr(m_NUWMul(m_Specific(Op1), m_Value(A)), m_Value(B))) || | |||
1367 | match(Op0, m_LShr(m_NUWMul(m_Value(A), m_Specific(Op1)), m_Value(B)))) { | |||
1368 | Instruction *Lshr = BinaryOperator::CreateLShr(A, B); | |||
1369 | if (I.isExact() && cast<PossiblyExactOperator>(Op0)->isExact()) | |||
1370 | Lshr->setIsExact(); | |||
1371 | return Lshr; | |||
1372 | } | |||
1373 | ||||
1374 | // Op1 udiv Op2 -> Op1 lshr log2(Op2), if log2() folds away. | |||
1375 | if (takeLog2(Builder, Op1, /*Depth*/ 0, /*AssumeNonZero*/ true, | |||
1376 | /*DoFold*/ false)) { | |||
1377 | Value *Res = takeLog2(Builder, Op1, /*Depth*/ 0, | |||
1378 | /*AssumeNonZero*/ true, /*DoFold*/ true); | |||
1379 | return replaceInstUsesWith( | |||
1380 | I, Builder.CreateLShr(Op0, Res, I.getName(), I.isExact())); | |||
1381 | } | |||
1382 | ||||
1383 | return nullptr; | |||
1384 | } | |||
1385 | ||||
1386 | Instruction *InstCombinerImpl::visitSDiv(BinaryOperator &I) { | |||
1387 | if (Value *V = simplifySDivInst(I.getOperand(0), I.getOperand(1), I.isExact(), | |||
1388 | SQ.getWithInstruction(&I))) | |||
1389 | return replaceInstUsesWith(I, V); | |||
1390 | ||||
1391 | if (Instruction *X = foldVectorBinop(I)) | |||
1392 | return X; | |||
1393 | ||||
1394 | // Handle the integer div common cases | |||
1395 | if (Instruction *Common = commonIDivTransforms(I)) | |||
1396 | return Common; | |||
1397 | ||||
1398 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
1399 | Type *Ty = I.getType(); | |||
1400 | Value *X; | |||
1401 | // sdiv Op0, -1 --> -Op0 | |||
1402 | // sdiv Op0, (sext i1 X) --> -Op0 (because if X is 0, the op is undefined) | |||
1403 | if (match(Op1, m_AllOnes()) || | |||
1404 | (match(Op1, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1))) | |||
1405 | return BinaryOperator::CreateNeg(Op0); | |||
1406 | ||||
1407 | // X / INT_MIN --> X == INT_MIN | |||
1408 | if (match(Op1, m_SignMask())) | |||
1409 | return new ZExtInst(Builder.CreateICmpEQ(Op0, Op1), Ty); | |||
1410 | ||||
1411 | if (I.isExact()) { | |||
1412 | // sdiv exact X, 1<<C --> ashr exact X, C iff 1<<C is non-negative | |||
1413 | if (match(Op1, m_Power2()) && match(Op1, m_NonNegative())) { | |||
1414 | Constant *C = ConstantExpr::getExactLogBase2(cast<Constant>(Op1)); | |||
1415 | return BinaryOperator::CreateExactAShr(Op0, C); | |||
1416 | } | |||
1417 | ||||
1418 | // sdiv exact X, (1<<ShAmt) --> ashr exact X, ShAmt (if shl is non-negative) | |||
1419 | Value *ShAmt; | |||
1420 | if (match(Op1, m_NSWShl(m_One(), m_Value(ShAmt)))) | |||
1421 | return BinaryOperator::CreateExactAShr(Op0, ShAmt); | |||
1422 | ||||
1423 | // sdiv exact X, -1<<C --> -(ashr exact X, C) | |||
1424 | if (match(Op1, m_NegatedPower2())) { | |||
1425 | Constant *NegPow2C = ConstantExpr::getNeg(cast<Constant>(Op1)); | |||
1426 | Constant *C = ConstantExpr::getExactLogBase2(NegPow2C); | |||
1427 | Value *Ashr = Builder.CreateAShr(Op0, C, I.getName() + ".neg", true); | |||
1428 | return BinaryOperator::CreateNeg(Ashr); | |||
1429 | } | |||
1430 | } | |||
1431 | ||||
1432 | const APInt *Op1C; | |||
1433 | if (match(Op1, m_APInt(Op1C))) { | |||
1434 | // If the dividend is sign-extended and the constant divisor is small enough | |||
1435 | // to fit in the source type, shrink the division to the narrower type: | |||
1436 | // (sext X) sdiv C --> sext (X sdiv C) | |||
1437 | Value *Op0Src; | |||
1438 | if (match(Op0, m_OneUse(m_SExt(m_Value(Op0Src)))) && | |||
1439 | Op0Src->getType()->getScalarSizeInBits() >= | |||
1440 | Op1C->getSignificantBits()) { | |||
1441 | ||||
1442 | // In the general case, we need to make sure that the dividend is not the | |||
1443 | // minimum signed value because dividing that by -1 is UB. But here, we | |||
1444 | // know that the -1 divisor case is already handled above. | |||
1445 | ||||
1446 | Constant *NarrowDivisor = | |||
1447 | ConstantExpr::getTrunc(cast<Constant>(Op1), Op0Src->getType()); | |||
1448 | Value *NarrowOp = Builder.CreateSDiv(Op0Src, NarrowDivisor); | |||
1449 | return new SExtInst(NarrowOp, Ty); | |||
1450 | } | |||
1451 | ||||
1452 | // -X / C --> X / -C (if the negation doesn't overflow). | |||
1453 | // TODO: This could be enhanced to handle arbitrary vector constants by | |||
1454 | // checking if all elements are not the min-signed-val. | |||
1455 | if (!Op1C->isMinSignedValue() && | |||
1456 | match(Op0, m_NSWSub(m_Zero(), m_Value(X)))) { | |||
1457 | Constant *NegC = ConstantInt::get(Ty, -(*Op1C)); | |||
1458 | Instruction *BO = BinaryOperator::CreateSDiv(X, NegC); | |||
1459 | BO->setIsExact(I.isExact()); | |||
1460 | return BO; | |||
1461 | } | |||
1462 | } | |||
1463 | ||||
1464 | // -X / Y --> -(X / Y) | |||
1465 | Value *Y; | |||
1466 | if (match(&I, m_SDiv(m_OneUse(m_NSWSub(m_Zero(), m_Value(X))), m_Value(Y)))) | |||
1467 | return BinaryOperator::CreateNSWNeg( | |||
1468 | Builder.CreateSDiv(X, Y, I.getName(), I.isExact())); | |||
1469 | ||||
1470 | // abs(X) / X --> X > -1 ? 1 : -1 | |||
1471 | // X / abs(X) --> X > -1 ? 1 : -1 | |||
1472 | if (match(&I, m_c_BinOp( | |||
1473 | m_OneUse(m_Intrinsic<Intrinsic::abs>(m_Value(X), m_One())), | |||
1474 | m_Deferred(X)))) { | |||
1475 | Value *Cond = Builder.CreateIsNotNeg(X); | |||
1476 | return SelectInst::Create(Cond, ConstantInt::get(Ty, 1), | |||
1477 | ConstantInt::getAllOnesValue(Ty)); | |||
1478 | } | |||
1479 | ||||
1480 | KnownBits KnownDividend = computeKnownBits(Op0, 0, &I); | |||
1481 | if (!I.isExact() && | |||
1482 | (match(Op1, m_Power2(Op1C)) || match(Op1, m_NegatedPower2(Op1C))) && | |||
1483 | KnownDividend.countMinTrailingZeros() >= Op1C->countr_zero()) { | |||
1484 | I.setIsExact(); | |||
1485 | return &I; | |||
1486 | } | |||
1487 | ||||
1488 | if (KnownDividend.isNonNegative()) { | |||
1489 | // If both operands are unsigned, turn this into a udiv. | |||
1490 | if (isKnownNonNegative(Op1, DL, 0, &AC, &I, &DT)) { | |||
1491 | auto *BO = BinaryOperator::CreateUDiv(Op0, Op1, I.getName()); | |||
1492 | BO->setIsExact(I.isExact()); | |||
1493 | return BO; | |||
1494 | } | |||
1495 | ||||
1496 | if (match(Op1, m_NegatedPower2())) { | |||
1497 | // X sdiv (-(1 << C)) -> -(X sdiv (1 << C)) -> | |||
1498 | // -> -(X udiv (1 << C)) -> -(X u>> C) | |||
1499 | Constant *CNegLog2 = ConstantExpr::getExactLogBase2( | |||
1500 | ConstantExpr::getNeg(cast<Constant>(Op1))); | |||
1501 | Value *Shr = Builder.CreateLShr(Op0, CNegLog2, I.getName(), I.isExact()); | |||
1502 | return BinaryOperator::CreateNeg(Shr); | |||
1503 | } | |||
1504 | ||||
1505 | if (isKnownToBeAPowerOfTwo(Op1, /*OrZero*/ true, 0, &I)) { | |||
1506 | // X sdiv (1 << Y) -> X udiv (1 << Y) ( -> X u>> Y) | |||
1507 | // Safe because the only negative value (1 << Y) can take on is | |||
1508 | // INT_MIN, and X sdiv INT_MIN == X udiv INT_MIN == 0 if X doesn't have | |||
1509 | // the sign bit set. | |||
1510 | auto *BO = BinaryOperator::CreateUDiv(Op0, Op1, I.getName()); | |||
1511 | BO->setIsExact(I.isExact()); | |||
1512 | return BO; | |||
1513 | } | |||
1514 | } | |||
1515 | ||||
1516 | return nullptr; | |||
1517 | } | |||
1518 | ||||
1519 | /// Remove negation and try to convert division into multiplication. | |||
1520 | Instruction *InstCombinerImpl::foldFDivConstantDivisor(BinaryOperator &I) { | |||
1521 | Constant *C; | |||
1522 | if (!match(I.getOperand(1), m_Constant(C))) | |||
1523 | return nullptr; | |||
1524 | ||||
1525 | // -X / C --> X / -C | |||
1526 | Value *X; | |||
1527 | const DataLayout &DL = I.getModule()->getDataLayout(); | |||
1528 | if (match(I.getOperand(0), m_FNeg(m_Value(X)))) | |||
1529 | if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL)) | |||
1530 | return BinaryOperator::CreateFDivFMF(X, NegC, &I); | |||
1531 | ||||
1532 | // nnan X / +0.0 -> copysign(inf, X) | |||
1533 | if (I.hasNoNaNs() && match(I.getOperand(1), m_Zero())) { | |||
1534 | IRBuilder<> B(&I); | |||
1535 | // TODO: nnan nsz X / -0.0 -> copysign(inf, X) | |||
1536 | CallInst *CopySign = B.CreateIntrinsic( | |||
1537 | Intrinsic::copysign, {C->getType()}, | |||
1538 | {ConstantFP::getInfinity(I.getType()), I.getOperand(0)}, &I); | |||
1539 | CopySign->takeName(&I); | |||
1540 | return replaceInstUsesWith(I, CopySign); | |||
1541 | } | |||
1542 | ||||
1543 | // If the constant divisor has an exact inverse, this is always safe. If not, | |||
1544 | // then we can still create a reciprocal if fast-math-flags allow it and the | |||
1545 | // constant is a regular number (not zero, infinite, or denormal). | |||
1546 | if (!(C->hasExactInverseFP() || (I.hasAllowReciprocal() && C->isNormalFP()))) | |||
1547 | return nullptr; | |||
1548 | ||||
1549 | // Disallow denormal constants because we don't know what would happen | |||
1550 | // on all targets. | |||
1551 | // TODO: Use Intrinsic::canonicalize or let function attributes tell us that | |||
1552 | // denorms are flushed? | |||
1553 | auto *RecipC = ConstantFoldBinaryOpOperands( | |||
1554 | Instruction::FDiv, ConstantFP::get(I.getType(), 1.0), C, DL); | |||
1555 | if (!RecipC || !RecipC->isNormalFP()) | |||
1556 | return nullptr; | |||
1557 | ||||
1558 | // X / C --> X * (1 / C) | |||
1559 | return BinaryOperator::CreateFMulFMF(I.getOperand(0), RecipC, &I); | |||
1560 | } | |||
1561 | ||||
1562 | /// Remove negation and try to reassociate constant math. | |||
1563 | static Instruction *foldFDivConstantDividend(BinaryOperator &I) { | |||
1564 | Constant *C; | |||
1565 | if (!match(I.getOperand(0), m_Constant(C))) | |||
1566 | return nullptr; | |||
1567 | ||||
1568 | // C / -X --> -C / X | |||
1569 | Value *X; | |||
1570 | const DataLayout &DL = I.getModule()->getDataLayout(); | |||
1571 | if (match(I.getOperand(1), m_FNeg(m_Value(X)))) | |||
1572 | if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL)) | |||
1573 | return BinaryOperator::CreateFDivFMF(NegC, X, &I); | |||
1574 | ||||
1575 | if (!I.hasAllowReassoc() || !I.hasAllowReciprocal()) | |||
1576 | return nullptr; | |||
1577 | ||||
1578 | // Try to reassociate C / X expressions where X includes another constant. | |||
1579 | Constant *C2, *NewC = nullptr; | |||
1580 | if (match(I.getOperand(1), m_FMul(m_Value(X), m_Constant(C2)))) { | |||
1581 | // C / (X * C2) --> (C / C2) / X | |||
1582 | NewC = ConstantFoldBinaryOpOperands(Instruction::FDiv, C, C2, DL); | |||
1583 | } else if (match(I.getOperand(1), m_FDiv(m_Value(X), m_Constant(C2)))) { | |||
1584 | // C / (X / C2) --> (C * C2) / X | |||
1585 | NewC = ConstantFoldBinaryOpOperands(Instruction::FMul, C, C2, DL); | |||
1586 | } | |||
1587 | // Disallow denormal constants because we don't know what would happen | |||
1588 | // on all targets. | |||
1589 | // TODO: Use Intrinsic::canonicalize or let function attributes tell us that | |||
1590 | // denorms are flushed? | |||
1591 | if (!NewC || !NewC->isNormalFP()) | |||
1592 | return nullptr; | |||
1593 | ||||
1594 | return BinaryOperator::CreateFDivFMF(NewC, X, &I); | |||
1595 | } | |||
1596 | ||||
1597 | /// Negate the exponent of pow/exp to fold division-by-pow() into multiply. | |||
1598 | static Instruction *foldFDivPowDivisor(BinaryOperator &I, | |||
1599 | InstCombiner::BuilderTy &Builder) { | |||
1600 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
1601 | auto *II = dyn_cast<IntrinsicInst>(Op1); | |||
1602 | if (!II || !II->hasOneUse() || !I.hasAllowReassoc() || | |||
1603 | !I.hasAllowReciprocal()) | |||
1604 | return nullptr; | |||
1605 | ||||
1606 | // Z / pow(X, Y) --> Z * pow(X, -Y) | |||
1607 | // Z / exp{2}(Y) --> Z * exp{2}(-Y) | |||
1608 | // In the general case, this creates an extra instruction, but fmul allows | |||
1609 | // for better canonicalization and optimization than fdiv. | |||
1610 | Intrinsic::ID IID = II->getIntrinsicID(); | |||
1611 | SmallVector<Value *> Args; | |||
1612 | switch (IID) { | |||
1613 | case Intrinsic::pow: | |||
1614 | Args.push_back(II->getArgOperand(0)); | |||
1615 | Args.push_back(Builder.CreateFNegFMF(II->getArgOperand(1), &I)); | |||
1616 | break; | |||
1617 | case Intrinsic::powi: { | |||
1618 | // Require 'ninf' assuming that makes powi(X, -INT_MIN) acceptable. | |||
1619 | // That is, X ** (huge negative number) is 0.0, ~1.0, or INF and so | |||
1620 | // dividing by that is INF, ~1.0, or 0.0. Code that uses powi allows | |||
1621 | // non-standard results, so this corner case should be acceptable if the | |||
1622 | // code rules out INF values. | |||
1623 | if (!I.hasNoInfs()) | |||
1624 | return nullptr; | |||
1625 | Args.push_back(II->getArgOperand(0)); | |||
1626 | Args.push_back(Builder.CreateNeg(II->getArgOperand(1))); | |||
1627 | Type *Tys[] = {I.getType(), II->getArgOperand(1)->getType()}; | |||
1628 | Value *Pow = Builder.CreateIntrinsic(IID, Tys, Args, &I); | |||
1629 | return BinaryOperator::CreateFMulFMF(Op0, Pow, &I); | |||
1630 | } | |||
1631 | case Intrinsic::exp: | |||
1632 | case Intrinsic::exp2: | |||
1633 | Args.push_back(Builder.CreateFNegFMF(II->getArgOperand(0), &I)); | |||
1634 | break; | |||
1635 | default: | |||
1636 | return nullptr; | |||
1637 | } | |||
1638 | Value *Pow = Builder.CreateIntrinsic(IID, I.getType(), Args, &I); | |||
1639 | return BinaryOperator::CreateFMulFMF(Op0, Pow, &I); | |||
1640 | } | |||
1641 | ||||
1642 | Instruction *InstCombinerImpl::visitFDiv(BinaryOperator &I) { | |||
1643 | Module *M = I.getModule(); | |||
1644 | ||||
1645 | if (Value *V = simplifyFDivInst(I.getOperand(0), I.getOperand(1), | |||
1646 | I.getFastMathFlags(), | |||
1647 | SQ.getWithInstruction(&I))) | |||
1648 | return replaceInstUsesWith(I, V); | |||
1649 | ||||
1650 | if (Instruction *X = foldVectorBinop(I)) | |||
1651 | return X; | |||
1652 | ||||
1653 | if (Instruction *Phi = foldBinopWithPhiOperands(I)) | |||
1654 | return Phi; | |||
1655 | ||||
1656 | if (Instruction *R = foldFDivConstantDivisor(I)) | |||
1657 | return R; | |||
1658 | ||||
1659 | if (Instruction *R = foldFDivConstantDividend(I)) | |||
1660 | return R; | |||
1661 | ||||
1662 | if (Instruction *R = foldFPSignBitOps(I)) | |||
1663 | return R; | |||
1664 | ||||
1665 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
1666 | if (isa<Constant>(Op0)) | |||
1667 | if (SelectInst *SI = dyn_cast<SelectInst>(Op1)) | |||
1668 | if (Instruction *R = FoldOpIntoSelect(I, SI)) | |||
1669 | return R; | |||
1670 | ||||
1671 | if (isa<Constant>(Op1)) | |||
1672 | if (SelectInst *SI = dyn_cast<SelectInst>(Op0)) | |||
1673 | if (Instruction *R = FoldOpIntoSelect(I, SI)) | |||
1674 | return R; | |||
1675 | ||||
1676 | if (I.hasAllowReassoc() && I.hasAllowReciprocal()) { | |||
1677 | Value *X, *Y; | |||
1678 | if (match(Op0, m_OneUse(m_FDiv(m_Value(X), m_Value(Y)))) && | |||
1679 | (!isa<Constant>(Y) || !isa<Constant>(Op1))) { | |||
1680 | // (X / Y) / Z => X / (Y * Z) | |||
1681 | Value *YZ = Builder.CreateFMulFMF(Y, Op1, &I); | |||
1682 | return BinaryOperator::CreateFDivFMF(X, YZ, &I); | |||
1683 | } | |||
1684 | if (match(Op1, m_OneUse(m_FDiv(m_Value(X), m_Value(Y)))) && | |||
1685 | (!isa<Constant>(Y) || !isa<Constant>(Op0))) { | |||
1686 | // Z / (X / Y) => (Y * Z) / X | |||
1687 | Value *YZ = Builder.CreateFMulFMF(Y, Op0, &I); | |||
1688 | return BinaryOperator::CreateFDivFMF(YZ, X, &I); | |||
1689 | } | |||
1690 | // Z / (1.0 / Y) => (Y * Z) | |||
1691 | // | |||
1692 | // This is a special case of Z / (X / Y) => (Y * Z) / X, with X = 1.0. The | |||
1693 | // m_OneUse check is avoided because even in the case of the multiple uses | |||
1694 | // for 1.0/Y, the number of instructions remain the same and a division is | |||
1695 | // replaced by a multiplication. | |||
1696 | if (match(Op1, m_FDiv(m_SpecificFP(1.0), m_Value(Y)))) | |||
1697 | return BinaryOperator::CreateFMulFMF(Y, Op0, &I); | |||
1698 | } | |||
1699 | ||||
1700 | if (I.hasAllowReassoc() && Op0->hasOneUse() && Op1->hasOneUse()) { | |||
1701 | // sin(X) / cos(X) -> tan(X) | |||
1702 | // cos(X) / sin(X) -> 1/tan(X) (cotangent) | |||
1703 | Value *X; | |||
1704 | bool IsTan = match(Op0, m_Intrinsic<Intrinsic::sin>(m_Value(X))) && | |||
1705 | match(Op1, m_Intrinsic<Intrinsic::cos>(m_Specific(X))); | |||
1706 | bool IsCot = | |||
1707 | !IsTan && match(Op0, m_Intrinsic<Intrinsic::cos>(m_Value(X))) && | |||
1708 | match(Op1, m_Intrinsic<Intrinsic::sin>(m_Specific(X))); | |||
1709 | ||||
1710 | if ((IsTan || IsCot) && hasFloatFn(M, &TLI, I.getType(), LibFunc_tan, | |||
1711 | LibFunc_tanf, LibFunc_tanl)) { | |||
1712 | IRBuilder<> B(&I); | |||
1713 | IRBuilder<>::FastMathFlagGuard FMFGuard(B); | |||
1714 | B.setFastMathFlags(I.getFastMathFlags()); | |||
1715 | AttributeList Attrs = | |||
1716 | cast<CallBase>(Op0)->getCalledFunction()->getAttributes(); | |||
1717 | Value *Res = emitUnaryFloatFnCall(X, &TLI, LibFunc_tan, LibFunc_tanf, | |||
1718 | LibFunc_tanl, B, Attrs); | |||
1719 | if (IsCot) | |||
1720 | Res = B.CreateFDiv(ConstantFP::get(I.getType(), 1.0), Res); | |||
1721 | return replaceInstUsesWith(I, Res); | |||
1722 | } | |||
1723 | } | |||
1724 | ||||
1725 | // X / (X * Y) --> 1.0 / Y | |||
1726 | // Reassociate to (X / X -> 1.0) is legal when NaNs are not allowed. | |||
1727 | // We can ignore the possibility that X is infinity because INF/INF is NaN. | |||
1728 | Value *X, *Y; | |||
1729 | if (I.hasNoNaNs() && I.hasAllowReassoc() && | |||
1730 | match(Op1, m_c_FMul(m_Specific(Op0), m_Value(Y)))) { | |||
1731 | replaceOperand(I, 0, ConstantFP::get(I.getType(), 1.0)); | |||
1732 | replaceOperand(I, 1, Y); | |||
1733 | return &I; | |||
1734 | } | |||
1735 | ||||
1736 | // X / fabs(X) -> copysign(1.0, X) | |||
1737 | // fabs(X) / X -> copysign(1.0, X) | |||
1738 | if (I.hasNoNaNs() && I.hasNoInfs() && | |||
1739 | (match(&I, m_FDiv(m_Value(X), m_FAbs(m_Deferred(X)))) || | |||
1740 | match(&I, m_FDiv(m_FAbs(m_Value(X)), m_Deferred(X))))) { | |||
1741 | Value *V = Builder.CreateBinaryIntrinsic( | |||
1742 | Intrinsic::copysign, ConstantFP::get(I.getType(), 1.0), X, &I); | |||
1743 | return replaceInstUsesWith(I, V); | |||
1744 | } | |||
1745 | ||||
1746 | if (Instruction *Mul = foldFDivPowDivisor(I, Builder)) | |||
1747 | return Mul; | |||
1748 | ||||
1749 | // pow(X, Y) / X --> pow(X, Y-1) | |||
1750 | if (I.hasAllowReassoc() && | |||
1751 | match(Op0, m_OneUse(m_Intrinsic<Intrinsic::pow>(m_Specific(Op1), | |||
1752 | m_Value(Y))))) { | |||
1753 | Value *Y1 = | |||
1754 | Builder.CreateFAddFMF(Y, ConstantFP::get(I.getType(), -1.0), &I); | |||
1755 | Value *Pow = Builder.CreateBinaryIntrinsic(Intrinsic::pow, Op1, Y1, &I); | |||
1756 | return replaceInstUsesWith(I, Pow); | |||
1757 | } | |||
1758 | ||||
1759 | return nullptr; | |||
1760 | } | |||
1761 | ||||
1762 | // Variety of transform for (urem/srem (mul/shl X, Y), (mul/shl X, Z)) | |||
1763 | static Instruction *simplifyIRemMulShl(BinaryOperator &I, | |||
1764 | InstCombinerImpl &IC) { | |||
1765 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1), *X; | |||
1766 | const APInt *Y, *Z; | |||
1767 | if (!(match(Op0, m_Mul(m_Value(X), m_APInt(Y))) && | |||
1768 | match(Op1, m_c_Mul(m_Specific(X), m_APInt(Z)))) && | |||
1769 | !(match(Op0, m_Mul(m_APInt(Y), m_Value(X))) && | |||
1770 | match(Op1, m_c_Mul(m_Specific(X), m_APInt(Z))))) | |||
1771 | return nullptr; | |||
1772 | ||||
1773 | bool IsSRem = I.getOpcode() == Instruction::SRem; | |||
1774 | ||||
1775 | OverflowingBinaryOperator *BO0 = cast<OverflowingBinaryOperator>(Op0); | |||
1776 | // TODO: We may be able to deduce more about nsw/nuw of BO0/BO1 based on Y >= | |||
1777 | // Z or Z >= Y. | |||
1778 | bool BO0HasNSW = BO0->hasNoSignedWrap(); | |||
1779 | bool BO0HasNUW = BO0->hasNoUnsignedWrap(); | |||
1780 | bool BO0NoWrap = IsSRem ? BO0HasNSW : BO0HasNUW; | |||
1781 | ||||
1782 | APInt RemYZ = IsSRem ? Y->srem(*Z) : Y->urem(*Z); | |||
1783 | // (rem (mul nuw/nsw X, Y), (mul X, Z)) | |||
1784 | // if (rem Y, Z) == 0 | |||
1785 | // -> 0 | |||
1786 | if (RemYZ.isZero() && BO0NoWrap) | |||
1787 | return IC.replaceInstUsesWith(I, ConstantInt::getNullValue(I.getType())); | |||
1788 | ||||
1789 | OverflowingBinaryOperator *BO1 = cast<OverflowingBinaryOperator>(Op1); | |||
1790 | bool BO1HasNSW = BO1->hasNoSignedWrap(); | |||
1791 | bool BO1HasNUW = BO1->hasNoUnsignedWrap(); | |||
1792 | bool BO1NoWrap = IsSRem ? BO1HasNSW : BO1HasNUW; | |||
1793 | // (rem (mul X, Y), (mul nuw/nsw X, Z)) | |||
1794 | // if (rem Y, Z) == Y | |||
1795 | // -> (mul nuw/nsw X, Y) | |||
1796 | if (RemYZ == *Y && BO1NoWrap) { | |||
1797 | BinaryOperator *BO = | |||
1798 | BinaryOperator::CreateMul(X, ConstantInt::get(I.getType(), *Y)); | |||
1799 | // Copy any overflow flags from Op0. | |||
1800 | BO->setHasNoSignedWrap(IsSRem || BO0HasNSW); | |||
1801 | BO->setHasNoUnsignedWrap(!IsSRem || BO0HasNUW); | |||
1802 | return BO; | |||
1803 | } | |||
1804 | ||||
1805 | // (rem (mul nuw/nsw X, Y), (mul {nsw} X, Z)) | |||
1806 | // if Y >= Z | |||
1807 | // -> (mul {nuw} nsw X, (rem Y, Z)) | |||
1808 | if (Y->uge(*Z) && (IsSRem ? (BO0HasNSW && BO1HasNSW) : BO0HasNUW)) { | |||
1809 | BinaryOperator *BO = | |||
1810 | BinaryOperator::CreateMul(X, ConstantInt::get(I.getType(), RemYZ)); | |||
1811 | BO->setHasNoSignedWrap(); | |||
1812 | BO->setHasNoUnsignedWrap(BO0HasNUW); | |||
1813 | return BO; | |||
1814 | } | |||
1815 | ||||
1816 | return nullptr; | |||
1817 | } | |||
1818 | ||||
1819 | /// This function implements the transforms common to both integer remainder | |||
1820 | /// instructions (urem and srem). It is called by the visitors to those integer | |||
1821 | /// remainder instructions. | |||
1822 | /// Common integer remainder transforms | |||
1823 | Instruction *InstCombinerImpl::commonIRemTransforms(BinaryOperator &I) { | |||
1824 | if (Instruction *Phi = foldBinopWithPhiOperands(I)) | |||
1825 | return Phi; | |||
1826 | ||||
1827 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
1828 | ||||
1829 | // The RHS is known non-zero. | |||
1830 | if (Value *V = simplifyValueKnownNonZero(I.getOperand(1), *this, I)) | |||
1831 | return replaceOperand(I, 1, V); | |||
1832 | ||||
1833 | // Handle cases involving: rem X, (select Cond, Y, Z) | |||
1834 | if (simplifyDivRemOfSelectWithZeroOp(I)) | |||
1835 | return &I; | |||
1836 | ||||
1837 | // If the divisor is a select-of-constants, try to constant fold all rem ops: | |||
1838 | // C % (select Cond, TrueC, FalseC) --> select Cond, (C % TrueC), (C % FalseC) | |||
1839 | // TODO: Adapt simplifyDivRemOfSelectWithZeroOp to allow this and other folds. | |||
1840 | if (match(Op0, m_ImmConstant()) && | |||
1841 | match(Op1, m_Select(m_Value(), m_ImmConstant(), m_ImmConstant()))) { | |||
1842 | if (Instruction *R = FoldOpIntoSelect(I, cast<SelectInst>(Op1), | |||
1843 | /*FoldWithMultiUse*/ true)) | |||
1844 | return R; | |||
1845 | } | |||
1846 | ||||
1847 | if (isa<Constant>(Op1)) { | |||
1848 | if (Instruction *Op0I = dyn_cast<Instruction>(Op0)) { | |||
1849 | if (SelectInst *SI = dyn_cast<SelectInst>(Op0I)) { | |||
1850 | if (Instruction *R = FoldOpIntoSelect(I, SI)) | |||
1851 | return R; | |||
1852 | } else if (auto *PN = dyn_cast<PHINode>(Op0I)) { | |||
1853 | const APInt *Op1Int; | |||
1854 | if (match(Op1, m_APInt(Op1Int)) && !Op1Int->isMinValue() && | |||
1855 | (I.getOpcode() == Instruction::URem || | |||
1856 | !Op1Int->isMinSignedValue())) { | |||
1857 | // foldOpIntoPhi will speculate instructions to the end of the PHI's | |||
1858 | // predecessor blocks, so do this only if we know the srem or urem | |||
1859 | // will not fault. | |||
1860 | if (Instruction *NV = foldOpIntoPhi(I, PN)) | |||
1861 | return NV; | |||
1862 | } | |||
1863 | } | |||
1864 | ||||
1865 | // See if we can fold away this rem instruction. | |||
1866 | if (SimplifyDemandedInstructionBits(I)) | |||
1867 | return &I; | |||
1868 | } | |||
1869 | } | |||
1870 | ||||
1871 | if (Instruction *R = simplifyIRemMulShl(I, *this)) | |||
1872 | return R; | |||
1873 | ||||
1874 | return nullptr; | |||
1875 | } | |||
1876 | ||||
1877 | Instruction *InstCombinerImpl::visitURem(BinaryOperator &I) { | |||
1878 | if (Value *V = simplifyURemInst(I.getOperand(0), I.getOperand(1), | |||
1879 | SQ.getWithInstruction(&I))) | |||
1880 | return replaceInstUsesWith(I, V); | |||
1881 | ||||
1882 | if (Instruction *X = foldVectorBinop(I)) | |||
1883 | return X; | |||
1884 | ||||
1885 | if (Instruction *common = commonIRemTransforms(I)) | |||
1886 | return common; | |||
1887 | ||||
1888 | if (Instruction *NarrowRem = narrowUDivURem(I, Builder)) | |||
1889 | return NarrowRem; | |||
1890 | ||||
1891 | // X urem Y -> X and Y-1, where Y is a power of 2, | |||
1892 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
1893 | Type *Ty = I.getType(); | |||
1894 | if (isKnownToBeAPowerOfTwo(Op1, /*OrZero*/ true, 0, &I)) { | |||
1895 | // This may increase instruction count, we don't enforce that Y is a | |||
1896 | // constant. | |||
1897 | Constant *N1 = Constant::getAllOnesValue(Ty); | |||
1898 | Value *Add = Builder.CreateAdd(Op1, N1); | |||
1899 | return BinaryOperator::CreateAnd(Op0, Add); | |||
1900 | } | |||
1901 | ||||
1902 | // 1 urem X -> zext(X != 1) | |||
1903 | if (match(Op0, m_One())) { | |||
1904 | Value *Cmp = Builder.CreateICmpNE(Op1, ConstantInt::get(Ty, 1)); | |||
1905 | return CastInst::CreateZExtOrBitCast(Cmp, Ty); | |||
1906 | } | |||
1907 | ||||
1908 | // Op0 urem C -> Op0 < C ? Op0 : Op0 - C, where C >= signbit. | |||
1909 | // Op0 must be frozen because we are increasing its number of uses. | |||
1910 | if (match(Op1, m_Negative())) { | |||
1911 | Value *F0 = Builder.CreateFreeze(Op0, Op0->getName() + ".fr"); | |||
1912 | Value *Cmp = Builder.CreateICmpULT(F0, Op1); | |||
1913 | Value *Sub = Builder.CreateSub(F0, Op1); | |||
1914 | return SelectInst::Create(Cmp, F0, Sub); | |||
1915 | } | |||
1916 | ||||
1917 | // If the divisor is a sext of a boolean, then the divisor must be max | |||
1918 | // unsigned value (-1). Therefore, the remainder is Op0 unless Op0 is also | |||
1919 | // max unsigned value. In that case, the remainder is 0: | |||
1920 | // urem Op0, (sext i1 X) --> (Op0 == -1) ? 0 : Op0 | |||
1921 | Value *X; | |||
1922 | if (match(Op1, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) { | |||
1923 | Value *FrozenOp0 = Builder.CreateFreeze(Op0, Op0->getName() + ".frozen"); | |||
1924 | Value *Cmp = | |||
1925 | Builder.CreateICmpEQ(FrozenOp0, ConstantInt::getAllOnesValue(Ty)); | |||
1926 | return SelectInst::Create(Cmp, ConstantInt::getNullValue(Ty), FrozenOp0); | |||
1927 | } | |||
1928 | ||||
1929 | // For "(X + 1) % Op1" and if (X u< Op1) => (X + 1) == Op1 ? 0 : X + 1 . | |||
1930 | if (match(Op0, m_Add(m_Value(X), m_One()))) { | |||
1931 | Value *Val = | |||
1932 | simplifyICmpInst(ICmpInst::ICMP_ULT, X, Op1, SQ.getWithInstruction(&I)); | |||
1933 | if (Val && match(Val, m_One())) { | |||
1934 | Value *FrozenOp0 = Builder.CreateFreeze(Op0, Op0->getName() + ".frozen"); | |||
1935 | Value *Cmp = Builder.CreateICmpEQ(FrozenOp0, Op1); | |||
1936 | return SelectInst::Create(Cmp, ConstantInt::getNullValue(Ty), FrozenOp0); | |||
1937 | } | |||
1938 | } | |||
1939 | ||||
1940 | return nullptr; | |||
1941 | } | |||
1942 | ||||
1943 | Instruction *InstCombinerImpl::visitSRem(BinaryOperator &I) { | |||
1944 | if (Value *V = simplifySRemInst(I.getOperand(0), I.getOperand(1), | |||
1945 | SQ.getWithInstruction(&I))) | |||
1946 | return replaceInstUsesWith(I, V); | |||
1947 | ||||
1948 | if (Instruction *X = foldVectorBinop(I)) | |||
1949 | return X; | |||
1950 | ||||
1951 | // Handle the integer rem common cases | |||
1952 | if (Instruction *Common = commonIRemTransforms(I)) | |||
1953 | return Common; | |||
1954 | ||||
1955 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | |||
1956 | { | |||
1957 | const APInt *Y; | |||
1958 | // X % -Y -> X % Y | |||
1959 | if (match(Op1, m_Negative(Y)) && !Y->isMinSignedValue()) | |||
1960 | return replaceOperand(I, 1, ConstantInt::get(I.getType(), -*Y)); | |||
1961 | } | |||
1962 | ||||
1963 | // -X srem Y --> -(X srem Y) | |||
1964 | Value *X, *Y; | |||
1965 | if (match(&I, m_SRem(m_OneUse(m_NSWSub(m_Zero(), m_Value(X))), m_Value(Y)))) | |||
1966 | return BinaryOperator::CreateNSWNeg(Builder.CreateSRem(X, Y)); | |||
1967 | ||||
1968 | // If the sign bits of both operands are zero (i.e. we can prove they are | |||
1969 | // unsigned inputs), turn this into a urem. | |||
1970 | APInt Mask(APInt::getSignMask(I.getType()->getScalarSizeInBits())); | |||
1971 | if (MaskedValueIsZero(Op1, Mask, 0, &I) && | |||
1972 | MaskedValueIsZero(Op0, Mask, 0, &I)) { | |||
1973 | // X srem Y -> X urem Y, iff X and Y don't have sign bit set | |||
1974 | return BinaryOperator::CreateURem(Op0, Op1, I.getName()); | |||
1975 | } | |||
1976 | ||||
1977 | // If it's a constant vector, flip any negative values positive. | |||
1978 | if (isa<ConstantVector>(Op1) || isa<ConstantDataVector>(Op1)) { | |||
1979 | Constant *C = cast<Constant>(Op1); | |||
1980 | unsigned VWidth = cast<FixedVectorType>(C->getType())->getNumElements(); | |||
1981 | ||||
1982 | bool hasNegative = false; | |||
1983 | bool hasMissing = false; | |||
1984 | for (unsigned i = 0; i != VWidth; ++i) { | |||
1985 | Constant *Elt = C->getAggregateElement(i); | |||
1986 | if (!Elt) { | |||
1987 | hasMissing = true; | |||
1988 | break; | |||
1989 | } | |||
1990 | ||||
1991 | if (ConstantInt *RHS = dyn_cast<ConstantInt>(Elt)) | |||
1992 | if (RHS->isNegative()) | |||
1993 | hasNegative = true; | |||
1994 | } | |||
1995 | ||||
1996 | if (hasNegative && !hasMissing) { | |||
1997 | SmallVector<Constant *, 16> Elts(VWidth); | |||
1998 | for (unsigned i = 0; i != VWidth; ++i) { | |||
1999 | Elts[i] = C->getAggregateElement(i); // Handle undef, etc. | |||
2000 | if (ConstantInt *RHS = dyn_cast<ConstantInt>(Elts[i])) { | |||
2001 | if (RHS->isNegative()) | |||
2002 | Elts[i] = cast<ConstantInt>(ConstantExpr::getNeg(RHS)); | |||
2003 | } | |||
2004 | } | |||
2005 | ||||
2006 | Constant *NewRHSV = ConstantVector::get(Elts); | |||
2007 | if (NewRHSV != C) // Don't loop on -MININT | |||
2008 | return replaceOperand(I, 1, NewRHSV); | |||
2009 | } | |||
2010 | } | |||
2011 | ||||
2012 | return nullptr; | |||
2013 | } | |||
2014 | ||||
2015 | Instruction *InstCombinerImpl::visitFRem(BinaryOperator &I) { | |||
2016 | if (Value *V = simplifyFRemInst(I.getOperand(0), I.getOperand(1), | |||
2017 | I.getFastMathFlags(), | |||
2018 | SQ.getWithInstruction(&I))) | |||
2019 | return replaceInstUsesWith(I, V); | |||
2020 | ||||
2021 | if (Instruction *X = foldVectorBinop(I)) | |||
2022 | return X; | |||
2023 | ||||
2024 | if (Instruction *Phi = foldBinopWithPhiOperands(I)) | |||
2025 | return Phi; | |||
2026 | ||||
2027 | return nullptr; | |||
2028 | } |