File: | build/source/llvm/include/llvm/IR/PatternMatch.h |
Warning: | line 1450, column 9 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- InstCombineAddSub.cpp ------------------------------------*- C++ -*-===// | ||||
2 | // | ||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||
6 | // | ||||
7 | //===----------------------------------------------------------------------===// | ||||
8 | // | ||||
9 | // This file implements the visit functions for add, fadd, sub, and fsub. | ||||
10 | // | ||||
11 | //===----------------------------------------------------------------------===// | ||||
12 | |||||
13 | #include "InstCombineInternal.h" | ||||
14 | #include "llvm/ADT/APFloat.h" | ||||
15 | #include "llvm/ADT/APInt.h" | ||||
16 | #include "llvm/ADT/STLExtras.h" | ||||
17 | #include "llvm/ADT/SmallVector.h" | ||||
18 | #include "llvm/Analysis/InstructionSimplify.h" | ||||
19 | #include "llvm/Analysis/ValueTracking.h" | ||||
20 | #include "llvm/IR/Constant.h" | ||||
21 | #include "llvm/IR/Constants.h" | ||||
22 | #include "llvm/IR/InstrTypes.h" | ||||
23 | #include "llvm/IR/Instruction.h" | ||||
24 | #include "llvm/IR/Instructions.h" | ||||
25 | #include "llvm/IR/Operator.h" | ||||
26 | #include "llvm/IR/PatternMatch.h" | ||||
27 | #include "llvm/IR/Type.h" | ||||
28 | #include "llvm/IR/Value.h" | ||||
29 | #include "llvm/Support/AlignOf.h" | ||||
30 | #include "llvm/Support/Casting.h" | ||||
31 | #include "llvm/Support/KnownBits.h" | ||||
32 | #include "llvm/Transforms/InstCombine/InstCombiner.h" | ||||
33 | #include <cassert> | ||||
34 | #include <utility> | ||||
35 | |||||
36 | using namespace llvm; | ||||
37 | using namespace PatternMatch; | ||||
38 | |||||
39 | #define DEBUG_TYPE"instcombine" "instcombine" | ||||
40 | |||||
41 | namespace { | ||||
42 | |||||
43 | /// Class representing coefficient of floating-point addend. | ||||
44 | /// This class needs to be highly efficient, which is especially true for | ||||
45 | /// the constructor. As of I write this comment, the cost of the default | ||||
46 | /// constructor is merely 4-byte-store-zero (Assuming compiler is able to | ||||
47 | /// perform write-merging). | ||||
48 | /// | ||||
49 | class FAddendCoef { | ||||
50 | public: | ||||
51 | // The constructor has to initialize a APFloat, which is unnecessary for | ||||
52 | // most addends which have coefficient either 1 or -1. So, the constructor | ||||
53 | // is expensive. In order to avoid the cost of the constructor, we should | ||||
54 | // reuse some instances whenever possible. The pre-created instances | ||||
55 | // FAddCombine::Add[0-5] embodies this idea. | ||||
56 | FAddendCoef() = default; | ||||
57 | ~FAddendCoef(); | ||||
58 | |||||
59 | // If possible, don't define operator+/operator- etc because these | ||||
60 | // operators inevitably call FAddendCoef's constructor which is not cheap. | ||||
61 | void operator=(const FAddendCoef &A); | ||||
62 | void operator+=(const FAddendCoef &A); | ||||
63 | void operator*=(const FAddendCoef &S); | ||||
64 | |||||
65 | void set(short C) { | ||||
66 | assert(!insaneIntVal(C) && "Insane coefficient")(static_cast <bool> (!insaneIntVal(C) && "Insane coefficient" ) ? void (0) : __assert_fail ("!insaneIntVal(C) && \"Insane coefficient\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 66 , __extension__ __PRETTY_FUNCTION__)); | ||||
67 | IsFp = false; IntVal = C; | ||||
68 | } | ||||
69 | |||||
70 | void set(const APFloat& C); | ||||
71 | |||||
72 | void negate(); | ||||
73 | |||||
74 | bool isZero() const { return isInt() ? !IntVal : getFpVal().isZero(); } | ||||
75 | Value *getValue(Type *) const; | ||||
76 | |||||
77 | bool isOne() const { return isInt() && IntVal == 1; } | ||||
78 | bool isTwo() const { return isInt() && IntVal == 2; } | ||||
79 | bool isMinusOne() const { return isInt() && IntVal == -1; } | ||||
80 | bool isMinusTwo() const { return isInt() && IntVal == -2; } | ||||
81 | |||||
82 | private: | ||||
83 | bool insaneIntVal(int V) { return V > 4 || V < -4; } | ||||
84 | |||||
85 | APFloat *getFpValPtr() { return reinterpret_cast<APFloat *>(&FpValBuf); } | ||||
86 | |||||
87 | const APFloat *getFpValPtr() const { | ||||
88 | return reinterpret_cast<const APFloat *>(&FpValBuf); | ||||
89 | } | ||||
90 | |||||
91 | const APFloat &getFpVal() const { | ||||
92 | assert(IsFp && BufHasFpVal && "Incorret state")(static_cast <bool> (IsFp && BufHasFpVal && "Incorret state") ? void (0) : __assert_fail ("IsFp && BufHasFpVal && \"Incorret state\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 92 , __extension__ __PRETTY_FUNCTION__)); | ||||
93 | return *getFpValPtr(); | ||||
94 | } | ||||
95 | |||||
96 | APFloat &getFpVal() { | ||||
97 | assert(IsFp && BufHasFpVal && "Incorret state")(static_cast <bool> (IsFp && BufHasFpVal && "Incorret state") ? void (0) : __assert_fail ("IsFp && BufHasFpVal && \"Incorret state\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 97 , __extension__ __PRETTY_FUNCTION__)); | ||||
98 | return *getFpValPtr(); | ||||
99 | } | ||||
100 | |||||
101 | bool isInt() const { return !IsFp; } | ||||
102 | |||||
103 | // If the coefficient is represented by an integer, promote it to a | ||||
104 | // floating point. | ||||
105 | void convertToFpType(const fltSemantics &Sem); | ||||
106 | |||||
107 | // Construct an APFloat from a signed integer. | ||||
108 | // TODO: We should get rid of this function when APFloat can be constructed | ||||
109 | // from an *SIGNED* integer. | ||||
110 | APFloat createAPFloatFromInt(const fltSemantics &Sem, int Val); | ||||
111 | |||||
112 | bool IsFp = false; | ||||
113 | |||||
114 | // True iff FpValBuf contains an instance of APFloat. | ||||
115 | bool BufHasFpVal = false; | ||||
116 | |||||
117 | // The integer coefficient of an individual addend is either 1 or -1, | ||||
118 | // and we try to simplify at most 4 addends from neighboring at most | ||||
119 | // two instructions. So the range of <IntVal> falls in [-4, 4]. APInt | ||||
120 | // is overkill of this end. | ||||
121 | short IntVal = 0; | ||||
122 | |||||
123 | AlignedCharArrayUnion<APFloat> FpValBuf; | ||||
124 | }; | ||||
125 | |||||
126 | /// FAddend is used to represent floating-point addend. An addend is | ||||
127 | /// represented as <C, V>, where the V is a symbolic value, and C is a | ||||
128 | /// constant coefficient. A constant addend is represented as <C, 0>. | ||||
129 | class FAddend { | ||||
130 | public: | ||||
131 | FAddend() = default; | ||||
132 | |||||
133 | void operator+=(const FAddend &T) { | ||||
134 | assert((Val == T.Val) && "Symbolic-values disagree")(static_cast <bool> ((Val == T.Val) && "Symbolic-values disagree" ) ? void (0) : __assert_fail ("(Val == T.Val) && \"Symbolic-values disagree\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 134 , __extension__ __PRETTY_FUNCTION__)); | ||||
135 | Coeff += T.Coeff; | ||||
136 | } | ||||
137 | |||||
138 | Value *getSymVal() const { return Val; } | ||||
139 | const FAddendCoef &getCoef() const { return Coeff; } | ||||
140 | |||||
141 | bool isConstant() const { return Val == nullptr; } | ||||
142 | bool isZero() const { return Coeff.isZero(); } | ||||
143 | |||||
144 | void set(short Coefficient, Value *V) { | ||||
145 | Coeff.set(Coefficient); | ||||
146 | Val = V; | ||||
147 | } | ||||
148 | void set(const APFloat &Coefficient, Value *V) { | ||||
149 | Coeff.set(Coefficient); | ||||
150 | Val = V; | ||||
151 | } | ||||
152 | void set(const ConstantFP *Coefficient, Value *V) { | ||||
153 | Coeff.set(Coefficient->getValueAPF()); | ||||
154 | Val = V; | ||||
155 | } | ||||
156 | |||||
157 | void negate() { Coeff.negate(); } | ||||
158 | |||||
159 | /// Drill down the U-D chain one step to find the definition of V, and | ||||
160 | /// try to break the definition into one or two addends. | ||||
161 | static unsigned drillValueDownOneStep(Value* V, FAddend &A0, FAddend &A1); | ||||
162 | |||||
163 | /// Similar to FAddend::drillDownOneStep() except that the value being | ||||
164 | /// splitted is the addend itself. | ||||
165 | unsigned drillAddendDownOneStep(FAddend &Addend0, FAddend &Addend1) const; | ||||
166 | |||||
167 | private: | ||||
168 | void Scale(const FAddendCoef& ScaleAmt) { Coeff *= ScaleAmt; } | ||||
169 | |||||
170 | // This addend has the value of "Coeff * Val". | ||||
171 | Value *Val = nullptr; | ||||
172 | FAddendCoef Coeff; | ||||
173 | }; | ||||
174 | |||||
175 | /// FAddCombine is the class for optimizing an unsafe fadd/fsub along | ||||
176 | /// with its neighboring at most two instructions. | ||||
177 | /// | ||||
178 | class FAddCombine { | ||||
179 | public: | ||||
180 | FAddCombine(InstCombiner::BuilderTy &B) : Builder(B) {} | ||||
181 | |||||
182 | Value *simplify(Instruction *FAdd); | ||||
183 | |||||
184 | private: | ||||
185 | using AddendVect = SmallVector<const FAddend *, 4>; | ||||
186 | |||||
187 | Value *simplifyFAdd(AddendVect& V, unsigned InstrQuota); | ||||
188 | |||||
189 | /// Convert given addend to a Value | ||||
190 | Value *createAddendVal(const FAddend &A, bool& NeedNeg); | ||||
191 | |||||
192 | /// Return the number of instructions needed to emit the N-ary addition. | ||||
193 | unsigned calcInstrNumber(const AddendVect& Vect); | ||||
194 | |||||
195 | Value *createFSub(Value *Opnd0, Value *Opnd1); | ||||
196 | Value *createFAdd(Value *Opnd0, Value *Opnd1); | ||||
197 | Value *createFMul(Value *Opnd0, Value *Opnd1); | ||||
198 | Value *createFNeg(Value *V); | ||||
199 | Value *createNaryFAdd(const AddendVect& Opnds, unsigned InstrQuota); | ||||
200 | void createInstPostProc(Instruction *NewInst, bool NoNumber = false); | ||||
201 | |||||
202 | // Debugging stuff are clustered here. | ||||
203 | #ifndef NDEBUG | ||||
204 | unsigned CreateInstrNum; | ||||
205 | void initCreateInstNum() { CreateInstrNum = 0; } | ||||
206 | void incCreateInstNum() { CreateInstrNum++; } | ||||
207 | #else | ||||
208 | void initCreateInstNum() {} | ||||
209 | void incCreateInstNum() {} | ||||
210 | #endif | ||||
211 | |||||
212 | InstCombiner::BuilderTy &Builder; | ||||
213 | Instruction *Instr = nullptr; | ||||
214 | }; | ||||
215 | |||||
216 | } // end anonymous namespace | ||||
217 | |||||
218 | //===----------------------------------------------------------------------===// | ||||
219 | // | ||||
220 | // Implementation of | ||||
221 | // {FAddendCoef, FAddend, FAddition, FAddCombine}. | ||||
222 | // | ||||
223 | //===----------------------------------------------------------------------===// | ||||
224 | FAddendCoef::~FAddendCoef() { | ||||
225 | if (BufHasFpVal) | ||||
226 | getFpValPtr()->~APFloat(); | ||||
227 | } | ||||
228 | |||||
229 | void FAddendCoef::set(const APFloat& C) { | ||||
230 | APFloat *P = getFpValPtr(); | ||||
231 | |||||
232 | if (isInt()) { | ||||
233 | // As the buffer is meanless byte stream, we cannot call | ||||
234 | // APFloat::operator=(). | ||||
235 | new(P) APFloat(C); | ||||
236 | } else | ||||
237 | *P = C; | ||||
238 | |||||
239 | IsFp = BufHasFpVal = true; | ||||
240 | } | ||||
241 | |||||
242 | void FAddendCoef::convertToFpType(const fltSemantics &Sem) { | ||||
243 | if (!isInt()) | ||||
244 | return; | ||||
245 | |||||
246 | APFloat *P = getFpValPtr(); | ||||
247 | if (IntVal > 0) | ||||
248 | new(P) APFloat(Sem, IntVal); | ||||
249 | else { | ||||
250 | new(P) APFloat(Sem, 0 - IntVal); | ||||
251 | P->changeSign(); | ||||
252 | } | ||||
253 | IsFp = BufHasFpVal = true; | ||||
254 | } | ||||
255 | |||||
256 | APFloat FAddendCoef::createAPFloatFromInt(const fltSemantics &Sem, int Val) { | ||||
257 | if (Val >= 0) | ||||
258 | return APFloat(Sem, Val); | ||||
259 | |||||
260 | APFloat T(Sem, 0 - Val); | ||||
261 | T.changeSign(); | ||||
262 | |||||
263 | return T; | ||||
264 | } | ||||
265 | |||||
266 | void FAddendCoef::operator=(const FAddendCoef &That) { | ||||
267 | if (That.isInt()) | ||||
268 | set(That.IntVal); | ||||
269 | else | ||||
270 | set(That.getFpVal()); | ||||
271 | } | ||||
272 | |||||
273 | void FAddendCoef::operator+=(const FAddendCoef &That) { | ||||
274 | RoundingMode RndMode = RoundingMode::NearestTiesToEven; | ||||
275 | if (isInt() == That.isInt()) { | ||||
276 | if (isInt()) | ||||
277 | IntVal += That.IntVal; | ||||
278 | else | ||||
279 | getFpVal().add(That.getFpVal(), RndMode); | ||||
280 | return; | ||||
281 | } | ||||
282 | |||||
283 | if (isInt()) { | ||||
284 | const APFloat &T = That.getFpVal(); | ||||
285 | convertToFpType(T.getSemantics()); | ||||
286 | getFpVal().add(T, RndMode); | ||||
287 | return; | ||||
288 | } | ||||
289 | |||||
290 | APFloat &T = getFpVal(); | ||||
291 | T.add(createAPFloatFromInt(T.getSemantics(), That.IntVal), RndMode); | ||||
292 | } | ||||
293 | |||||
294 | void FAddendCoef::operator*=(const FAddendCoef &That) { | ||||
295 | if (That.isOne()) | ||||
296 | return; | ||||
297 | |||||
298 | if (That.isMinusOne()) { | ||||
299 | negate(); | ||||
300 | return; | ||||
301 | } | ||||
302 | |||||
303 | if (isInt() && That.isInt()) { | ||||
304 | int Res = IntVal * (int)That.IntVal; | ||||
305 | assert(!insaneIntVal(Res) && "Insane int value")(static_cast <bool> (!insaneIntVal(Res) && "Insane int value" ) ? void (0) : __assert_fail ("!insaneIntVal(Res) && \"Insane int value\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 305 , __extension__ __PRETTY_FUNCTION__)); | ||||
306 | IntVal = Res; | ||||
307 | return; | ||||
308 | } | ||||
309 | |||||
310 | const fltSemantics &Semantic = | ||||
311 | isInt() ? That.getFpVal().getSemantics() : getFpVal().getSemantics(); | ||||
312 | |||||
313 | if (isInt()) | ||||
314 | convertToFpType(Semantic); | ||||
315 | APFloat &F0 = getFpVal(); | ||||
316 | |||||
317 | if (That.isInt()) | ||||
318 | F0.multiply(createAPFloatFromInt(Semantic, That.IntVal), | ||||
319 | APFloat::rmNearestTiesToEven); | ||||
320 | else | ||||
321 | F0.multiply(That.getFpVal(), APFloat::rmNearestTiesToEven); | ||||
322 | } | ||||
323 | |||||
324 | void FAddendCoef::negate() { | ||||
325 | if (isInt()) | ||||
326 | IntVal = 0 - IntVal; | ||||
327 | else | ||||
328 | getFpVal().changeSign(); | ||||
329 | } | ||||
330 | |||||
331 | Value *FAddendCoef::getValue(Type *Ty) const { | ||||
332 | return isInt() ? | ||||
333 | ConstantFP::get(Ty, float(IntVal)) : | ||||
334 | ConstantFP::get(Ty->getContext(), getFpVal()); | ||||
335 | } | ||||
336 | |||||
337 | // The definition of <Val> Addends | ||||
338 | // ========================================= | ||||
339 | // A + B <1, A>, <1,B> | ||||
340 | // A - B <1, A>, <1,B> | ||||
341 | // 0 - B <-1, B> | ||||
342 | // C * A, <C, A> | ||||
343 | // A + C <1, A> <C, NULL> | ||||
344 | // 0 +/- 0 <0, NULL> (corner case) | ||||
345 | // | ||||
346 | // Legend: A and B are not constant, C is constant | ||||
347 | unsigned FAddend::drillValueDownOneStep | ||||
348 | (Value *Val, FAddend &Addend0, FAddend &Addend1) { | ||||
349 | Instruction *I = nullptr; | ||||
350 | if (!Val || !(I = dyn_cast<Instruction>(Val))) | ||||
351 | return 0; | ||||
352 | |||||
353 | unsigned Opcode = I->getOpcode(); | ||||
354 | |||||
355 | if (Opcode == Instruction::FAdd || Opcode == Instruction::FSub) { | ||||
356 | ConstantFP *C0, *C1; | ||||
357 | Value *Opnd0 = I->getOperand(0); | ||||
358 | Value *Opnd1 = I->getOperand(1); | ||||
359 | if ((C0 = dyn_cast<ConstantFP>(Opnd0)) && C0->isZero()) | ||||
360 | Opnd0 = nullptr; | ||||
361 | |||||
362 | if ((C1 = dyn_cast<ConstantFP>(Opnd1)) && C1->isZero()) | ||||
363 | Opnd1 = nullptr; | ||||
364 | |||||
365 | if (Opnd0) { | ||||
366 | if (!C0) | ||||
367 | Addend0.set(1, Opnd0); | ||||
368 | else | ||||
369 | Addend0.set(C0, nullptr); | ||||
370 | } | ||||
371 | |||||
372 | if (Opnd1) { | ||||
373 | FAddend &Addend = Opnd0 ? Addend1 : Addend0; | ||||
374 | if (!C1) | ||||
375 | Addend.set(1, Opnd1); | ||||
376 | else | ||||
377 | Addend.set(C1, nullptr); | ||||
378 | if (Opcode == Instruction::FSub) | ||||
379 | Addend.negate(); | ||||
380 | } | ||||
381 | |||||
382 | if (Opnd0 || Opnd1) | ||||
383 | return Opnd0 && Opnd1 ? 2 : 1; | ||||
384 | |||||
385 | // Both operands are zero. Weird! | ||||
386 | Addend0.set(APFloat(C0->getValueAPF().getSemantics()), nullptr); | ||||
387 | return 1; | ||||
388 | } | ||||
389 | |||||
390 | if (I->getOpcode() == Instruction::FMul) { | ||||
391 | Value *V0 = I->getOperand(0); | ||||
392 | Value *V1 = I->getOperand(1); | ||||
393 | if (ConstantFP *C = dyn_cast<ConstantFP>(V0)) { | ||||
394 | Addend0.set(C, V1); | ||||
395 | return 1; | ||||
396 | } | ||||
397 | |||||
398 | if (ConstantFP *C = dyn_cast<ConstantFP>(V1)) { | ||||
399 | Addend0.set(C, V0); | ||||
400 | return 1; | ||||
401 | } | ||||
402 | } | ||||
403 | |||||
404 | return 0; | ||||
405 | } | ||||
406 | |||||
407 | // Try to break *this* addend into two addends. e.g. Suppose this addend is | ||||
408 | // <2.3, V>, and V = X + Y, by calling this function, we obtain two addends, | ||||
409 | // i.e. <2.3, X> and <2.3, Y>. | ||||
410 | unsigned FAddend::drillAddendDownOneStep | ||||
411 | (FAddend &Addend0, FAddend &Addend1) const { | ||||
412 | if (isConstant()) | ||||
413 | return 0; | ||||
414 | |||||
415 | unsigned BreakNum = FAddend::drillValueDownOneStep(Val, Addend0, Addend1); | ||||
416 | if (!BreakNum || Coeff.isOne()) | ||||
417 | return BreakNum; | ||||
418 | |||||
419 | Addend0.Scale(Coeff); | ||||
420 | |||||
421 | if (BreakNum == 2) | ||||
422 | Addend1.Scale(Coeff); | ||||
423 | |||||
424 | return BreakNum; | ||||
425 | } | ||||
426 | |||||
427 | Value *FAddCombine::simplify(Instruction *I) { | ||||
428 | assert(I->hasAllowReassoc() && I->hasNoSignedZeros() &&(static_cast <bool> (I->hasAllowReassoc() && I->hasNoSignedZeros() && "Expected 'reassoc'+'nsz' instruction" ) ? void (0) : __assert_fail ("I->hasAllowReassoc() && I->hasNoSignedZeros() && \"Expected 'reassoc'+'nsz' instruction\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 429 , __extension__ __PRETTY_FUNCTION__)) | ||||
429 | "Expected 'reassoc'+'nsz' instruction")(static_cast <bool> (I->hasAllowReassoc() && I->hasNoSignedZeros() && "Expected 'reassoc'+'nsz' instruction" ) ? void (0) : __assert_fail ("I->hasAllowReassoc() && I->hasNoSignedZeros() && \"Expected 'reassoc'+'nsz' instruction\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 429 , __extension__ __PRETTY_FUNCTION__)); | ||||
430 | |||||
431 | // Currently we are not able to handle vector type. | ||||
432 | if (I->getType()->isVectorTy()) | ||||
433 | return nullptr; | ||||
434 | |||||
435 | assert((I->getOpcode() == Instruction::FAdd ||(static_cast <bool> ((I->getOpcode() == Instruction:: FAdd || I->getOpcode() == Instruction::FSub) && "Expect add/sub" ) ? void (0) : __assert_fail ("(I->getOpcode() == Instruction::FAdd || I->getOpcode() == Instruction::FSub) && \"Expect add/sub\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 436 , __extension__ __PRETTY_FUNCTION__)) | ||||
436 | I->getOpcode() == Instruction::FSub) && "Expect add/sub")(static_cast <bool> ((I->getOpcode() == Instruction:: FAdd || I->getOpcode() == Instruction::FSub) && "Expect add/sub" ) ? void (0) : __assert_fail ("(I->getOpcode() == Instruction::FAdd || I->getOpcode() == Instruction::FSub) && \"Expect add/sub\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 436 , __extension__ __PRETTY_FUNCTION__)); | ||||
437 | |||||
438 | // Save the instruction before calling other member-functions. | ||||
439 | Instr = I; | ||||
440 | |||||
441 | FAddend Opnd0, Opnd1, Opnd0_0, Opnd0_1, Opnd1_0, Opnd1_1; | ||||
442 | |||||
443 | unsigned OpndNum = FAddend::drillValueDownOneStep(I, Opnd0, Opnd1); | ||||
444 | |||||
445 | // Step 1: Expand the 1st addend into Opnd0_0 and Opnd0_1. | ||||
446 | unsigned Opnd0_ExpNum = 0; | ||||
447 | unsigned Opnd1_ExpNum = 0; | ||||
448 | |||||
449 | if (!Opnd0.isConstant()) | ||||
450 | Opnd0_ExpNum = Opnd0.drillAddendDownOneStep(Opnd0_0, Opnd0_1); | ||||
451 | |||||
452 | // Step 2: Expand the 2nd addend into Opnd1_0 and Opnd1_1. | ||||
453 | if (OpndNum == 2 && !Opnd1.isConstant()) | ||||
454 | Opnd1_ExpNum = Opnd1.drillAddendDownOneStep(Opnd1_0, Opnd1_1); | ||||
455 | |||||
456 | // Step 3: Try to optimize Opnd0_0 + Opnd0_1 + Opnd1_0 + Opnd1_1 | ||||
457 | if (Opnd0_ExpNum && Opnd1_ExpNum) { | ||||
458 | AddendVect AllOpnds; | ||||
459 | AllOpnds.push_back(&Opnd0_0); | ||||
460 | AllOpnds.push_back(&Opnd1_0); | ||||
461 | if (Opnd0_ExpNum == 2) | ||||
462 | AllOpnds.push_back(&Opnd0_1); | ||||
463 | if (Opnd1_ExpNum == 2) | ||||
464 | AllOpnds.push_back(&Opnd1_1); | ||||
465 | |||||
466 | // Compute instruction quota. We should save at least one instruction. | ||||
467 | unsigned InstQuota = 0; | ||||
468 | |||||
469 | Value *V0 = I->getOperand(0); | ||||
470 | Value *V1 = I->getOperand(1); | ||||
471 | InstQuota = ((!isa<Constant>(V0) && V0->hasOneUse()) && | ||||
472 | (!isa<Constant>(V1) && V1->hasOneUse())) ? 2 : 1; | ||||
473 | |||||
474 | if (Value *R = simplifyFAdd(AllOpnds, InstQuota)) | ||||
475 | return R; | ||||
476 | } | ||||
477 | |||||
478 | if (OpndNum != 2) { | ||||
479 | // The input instruction is : "I=0.0 +/- V". If the "V" were able to be | ||||
480 | // splitted into two addends, say "V = X - Y", the instruction would have | ||||
481 | // been optimized into "I = Y - X" in the previous steps. | ||||
482 | // | ||||
483 | const FAddendCoef &CE = Opnd0.getCoef(); | ||||
484 | return CE.isOne() ? Opnd0.getSymVal() : nullptr; | ||||
485 | } | ||||
486 | |||||
487 | // step 4: Try to optimize Opnd0 + Opnd1_0 [+ Opnd1_1] | ||||
488 | if (Opnd1_ExpNum) { | ||||
489 | AddendVect AllOpnds; | ||||
490 | AllOpnds.push_back(&Opnd0); | ||||
491 | AllOpnds.push_back(&Opnd1_0); | ||||
492 | if (Opnd1_ExpNum == 2) | ||||
493 | AllOpnds.push_back(&Opnd1_1); | ||||
494 | |||||
495 | if (Value *R = simplifyFAdd(AllOpnds, 1)) | ||||
496 | return R; | ||||
497 | } | ||||
498 | |||||
499 | // step 5: Try to optimize Opnd1 + Opnd0_0 [+ Opnd0_1] | ||||
500 | if (Opnd0_ExpNum) { | ||||
501 | AddendVect AllOpnds; | ||||
502 | AllOpnds.push_back(&Opnd1); | ||||
503 | AllOpnds.push_back(&Opnd0_0); | ||||
504 | if (Opnd0_ExpNum == 2) | ||||
505 | AllOpnds.push_back(&Opnd0_1); | ||||
506 | |||||
507 | if (Value *R = simplifyFAdd(AllOpnds, 1)) | ||||
508 | return R; | ||||
509 | } | ||||
510 | |||||
511 | return nullptr; | ||||
512 | } | ||||
513 | |||||
514 | Value *FAddCombine::simplifyFAdd(AddendVect& Addends, unsigned InstrQuota) { | ||||
515 | unsigned AddendNum = Addends.size(); | ||||
516 | assert(AddendNum <= 4 && "Too many addends")(static_cast <bool> (AddendNum <= 4 && "Too many addends" ) ? void (0) : __assert_fail ("AddendNum <= 4 && \"Too many addends\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 516 , __extension__ __PRETTY_FUNCTION__)); | ||||
517 | |||||
518 | // For saving intermediate results; | ||||
519 | unsigned NextTmpIdx = 0; | ||||
520 | FAddend TmpResult[3]; | ||||
521 | |||||
522 | // Simplified addends are placed <SimpVect>. | ||||
523 | AddendVect SimpVect; | ||||
524 | |||||
525 | // The outer loop works on one symbolic-value at a time. Suppose the input | ||||
526 | // addends are : <a1, x>, <b1, y>, <a2, x>, <c1, z>, <b2, y>, ... | ||||
527 | // The symbolic-values will be processed in this order: x, y, z. | ||||
528 | for (unsigned SymIdx = 0; SymIdx < AddendNum; SymIdx++) { | ||||
529 | |||||
530 | const FAddend *ThisAddend = Addends[SymIdx]; | ||||
531 | if (!ThisAddend) { | ||||
532 | // This addend was processed before. | ||||
533 | continue; | ||||
534 | } | ||||
535 | |||||
536 | Value *Val = ThisAddend->getSymVal(); | ||||
537 | |||||
538 | // If the resulting expr has constant-addend, this constant-addend is | ||||
539 | // desirable to reside at the top of the resulting expression tree. Placing | ||||
540 | // constant close to super-expr(s) will potentially reveal some | ||||
541 | // optimization opportunities in super-expr(s). Here we do not implement | ||||
542 | // this logic intentionally and rely on SimplifyAssociativeOrCommutative | ||||
543 | // call later. | ||||
544 | |||||
545 | unsigned StartIdx = SimpVect.size(); | ||||
546 | SimpVect.push_back(ThisAddend); | ||||
547 | |||||
548 | // The inner loop collects addends sharing same symbolic-value, and these | ||||
549 | // addends will be later on folded into a single addend. Following above | ||||
550 | // example, if the symbolic value "y" is being processed, the inner loop | ||||
551 | // will collect two addends "<b1,y>" and "<b2,Y>". These two addends will | ||||
552 | // be later on folded into "<b1+b2, y>". | ||||
553 | for (unsigned SameSymIdx = SymIdx + 1; | ||||
554 | SameSymIdx < AddendNum; SameSymIdx++) { | ||||
555 | const FAddend *T = Addends[SameSymIdx]; | ||||
556 | if (T && T->getSymVal() == Val) { | ||||
557 | // Set null such that next iteration of the outer loop will not process | ||||
558 | // this addend again. | ||||
559 | Addends[SameSymIdx] = nullptr; | ||||
560 | SimpVect.push_back(T); | ||||
561 | } | ||||
562 | } | ||||
563 | |||||
564 | // If multiple addends share same symbolic value, fold them together. | ||||
565 | if (StartIdx + 1 != SimpVect.size()) { | ||||
566 | FAddend &R = TmpResult[NextTmpIdx ++]; | ||||
567 | R = *SimpVect[StartIdx]; | ||||
568 | for (unsigned Idx = StartIdx + 1; Idx < SimpVect.size(); Idx++) | ||||
569 | R += *SimpVect[Idx]; | ||||
570 | |||||
571 | // Pop all addends being folded and push the resulting folded addend. | ||||
572 | SimpVect.resize(StartIdx); | ||||
573 | if (!R.isZero()) { | ||||
574 | SimpVect.push_back(&R); | ||||
575 | } | ||||
576 | } | ||||
577 | } | ||||
578 | |||||
579 | assert((NextTmpIdx <= std::size(TmpResult) + 1) && "out-of-bound access")(static_cast <bool> ((NextTmpIdx <= std::size(TmpResult ) + 1) && "out-of-bound access") ? void (0) : __assert_fail ("(NextTmpIdx <= std::size(TmpResult) + 1) && \"out-of-bound access\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 579 , __extension__ __PRETTY_FUNCTION__)); | ||||
580 | |||||
581 | Value *Result; | ||||
582 | if (!SimpVect.empty()) | ||||
583 | Result = createNaryFAdd(SimpVect, InstrQuota); | ||||
584 | else { | ||||
585 | // The addition is folded to 0.0. | ||||
586 | Result = ConstantFP::get(Instr->getType(), 0.0); | ||||
587 | } | ||||
588 | |||||
589 | return Result; | ||||
590 | } | ||||
591 | |||||
592 | Value *FAddCombine::createNaryFAdd | ||||
593 | (const AddendVect &Opnds, unsigned InstrQuota) { | ||||
594 | assert(!Opnds.empty() && "Expect at least one addend")(static_cast <bool> (!Opnds.empty() && "Expect at least one addend" ) ? void (0) : __assert_fail ("!Opnds.empty() && \"Expect at least one addend\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 594 , __extension__ __PRETTY_FUNCTION__)); | ||||
595 | |||||
596 | // Step 1: Check if the # of instructions needed exceeds the quota. | ||||
597 | |||||
598 | unsigned InstrNeeded = calcInstrNumber(Opnds); | ||||
599 | if (InstrNeeded > InstrQuota) | ||||
600 | return nullptr; | ||||
601 | |||||
602 | initCreateInstNum(); | ||||
603 | |||||
604 | // step 2: Emit the N-ary addition. | ||||
605 | // Note that at most three instructions are involved in Fadd-InstCombine: the | ||||
606 | // addition in question, and at most two neighboring instructions. | ||||
607 | // The resulting optimized addition should have at least one less instruction | ||||
608 | // than the original addition expression tree. This implies that the resulting | ||||
609 | // N-ary addition has at most two instructions, and we don't need to worry | ||||
610 | // about tree-height when constructing the N-ary addition. | ||||
611 | |||||
612 | Value *LastVal = nullptr; | ||||
613 | bool LastValNeedNeg = false; | ||||
614 | |||||
615 | // Iterate the addends, creating fadd/fsub using adjacent two addends. | ||||
616 | for (const FAddend *Opnd : Opnds) { | ||||
617 | bool NeedNeg; | ||||
618 | Value *V = createAddendVal(*Opnd, NeedNeg); | ||||
619 | if (!LastVal) { | ||||
620 | LastVal = V; | ||||
621 | LastValNeedNeg = NeedNeg; | ||||
622 | continue; | ||||
623 | } | ||||
624 | |||||
625 | if (LastValNeedNeg == NeedNeg) { | ||||
626 | LastVal = createFAdd(LastVal, V); | ||||
627 | continue; | ||||
628 | } | ||||
629 | |||||
630 | if (LastValNeedNeg) | ||||
631 | LastVal = createFSub(V, LastVal); | ||||
632 | else | ||||
633 | LastVal = createFSub(LastVal, V); | ||||
634 | |||||
635 | LastValNeedNeg = false; | ||||
636 | } | ||||
637 | |||||
638 | if (LastValNeedNeg) { | ||||
639 | LastVal = createFNeg(LastVal); | ||||
640 | } | ||||
641 | |||||
642 | #ifndef NDEBUG | ||||
643 | assert(CreateInstrNum == InstrNeeded &&(static_cast <bool> (CreateInstrNum == InstrNeeded && "Inconsistent in instruction numbers") ? void (0) : __assert_fail ("CreateInstrNum == InstrNeeded && \"Inconsistent in instruction numbers\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 644 , __extension__ __PRETTY_FUNCTION__)) | ||||
644 | "Inconsistent in instruction numbers")(static_cast <bool> (CreateInstrNum == InstrNeeded && "Inconsistent in instruction numbers") ? void (0) : __assert_fail ("CreateInstrNum == InstrNeeded && \"Inconsistent in instruction numbers\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 644 , __extension__ __PRETTY_FUNCTION__)); | ||||
645 | #endif | ||||
646 | |||||
647 | return LastVal; | ||||
648 | } | ||||
649 | |||||
650 | Value *FAddCombine::createFSub(Value *Opnd0, Value *Opnd1) { | ||||
651 | Value *V = Builder.CreateFSub(Opnd0, Opnd1); | ||||
652 | if (Instruction *I = dyn_cast<Instruction>(V)) | ||||
653 | createInstPostProc(I); | ||||
654 | return V; | ||||
655 | } | ||||
656 | |||||
657 | Value *FAddCombine::createFNeg(Value *V) { | ||||
658 | Value *NewV = Builder.CreateFNeg(V); | ||||
659 | if (Instruction *I = dyn_cast<Instruction>(NewV)) | ||||
660 | createInstPostProc(I, true); // fneg's don't receive instruction numbers. | ||||
661 | return NewV; | ||||
662 | } | ||||
663 | |||||
664 | Value *FAddCombine::createFAdd(Value *Opnd0, Value *Opnd1) { | ||||
665 | Value *V = Builder.CreateFAdd(Opnd0, Opnd1); | ||||
666 | if (Instruction *I = dyn_cast<Instruction>(V)) | ||||
667 | createInstPostProc(I); | ||||
668 | return V; | ||||
669 | } | ||||
670 | |||||
671 | Value *FAddCombine::createFMul(Value *Opnd0, Value *Opnd1) { | ||||
672 | Value *V = Builder.CreateFMul(Opnd0, Opnd1); | ||||
673 | if (Instruction *I = dyn_cast<Instruction>(V)) | ||||
674 | createInstPostProc(I); | ||||
675 | return V; | ||||
676 | } | ||||
677 | |||||
678 | void FAddCombine::createInstPostProc(Instruction *NewInstr, bool NoNumber) { | ||||
679 | NewInstr->setDebugLoc(Instr->getDebugLoc()); | ||||
680 | |||||
681 | // Keep track of the number of instruction created. | ||||
682 | if (!NoNumber) | ||||
683 | incCreateInstNum(); | ||||
684 | |||||
685 | // Propagate fast-math flags | ||||
686 | NewInstr->setFastMathFlags(Instr->getFastMathFlags()); | ||||
687 | } | ||||
688 | |||||
689 | // Return the number of instruction needed to emit the N-ary addition. | ||||
690 | // NOTE: Keep this function in sync with createAddendVal(). | ||||
691 | unsigned FAddCombine::calcInstrNumber(const AddendVect &Opnds) { | ||||
692 | unsigned OpndNum = Opnds.size(); | ||||
693 | unsigned InstrNeeded = OpndNum - 1; | ||||
694 | |||||
695 | // Adjust the number of instructions needed to emit the N-ary add. | ||||
696 | for (const FAddend *Opnd : Opnds) { | ||||
697 | if (Opnd->isConstant()) | ||||
698 | continue; | ||||
699 | |||||
700 | // The constant check above is really for a few special constant | ||||
701 | // coefficients. | ||||
702 | if (isa<UndefValue>(Opnd->getSymVal())) | ||||
703 | continue; | ||||
704 | |||||
705 | const FAddendCoef &CE = Opnd->getCoef(); | ||||
706 | // Let the addend be "c * x". If "c == +/-1", the value of the addend | ||||
707 | // is immediately available; otherwise, it needs exactly one instruction | ||||
708 | // to evaluate the value. | ||||
709 | if (!CE.isMinusOne() && !CE.isOne()) | ||||
710 | InstrNeeded++; | ||||
711 | } | ||||
712 | return InstrNeeded; | ||||
713 | } | ||||
714 | |||||
715 | // Input Addend Value NeedNeg(output) | ||||
716 | // ================================================================ | ||||
717 | // Constant C C false | ||||
718 | // <+/-1, V> V coefficient is -1 | ||||
719 | // <2/-2, V> "fadd V, V" coefficient is -2 | ||||
720 | // <C, V> "fmul V, C" false | ||||
721 | // | ||||
722 | // NOTE: Keep this function in sync with FAddCombine::calcInstrNumber. | ||||
723 | Value *FAddCombine::createAddendVal(const FAddend &Opnd, bool &NeedNeg) { | ||||
724 | const FAddendCoef &Coeff = Opnd.getCoef(); | ||||
725 | |||||
726 | if (Opnd.isConstant()) { | ||||
727 | NeedNeg = false; | ||||
728 | return Coeff.getValue(Instr->getType()); | ||||
729 | } | ||||
730 | |||||
731 | Value *OpndVal = Opnd.getSymVal(); | ||||
732 | |||||
733 | if (Coeff.isMinusOne() || Coeff.isOne()) { | ||||
734 | NeedNeg = Coeff.isMinusOne(); | ||||
735 | return OpndVal; | ||||
736 | } | ||||
737 | |||||
738 | if (Coeff.isTwo() || Coeff.isMinusTwo()) { | ||||
739 | NeedNeg = Coeff.isMinusTwo(); | ||||
740 | return createFAdd(OpndVal, OpndVal); | ||||
741 | } | ||||
742 | |||||
743 | NeedNeg = false; | ||||
744 | return createFMul(OpndVal, Coeff.getValue(Instr->getType())); | ||||
745 | } | ||||
746 | |||||
747 | // Checks if any operand is negative and we can convert add to sub. | ||||
748 | // This function checks for following negative patterns | ||||
749 | // ADD(XOR(OR(Z, NOT(C)), C)), 1) == NEG(AND(Z, C)) | ||||
750 | // ADD(XOR(AND(Z, C), C), 1) == NEG(OR(Z, ~C)) | ||||
751 | // XOR(AND(Z, C), (C + 1)) == NEG(OR(Z, ~C)) if C is even | ||||
752 | static Value *checkForNegativeOperand(BinaryOperator &I, | ||||
753 | InstCombiner::BuilderTy &Builder) { | ||||
754 | Value *LHS = I.getOperand(0), *RHS = I.getOperand(1); | ||||
755 | |||||
756 | // This function creates 2 instructions to replace ADD, we need at least one | ||||
757 | // of LHS or RHS to have one use to ensure benefit in transform. | ||||
758 | if (!LHS->hasOneUse() && !RHS->hasOneUse()) | ||||
759 | return nullptr; | ||||
760 | |||||
761 | Value *X = nullptr, *Y = nullptr, *Z = nullptr; | ||||
762 | const APInt *C1 = nullptr, *C2 = nullptr; | ||||
763 | |||||
764 | // if ONE is on other side, swap | ||||
765 | if (match(RHS, m_Add(m_Value(X), m_One()))) | ||||
766 | std::swap(LHS, RHS); | ||||
767 | |||||
768 | if (match(LHS, m_Add(m_Value(X), m_One()))) { | ||||
769 | // if XOR on other side, swap | ||||
770 | if (match(RHS, m_Xor(m_Value(Y), m_APInt(C1)))) | ||||
771 | std::swap(X, RHS); | ||||
772 | |||||
773 | if (match(X, m_Xor(m_Value(Y), m_APInt(C1)))) { | ||||
774 | // X = XOR(Y, C1), Y = OR(Z, C2), C2 = NOT(C1) ==> X == NOT(AND(Z, C1)) | ||||
775 | // ADD(ADD(X, 1), RHS) == ADD(X, ADD(RHS, 1)) == SUB(RHS, AND(Z, C1)) | ||||
776 | if (match(Y, m_Or(m_Value(Z), m_APInt(C2))) && (*C2 == ~(*C1))) { | ||||
777 | Value *NewAnd = Builder.CreateAnd(Z, *C1); | ||||
778 | return Builder.CreateSub(RHS, NewAnd, "sub"); | ||||
779 | } else if (match(Y, m_And(m_Value(Z), m_APInt(C2))) && (*C1 == *C2)) { | ||||
780 | // X = XOR(Y, C1), Y = AND(Z, C2), C2 == C1 ==> X == NOT(OR(Z, ~C1)) | ||||
781 | // ADD(ADD(X, 1), RHS) == ADD(X, ADD(RHS, 1)) == SUB(RHS, OR(Z, ~C1)) | ||||
782 | Value *NewOr = Builder.CreateOr(Z, ~(*C1)); | ||||
783 | return Builder.CreateSub(RHS, NewOr, "sub"); | ||||
784 | } | ||||
785 | } | ||||
786 | } | ||||
787 | |||||
788 | // Restore LHS and RHS | ||||
789 | LHS = I.getOperand(0); | ||||
790 | RHS = I.getOperand(1); | ||||
791 | |||||
792 | // if XOR is on other side, swap | ||||
793 | if (match(RHS, m_Xor(m_Value(Y), m_APInt(C1)))) | ||||
794 | std::swap(LHS, RHS); | ||||
795 | |||||
796 | // C2 is ODD | ||||
797 | // LHS = XOR(Y, C1), Y = AND(Z, C2), C1 == (C2 + 1) => LHS == NEG(OR(Z, ~C2)) | ||||
798 | // ADD(LHS, RHS) == SUB(RHS, OR(Z, ~C2)) | ||||
799 | if (match(LHS, m_Xor(m_Value(Y), m_APInt(C1)))) | ||||
800 | if (C1->countr_zero() == 0) | ||||
801 | if (match(Y, m_And(m_Value(Z), m_APInt(C2))) && *C1 == (*C2 + 1)) { | ||||
802 | Value *NewOr = Builder.CreateOr(Z, ~(*C2)); | ||||
803 | return Builder.CreateSub(RHS, NewOr, "sub"); | ||||
804 | } | ||||
805 | return nullptr; | ||||
806 | } | ||||
807 | |||||
808 | /// Wrapping flags may allow combining constants separated by an extend. | ||||
809 | static Instruction *foldNoWrapAdd(BinaryOperator &Add, | ||||
810 | InstCombiner::BuilderTy &Builder) { | ||||
811 | Value *Op0 = Add.getOperand(0), *Op1 = Add.getOperand(1); | ||||
812 | Type *Ty = Add.getType(); | ||||
813 | Constant *Op1C; | ||||
814 | if (!match(Op1, m_Constant(Op1C))) | ||||
815 | return nullptr; | ||||
816 | |||||
817 | // Try this match first because it results in an add in the narrow type. | ||||
818 | // (zext (X +nuw C2)) + C1 --> zext (X + (C2 + trunc(C1))) | ||||
819 | Value *X; | ||||
820 | const APInt *C1, *C2; | ||||
821 | if (match(Op1, m_APInt(C1)) && | ||||
822 | match(Op0, m_OneUse(m_ZExt(m_NUWAdd(m_Value(X), m_APInt(C2))))) && | ||||
823 | C1->isNegative() && C1->sge(-C2->sext(C1->getBitWidth()))) { | ||||
824 | Constant *NewC = | ||||
825 | ConstantInt::get(X->getType(), *C2 + C1->trunc(C2->getBitWidth())); | ||||
826 | return new ZExtInst(Builder.CreateNUWAdd(X, NewC), Ty); | ||||
827 | } | ||||
828 | |||||
829 | // More general combining of constants in the wide type. | ||||
830 | // (sext (X +nsw NarrowC)) + C --> (sext X) + (sext(NarrowC) + C) | ||||
831 | Constant *NarrowC; | ||||
832 | if (match(Op0, m_OneUse(m_SExt(m_NSWAdd(m_Value(X), m_Constant(NarrowC)))))) { | ||||
833 | Constant *WideC = ConstantExpr::getSExt(NarrowC, Ty); | ||||
834 | Constant *NewC = ConstantExpr::getAdd(WideC, Op1C); | ||||
835 | Value *WideX = Builder.CreateSExt(X, Ty); | ||||
836 | return BinaryOperator::CreateAdd(WideX, NewC); | ||||
837 | } | ||||
838 | // (zext (X +nuw NarrowC)) + C --> (zext X) + (zext(NarrowC) + C) | ||||
839 | if (match(Op0, m_OneUse(m_ZExt(m_NUWAdd(m_Value(X), m_Constant(NarrowC)))))) { | ||||
840 | Constant *WideC = ConstantExpr::getZExt(NarrowC, Ty); | ||||
841 | Constant *NewC = ConstantExpr::getAdd(WideC, Op1C); | ||||
842 | Value *WideX = Builder.CreateZExt(X, Ty); | ||||
843 | return BinaryOperator::CreateAdd(WideX, NewC); | ||||
844 | } | ||||
845 | |||||
846 | return nullptr; | ||||
847 | } | ||||
848 | |||||
849 | Instruction *InstCombinerImpl::foldAddWithConstant(BinaryOperator &Add) { | ||||
850 | Value *Op0 = Add.getOperand(0), *Op1 = Add.getOperand(1); | ||||
851 | Type *Ty = Add.getType(); | ||||
852 | Constant *Op1C; | ||||
853 | if (!match(Op1, m_ImmConstant(Op1C))) | ||||
854 | return nullptr; | ||||
855 | |||||
856 | if (Instruction *NV = foldBinOpIntoSelectOrPhi(Add)) | ||||
857 | return NV; | ||||
858 | |||||
859 | Value *X; | ||||
860 | Constant *Op00C; | ||||
861 | |||||
862 | // add (sub C1, X), C2 --> sub (add C1, C2), X | ||||
863 | if (match(Op0, m_Sub(m_Constant(Op00C), m_Value(X)))) | ||||
864 | return BinaryOperator::CreateSub(ConstantExpr::getAdd(Op00C, Op1C), X); | ||||
865 | |||||
866 | Value *Y; | ||||
867 | |||||
868 | // add (sub X, Y), -1 --> add (not Y), X | ||||
869 | if (match(Op0, m_OneUse(m_Sub(m_Value(X), m_Value(Y)))) && | ||||
870 | match(Op1, m_AllOnes())) | ||||
871 | return BinaryOperator::CreateAdd(Builder.CreateNot(Y), X); | ||||
872 | |||||
873 | // zext(bool) + C -> bool ? C + 1 : C | ||||
874 | if (match(Op0, m_ZExt(m_Value(X))) && | ||||
875 | X->getType()->getScalarSizeInBits() == 1) | ||||
876 | return SelectInst::Create(X, InstCombiner::AddOne(Op1C), Op1); | ||||
877 | // sext(bool) + C -> bool ? C - 1 : C | ||||
878 | if (match(Op0, m_SExt(m_Value(X))) && | ||||
879 | X->getType()->getScalarSizeInBits() == 1) | ||||
880 | return SelectInst::Create(X, InstCombiner::SubOne(Op1C), Op1); | ||||
881 | |||||
882 | // ~X + C --> (C-1) - X | ||||
883 | if (match(Op0, m_Not(m_Value(X)))) | ||||
884 | return BinaryOperator::CreateSub(InstCombiner::SubOne(Op1C), X); | ||||
885 | |||||
886 | // (iN X s>> (N - 1)) + 1 --> zext (X > -1) | ||||
887 | const APInt *C; | ||||
888 | unsigned BitWidth = Ty->getScalarSizeInBits(); | ||||
889 | if (match(Op0, m_OneUse(m_AShr(m_Value(X), | ||||
890 | m_SpecificIntAllowUndef(BitWidth - 1)))) && | ||||
891 | match(Op1, m_One())) | ||||
892 | return new ZExtInst(Builder.CreateIsNotNeg(X, "isnotneg"), Ty); | ||||
893 | |||||
894 | if (!match(Op1, m_APInt(C))) | ||||
895 | return nullptr; | ||||
896 | |||||
897 | // (X | Op01C) + Op1C --> X + (Op01C + Op1C) iff the `or` is actually an `add` | ||||
898 | Constant *Op01C; | ||||
899 | if (match(Op0, m_Or(m_Value(X), m_ImmConstant(Op01C))) && | ||||
900 | haveNoCommonBitsSet(X, Op01C, DL, &AC, &Add, &DT)) | ||||
901 | return BinaryOperator::CreateAdd(X, ConstantExpr::getAdd(Op01C, Op1C)); | ||||
902 | |||||
903 | // (X | C2) + C --> (X | C2) ^ C2 iff (C2 == -C) | ||||
904 | const APInt *C2; | ||||
905 | if (match(Op0, m_Or(m_Value(), m_APInt(C2))) && *C2 == -*C) | ||||
906 | return BinaryOperator::CreateXor(Op0, ConstantInt::get(Add.getType(), *C2)); | ||||
907 | |||||
908 | if (C->isSignMask()) { | ||||
909 | // If wrapping is not allowed, then the addition must set the sign bit: | ||||
910 | // X + (signmask) --> X | signmask | ||||
911 | if (Add.hasNoSignedWrap() || Add.hasNoUnsignedWrap()) | ||||
912 | return BinaryOperator::CreateOr(Op0, Op1); | ||||
913 | |||||
914 | // If wrapping is allowed, then the addition flips the sign bit of LHS: | ||||
915 | // X + (signmask) --> X ^ signmask | ||||
916 | return BinaryOperator::CreateXor(Op0, Op1); | ||||
917 | } | ||||
918 | |||||
919 | // Is this add the last step in a convoluted sext? | ||||
920 | // add(zext(xor i16 X, -32768), -32768) --> sext X | ||||
921 | if (match(Op0, m_ZExt(m_Xor(m_Value(X), m_APInt(C2)))) && | ||||
922 | C2->isMinSignedValue() && C2->sext(Ty->getScalarSizeInBits()) == *C) | ||||
923 | return CastInst::Create(Instruction::SExt, X, Ty); | ||||
924 | |||||
925 | if (match(Op0, m_Xor(m_Value(X), m_APInt(C2)))) { | ||||
926 | // (X ^ signmask) + C --> (X + (signmask ^ C)) | ||||
927 | if (C2->isSignMask()) | ||||
928 | return BinaryOperator::CreateAdd(X, ConstantInt::get(Ty, *C2 ^ *C)); | ||||
929 | |||||
930 | // If X has no high-bits set above an xor mask: | ||||
931 | // add (xor X, LowMaskC), C --> sub (LowMaskC + C), X | ||||
932 | if (C2->isMask()) { | ||||
933 | KnownBits LHSKnown = computeKnownBits(X, 0, &Add); | ||||
934 | if ((*C2 | LHSKnown.Zero).isAllOnes()) | ||||
935 | return BinaryOperator::CreateSub(ConstantInt::get(Ty, *C2 + *C), X); | ||||
936 | } | ||||
937 | |||||
938 | // Look for a math+logic pattern that corresponds to sext-in-register of a | ||||
939 | // value with cleared high bits. Convert that into a pair of shifts: | ||||
940 | // add (xor X, 0x80), 0xF..F80 --> (X << ShAmtC) >>s ShAmtC | ||||
941 | // add (xor X, 0xF..F80), 0x80 --> (X << ShAmtC) >>s ShAmtC | ||||
942 | if (Op0->hasOneUse() && *C2 == -(*C)) { | ||||
943 | unsigned BitWidth = Ty->getScalarSizeInBits(); | ||||
944 | unsigned ShAmt = 0; | ||||
945 | if (C->isPowerOf2()) | ||||
946 | ShAmt = BitWidth - C->logBase2() - 1; | ||||
947 | else if (C2->isPowerOf2()) | ||||
948 | ShAmt = BitWidth - C2->logBase2() - 1; | ||||
949 | if (ShAmt && MaskedValueIsZero(X, APInt::getHighBitsSet(BitWidth, ShAmt), | ||||
950 | 0, &Add)) { | ||||
951 | Constant *ShAmtC = ConstantInt::get(Ty, ShAmt); | ||||
952 | Value *NewShl = Builder.CreateShl(X, ShAmtC, "sext"); | ||||
953 | return BinaryOperator::CreateAShr(NewShl, ShAmtC); | ||||
954 | } | ||||
955 | } | ||||
956 | } | ||||
957 | |||||
958 | if (C->isOne() && Op0->hasOneUse()) { | ||||
959 | // add (sext i1 X), 1 --> zext (not X) | ||||
960 | // TODO: The smallest IR representation is (select X, 0, 1), and that would | ||||
961 | // not require the one-use check. But we need to remove a transform in | ||||
962 | // visitSelect and make sure that IR value tracking for select is equal or | ||||
963 | // better than for these ops. | ||||
964 | if (match(Op0, m_SExt(m_Value(X))) && | ||||
965 | X->getType()->getScalarSizeInBits() == 1) | ||||
966 | return new ZExtInst(Builder.CreateNot(X), Ty); | ||||
967 | |||||
968 | // Shifts and add used to flip and mask off the low bit: | ||||
969 | // add (ashr (shl i32 X, 31), 31), 1 --> and (not X), 1 | ||||
970 | const APInt *C3; | ||||
971 | if (match(Op0, m_AShr(m_Shl(m_Value(X), m_APInt(C2)), m_APInt(C3))) && | ||||
972 | C2 == C3 && *C2 == Ty->getScalarSizeInBits() - 1) { | ||||
973 | Value *NotX = Builder.CreateNot(X); | ||||
974 | return BinaryOperator::CreateAnd(NotX, ConstantInt::get(Ty, 1)); | ||||
975 | } | ||||
976 | } | ||||
977 | |||||
978 | // Fold (add (zext (add X, -1)), 1) -> (zext X) if X is non-zero. | ||||
979 | // TODO: There's a general form for any constant on the outer add. | ||||
980 | if (C->isOne()) { | ||||
981 | if (match(Op0, m_ZExt(m_Add(m_Value(X), m_AllOnes())))) { | ||||
982 | const SimplifyQuery Q = SQ.getWithInstruction(&Add); | ||||
983 | if (llvm::isKnownNonZero(X, DL, 0, Q.AC, Q.CxtI, Q.DT)) | ||||
984 | return new ZExtInst(X, Ty); | ||||
985 | } | ||||
986 | } | ||||
987 | |||||
988 | return nullptr; | ||||
989 | } | ||||
990 | |||||
991 | // Matches multiplication expression Op * C where C is a constant. Returns the | ||||
992 | // constant value in C and the other operand in Op. Returns true if such a | ||||
993 | // match is found. | ||||
994 | static bool MatchMul(Value *E, Value *&Op, APInt &C) { | ||||
995 | const APInt *AI; | ||||
996 | if (match(E, m_Mul(m_Value(Op), m_APInt(AI)))) { | ||||
997 | C = *AI; | ||||
998 | return true; | ||||
999 | } | ||||
1000 | if (match(E, m_Shl(m_Value(Op), m_APInt(AI)))) { | ||||
1001 | C = APInt(AI->getBitWidth(), 1); | ||||
1002 | C <<= *AI; | ||||
1003 | return true; | ||||
1004 | } | ||||
1005 | return false; | ||||
1006 | } | ||||
1007 | |||||
1008 | // Matches remainder expression Op % C where C is a constant. Returns the | ||||
1009 | // constant value in C and the other operand in Op. Returns the signedness of | ||||
1010 | // the remainder operation in IsSigned. Returns true if such a match is | ||||
1011 | // found. | ||||
1012 | static bool MatchRem(Value *E, Value *&Op, APInt &C, bool &IsSigned) { | ||||
1013 | const APInt *AI; | ||||
1014 | IsSigned = false; | ||||
1015 | if (match(E, m_SRem(m_Value(Op), m_APInt(AI)))) { | ||||
1016 | IsSigned = true; | ||||
1017 | C = *AI; | ||||
1018 | return true; | ||||
1019 | } | ||||
1020 | if (match(E, m_URem(m_Value(Op), m_APInt(AI)))) { | ||||
1021 | C = *AI; | ||||
1022 | return true; | ||||
1023 | } | ||||
1024 | if (match(E, m_And(m_Value(Op), m_APInt(AI))) && (*AI + 1).isPowerOf2()) { | ||||
1025 | C = *AI + 1; | ||||
1026 | return true; | ||||
1027 | } | ||||
1028 | return false; | ||||
1029 | } | ||||
1030 | |||||
1031 | // Matches division expression Op / C with the given signedness as indicated | ||||
1032 | // by IsSigned, where C is a constant. Returns the constant value in C and the | ||||
1033 | // other operand in Op. Returns true if such a match is found. | ||||
1034 | static bool MatchDiv(Value *E, Value *&Op, APInt &C, bool IsSigned) { | ||||
1035 | const APInt *AI; | ||||
1036 | if (IsSigned && match(E, m_SDiv(m_Value(Op), m_APInt(AI)))) { | ||||
1037 | C = *AI; | ||||
1038 | return true; | ||||
1039 | } | ||||
1040 | if (!IsSigned) { | ||||
1041 | if (match(E, m_UDiv(m_Value(Op), m_APInt(AI)))) { | ||||
1042 | C = *AI; | ||||
1043 | return true; | ||||
1044 | } | ||||
1045 | if (match(E, m_LShr(m_Value(Op), m_APInt(AI)))) { | ||||
1046 | C = APInt(AI->getBitWidth(), 1); | ||||
1047 | C <<= *AI; | ||||
1048 | return true; | ||||
1049 | } | ||||
1050 | } | ||||
1051 | return false; | ||||
1052 | } | ||||
1053 | |||||
1054 | // Returns whether C0 * C1 with the given signedness overflows. | ||||
1055 | static bool MulWillOverflow(APInt &C0, APInt &C1, bool IsSigned) { | ||||
1056 | bool overflow; | ||||
1057 | if (IsSigned) | ||||
1058 | (void)C0.smul_ov(C1, overflow); | ||||
1059 | else | ||||
1060 | (void)C0.umul_ov(C1, overflow); | ||||
1061 | return overflow; | ||||
1062 | } | ||||
1063 | |||||
1064 | // Simplifies X % C0 + (( X / C0 ) % C1) * C0 to X % (C0 * C1), where (C0 * C1) | ||||
1065 | // does not overflow. | ||||
1066 | Value *InstCombinerImpl::SimplifyAddWithRemainder(BinaryOperator &I) { | ||||
1067 | Value *LHS = I.getOperand(0), *RHS = I.getOperand(1); | ||||
1068 | Value *X, *MulOpV; | ||||
1069 | APInt C0, MulOpC; | ||||
1070 | bool IsSigned; | ||||
1071 | // Match I = X % C0 + MulOpV * C0 | ||||
1072 | if (((MatchRem(LHS, X, C0, IsSigned) && MatchMul(RHS, MulOpV, MulOpC)) || | ||||
1073 | (MatchRem(RHS, X, C0, IsSigned) && MatchMul(LHS, MulOpV, MulOpC))) && | ||||
1074 | C0 == MulOpC) { | ||||
1075 | Value *RemOpV; | ||||
1076 | APInt C1; | ||||
1077 | bool Rem2IsSigned; | ||||
1078 | // Match MulOpC = RemOpV % C1 | ||||
1079 | if (MatchRem(MulOpV, RemOpV, C1, Rem2IsSigned) && | ||||
1080 | IsSigned == Rem2IsSigned) { | ||||
1081 | Value *DivOpV; | ||||
1082 | APInt DivOpC; | ||||
1083 | // Match RemOpV = X / C0 | ||||
1084 | if (MatchDiv(RemOpV, DivOpV, DivOpC, IsSigned) && X == DivOpV && | ||||
1085 | C0 == DivOpC && !MulWillOverflow(C0, C1, IsSigned)) { | ||||
1086 | Value *NewDivisor = ConstantInt::get(X->getType(), C0 * C1); | ||||
1087 | return IsSigned ? Builder.CreateSRem(X, NewDivisor, "srem") | ||||
1088 | : Builder.CreateURem(X, NewDivisor, "urem"); | ||||
1089 | } | ||||
1090 | } | ||||
1091 | } | ||||
1092 | |||||
1093 | return nullptr; | ||||
1094 | } | ||||
1095 | |||||
1096 | /// Fold | ||||
1097 | /// (1 << NBits) - 1 | ||||
1098 | /// Into: | ||||
1099 | /// ~(-(1 << NBits)) | ||||
1100 | /// Because a 'not' is better for bit-tracking analysis and other transforms | ||||
1101 | /// than an 'add'. The new shl is always nsw, and is nuw if old `and` was. | ||||
1102 | static Instruction *canonicalizeLowbitMask(BinaryOperator &I, | ||||
1103 | InstCombiner::BuilderTy &Builder) { | ||||
1104 | Value *NBits; | ||||
1105 | if (!match(&I, m_Add(m_OneUse(m_Shl(m_One(), m_Value(NBits))), m_AllOnes()))) | ||||
1106 | return nullptr; | ||||
1107 | |||||
1108 | Constant *MinusOne = Constant::getAllOnesValue(NBits->getType()); | ||||
1109 | Value *NotMask = Builder.CreateShl(MinusOne, NBits, "notmask"); | ||||
1110 | // Be wary of constant folding. | ||||
1111 | if (auto *BOp = dyn_cast<BinaryOperator>(NotMask)) { | ||||
1112 | // Always NSW. But NUW propagates from `add`. | ||||
1113 | BOp->setHasNoSignedWrap(); | ||||
1114 | BOp->setHasNoUnsignedWrap(I.hasNoUnsignedWrap()); | ||||
1115 | } | ||||
1116 | |||||
1117 | return BinaryOperator::CreateNot(NotMask, I.getName()); | ||||
1118 | } | ||||
1119 | |||||
1120 | static Instruction *foldToUnsignedSaturatedAdd(BinaryOperator &I) { | ||||
1121 | assert(I.getOpcode() == Instruction::Add && "Expecting add instruction")(static_cast <bool> (I.getOpcode() == Instruction::Add && "Expecting add instruction") ? void (0) : __assert_fail ("I.getOpcode() == Instruction::Add && \"Expecting add instruction\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1121 , __extension__ __PRETTY_FUNCTION__)); | ||||
1122 | Type *Ty = I.getType(); | ||||
1123 | auto getUAddSat = [&]() { | ||||
1124 | return Intrinsic::getDeclaration(I.getModule(), Intrinsic::uadd_sat, Ty); | ||||
1125 | }; | ||||
1126 | |||||
1127 | // add (umin X, ~Y), Y --> uaddsat X, Y | ||||
1128 | Value *X, *Y; | ||||
1129 | if (match(&I, m_c_Add(m_c_UMin(m_Value(X), m_Not(m_Value(Y))), | ||||
1130 | m_Deferred(Y)))) | ||||
1131 | return CallInst::Create(getUAddSat(), { X, Y }); | ||||
1132 | |||||
1133 | // add (umin X, ~C), C --> uaddsat X, C | ||||
1134 | const APInt *C, *NotC; | ||||
1135 | if (match(&I, m_Add(m_UMin(m_Value(X), m_APInt(NotC)), m_APInt(C))) && | ||||
1136 | *C == ~*NotC) | ||||
1137 | return CallInst::Create(getUAddSat(), { X, ConstantInt::get(Ty, *C) }); | ||||
1138 | |||||
1139 | return nullptr; | ||||
1140 | } | ||||
1141 | |||||
1142 | /// Try to reduce signed division by power-of-2 to an arithmetic shift right. | ||||
1143 | static Instruction *foldAddToAshr(BinaryOperator &Add) { | ||||
1144 | // Division must be by power-of-2, but not the minimum signed value. | ||||
1145 | Value *X; | ||||
1146 | const APInt *DivC; | ||||
1147 | if (!match(Add.getOperand(0), m_SDiv(m_Value(X), m_Power2(DivC))) || | ||||
1148 | DivC->isNegative()) | ||||
1149 | return nullptr; | ||||
1150 | |||||
1151 | // Rounding is done by adding -1 if the dividend (X) is negative and has any | ||||
1152 | // low bits set. The canonical pattern for that is an "ugt" compare with SMIN: | ||||
1153 | // sext (icmp ugt (X & (DivC - 1)), SMIN) | ||||
1154 | const APInt *MaskC; | ||||
1155 | ICmpInst::Predicate Pred; | ||||
1156 | if (!match(Add.getOperand(1), | ||||
1157 | m_SExt(m_ICmp(Pred, m_And(m_Specific(X), m_APInt(MaskC)), | ||||
1158 | m_SignMask()))) || | ||||
1159 | Pred != ICmpInst::ICMP_UGT) | ||||
1160 | return nullptr; | ||||
1161 | |||||
1162 | APInt SMin = APInt::getSignedMinValue(Add.getType()->getScalarSizeInBits()); | ||||
1163 | if (*MaskC != (SMin | (*DivC - 1))) | ||||
1164 | return nullptr; | ||||
1165 | |||||
1166 | // (X / DivC) + sext ((X & (SMin | (DivC - 1)) >u SMin) --> X >>s log2(DivC) | ||||
1167 | return BinaryOperator::CreateAShr( | ||||
1168 | X, ConstantInt::get(Add.getType(), DivC->exactLogBase2())); | ||||
1169 | } | ||||
1170 | |||||
1171 | Instruction *InstCombinerImpl:: | ||||
1172 | canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract( | ||||
1173 | BinaryOperator &I) { | ||||
1174 | assert((I.getOpcode() == Instruction::Add ||(static_cast <bool> ((I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Or || I.getOpcode() == Instruction ::Sub) && "Expecting add/or/sub instruction") ? void ( 0) : __assert_fail ("(I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Or || I.getOpcode() == Instruction::Sub) && \"Expecting add/or/sub instruction\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1177 , __extension__ __PRETTY_FUNCTION__)) | ||||
| |||||
1175 | I.getOpcode() == Instruction::Or ||(static_cast <bool> ((I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Or || I.getOpcode() == Instruction ::Sub) && "Expecting add/or/sub instruction") ? void ( 0) : __assert_fail ("(I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Or || I.getOpcode() == Instruction::Sub) && \"Expecting add/or/sub instruction\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1177 , __extension__ __PRETTY_FUNCTION__)) | ||||
1176 | I.getOpcode() == Instruction::Sub) &&(static_cast <bool> ((I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Or || I.getOpcode() == Instruction ::Sub) && "Expecting add/or/sub instruction") ? void ( 0) : __assert_fail ("(I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Or || I.getOpcode() == Instruction::Sub) && \"Expecting add/or/sub instruction\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1177 , __extension__ __PRETTY_FUNCTION__)) | ||||
1177 | "Expecting add/or/sub instruction")(static_cast <bool> ((I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Or || I.getOpcode() == Instruction ::Sub) && "Expecting add/or/sub instruction") ? void ( 0) : __assert_fail ("(I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Or || I.getOpcode() == Instruction::Sub) && \"Expecting add/or/sub instruction\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1177 , __extension__ __PRETTY_FUNCTION__)); | ||||
1178 | |||||
1179 | // We have a subtraction/addition between a (potentially truncated) *logical* | ||||
1180 | // right-shift of X and a "select". | ||||
1181 | Value *X, *Select; | ||||
1182 | Instruction *LowBitsToSkip, *Extract; | ||||
1183 | if (!match(&I, m_c_BinOp(m_TruncOrSelf(m_CombineAnd( | ||||
| |||||
| |||||
| |||||
| |||||
1184 | m_LShr(m_Value(X), m_Instruction(LowBitsToSkip)), | ||||
1185 | m_Instruction(Extract))), | ||||
1186 | m_Value(Select)))) | ||||
1187 | return nullptr; | ||||
1188 | |||||
1189 | // `add`/`or` is commutative; but for `sub`, "select" *must* be on RHS. | ||||
1190 | if (I.getOpcode() == Instruction::Sub && I.getOperand(1) != Select) | ||||
1191 | return nullptr; | ||||
1192 | |||||
1193 | Type *XTy = X->getType(); | ||||
1194 | bool HadTrunc = I.getType() != XTy; | ||||
1195 | |||||
1196 | // If there was a truncation of extracted value, then we'll need to produce | ||||
1197 | // one extra instruction, so we need to ensure one instruction will go away. | ||||
1198 | if (HadTrunc
| ||||
1199 | return nullptr; | ||||
1200 | |||||
1201 | // Extraction should extract high NBits bits, with shift amount calculated as: | ||||
1202 | // low bits to skip = shift bitwidth - high bits to extract | ||||
1203 | // The shift amount itself may be extended, and we need to look past zero-ext | ||||
1204 | // when matching NBits, that will matter for matching later. | ||||
1205 | Constant *C; | ||||
1206 | Value *NBits; | ||||
1207 | if (!match( | ||||
1208 | LowBitsToSkip, | ||||
1209 | m_ZExtOrSelf(m_Sub(m_Constant(C), m_ZExtOrSelf(m_Value(NBits))))) || | ||||
1210 | !match(C, m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_EQ, | ||||
1211 | APInt(C->getType()->getScalarSizeInBits(), | ||||
1212 | X->getType()->getScalarSizeInBits())))) | ||||
1213 | return nullptr; | ||||
1214 | |||||
1215 | // Sign-extending value can be zero-extended if we `sub`tract it, | ||||
1216 | // or sign-extended otherwise. | ||||
1217 | auto SkipExtInMagic = [&I](Value *&V) { | ||||
1218 | if (I.getOpcode() == Instruction::Sub) | ||||
1219 | match(V, m_ZExtOrSelf(m_Value(V))); | ||||
1220 | else | ||||
1221 | match(V, m_SExtOrSelf(m_Value(V))); | ||||
1222 | }; | ||||
1223 | |||||
1224 | // Now, finally validate the sign-extending magic. | ||||
1225 | // `select` itself may be appropriately extended, look past that. | ||||
1226 | SkipExtInMagic(Select); | ||||
1227 | |||||
1228 | ICmpInst::Predicate Pred; | ||||
1229 | const APInt *Thr; | ||||
1230 | Value *SignExtendingValue, *Zero; | ||||
1231 | bool ShouldSignext; | ||||
1232 | // It must be a select between two values we will later establish to be a | ||||
1233 | // sign-extending value and a zero constant. The condition guarding the | ||||
1234 | // sign-extension must be based on a sign bit of the same X we had in `lshr`. | ||||
1235 | if (!match(Select, m_Select(m_ICmp(Pred, m_Specific(X), m_APInt(Thr)), | ||||
| |||||
1236 | m_Value(SignExtendingValue), m_Value(Zero))) || | ||||
1237 | !isSignBitCheck(Pred, *Thr, ShouldSignext)) | ||||
1238 | return nullptr; | ||||
1239 | |||||
1240 | // icmp-select pair is commutative. | ||||
1241 | if (!ShouldSignext) | ||||
1242 | std::swap(SignExtendingValue, Zero); | ||||
1243 | |||||
1244 | // If we should not perform sign-extension then we must add/or/subtract zero. | ||||
1245 | if (!match(Zero, m_Zero())) | ||||
1246 | return nullptr; | ||||
1247 | // Otherwise, it should be some constant, left-shifted by the same NBits we | ||||
1248 | // had in `lshr`. Said left-shift can also be appropriately extended. | ||||
1249 | // Again, we must look past zero-ext when looking for NBits. | ||||
1250 | SkipExtInMagic(SignExtendingValue); | ||||
1251 | Constant *SignExtendingValueBaseConstant; | ||||
1252 | if (!match(SignExtendingValue, | ||||
1253 | m_Shl(m_Constant(SignExtendingValueBaseConstant), | ||||
1254 | m_ZExtOrSelf(m_Specific(NBits))))) | ||||
1255 | return nullptr; | ||||
1256 | // If we `sub`, then the constant should be one, else it should be all-ones. | ||||
1257 | if (I.getOpcode() == Instruction::Sub | ||||
1258 | ? !match(SignExtendingValueBaseConstant, m_One()) | ||||
1259 | : !match(SignExtendingValueBaseConstant, m_AllOnes())) | ||||
1260 | return nullptr; | ||||
1261 | |||||
1262 | auto *NewAShr = BinaryOperator::CreateAShr(X, LowBitsToSkip, | ||||
1263 | Extract->getName() + ".sext"); | ||||
1264 | NewAShr->copyIRFlags(Extract); // Preserve `exact`-ness. | ||||
1265 | if (!HadTrunc) | ||||
1266 | return NewAShr; | ||||
1267 | |||||
1268 | Builder.Insert(NewAShr); | ||||
1269 | return TruncInst::CreateTruncOrBitCast(NewAShr, I.getType()); | ||||
1270 | } | ||||
1271 | |||||
1272 | /// This is a specialization of a more general transform from | ||||
1273 | /// foldUsingDistributiveLaws. If that code can be made to work optimally | ||||
1274 | /// for multi-use cases or propagating nsw/nuw, then we would not need this. | ||||
1275 | static Instruction *factorizeMathWithShlOps(BinaryOperator &I, | ||||
1276 | InstCombiner::BuilderTy &Builder) { | ||||
1277 | // TODO: Also handle mul by doubling the shift amount? | ||||
1278 | assert((I.getOpcode() == Instruction::Add ||(static_cast <bool> ((I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Sub) && "Expected add/sub" ) ? void (0) : __assert_fail ("(I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Sub) && \"Expected add/sub\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1280 , __extension__ __PRETTY_FUNCTION__)) | ||||
1279 | I.getOpcode() == Instruction::Sub) &&(static_cast <bool> ((I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Sub) && "Expected add/sub" ) ? void (0) : __assert_fail ("(I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Sub) && \"Expected add/sub\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1280 , __extension__ __PRETTY_FUNCTION__)) | ||||
1280 | "Expected add/sub")(static_cast <bool> ((I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Sub) && "Expected add/sub" ) ? void (0) : __assert_fail ("(I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Sub) && \"Expected add/sub\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1280 , __extension__ __PRETTY_FUNCTION__)); | ||||
1281 | auto *Op0 = dyn_cast<BinaryOperator>(I.getOperand(0)); | ||||
1282 | auto *Op1 = dyn_cast<BinaryOperator>(I.getOperand(1)); | ||||
1283 | if (!Op0 || !Op1 || !(Op0->hasOneUse() || Op1->hasOneUse())) | ||||
1284 | return nullptr; | ||||
1285 | |||||
1286 | Value *X, *Y, *ShAmt; | ||||
1287 | if (!match(Op0, m_Shl(m_Value(X), m_Value(ShAmt))) || | ||||
1288 | !match(Op1, m_Shl(m_Value(Y), m_Specific(ShAmt)))) | ||||
1289 | return nullptr; | ||||
1290 | |||||
1291 | // No-wrap propagates only when all ops have no-wrap. | ||||
1292 | bool HasNSW = I.hasNoSignedWrap() && Op0->hasNoSignedWrap() && | ||||
1293 | Op1->hasNoSignedWrap(); | ||||
1294 | bool HasNUW = I.hasNoUnsignedWrap() && Op0->hasNoUnsignedWrap() && | ||||
1295 | Op1->hasNoUnsignedWrap(); | ||||
1296 | |||||
1297 | // add/sub (X << ShAmt), (Y << ShAmt) --> (add/sub X, Y) << ShAmt | ||||
1298 | Value *NewMath = Builder.CreateBinOp(I.getOpcode(), X, Y); | ||||
1299 | if (auto *NewI = dyn_cast<BinaryOperator>(NewMath)) { | ||||
1300 | NewI->setHasNoSignedWrap(HasNSW); | ||||
1301 | NewI->setHasNoUnsignedWrap(HasNUW); | ||||
1302 | } | ||||
1303 | auto *NewShl = BinaryOperator::CreateShl(NewMath, ShAmt); | ||||
1304 | NewShl->setHasNoSignedWrap(HasNSW); | ||||
1305 | NewShl->setHasNoUnsignedWrap(HasNUW); | ||||
1306 | return NewShl; | ||||
1307 | } | ||||
1308 | |||||
1309 | /// Reduce a sequence of masked half-width multiplies to a single multiply. | ||||
1310 | /// ((XLow * YHigh) + (YLow * XHigh)) << HalfBits) + (XLow * YLow) --> X * Y | ||||
1311 | static Instruction *foldBoxMultiply(BinaryOperator &I) { | ||||
1312 | unsigned BitWidth = I.getType()->getScalarSizeInBits(); | ||||
1313 | // Skip the odd bitwidth types. | ||||
1314 | if ((BitWidth & 0x1)) | ||||
1315 | return nullptr; | ||||
1316 | |||||
1317 | unsigned HalfBits = BitWidth >> 1; | ||||
1318 | APInt HalfMask = APInt::getMaxValue(HalfBits); | ||||
1319 | |||||
1320 | // ResLo = (CrossSum << HalfBits) + (YLo * XLo) | ||||
1321 | Value *XLo, *YLo; | ||||
1322 | Value *CrossSum; | ||||
1323 | if (!match(&I, m_c_Add(m_Shl(m_Value(CrossSum), m_SpecificInt(HalfBits)), | ||||
1324 | m_Mul(m_Value(YLo), m_Value(XLo))))) | ||||
1325 | return nullptr; | ||||
1326 | |||||
1327 | // XLo = X & HalfMask | ||||
1328 | // YLo = Y & HalfMask | ||||
1329 | // TODO: Refactor with SimplifyDemandedBits or KnownBits known leading zeros | ||||
1330 | // to enhance robustness | ||||
1331 | Value *X, *Y; | ||||
1332 | if (!match(XLo, m_And(m_Value(X), m_SpecificInt(HalfMask))) || | ||||
1333 | !match(YLo, m_And(m_Value(Y), m_SpecificInt(HalfMask)))) | ||||
1334 | return nullptr; | ||||
1335 | |||||
1336 | // CrossSum = (X' * (Y >> Halfbits)) + (Y' * (X >> HalfBits)) | ||||
1337 | // X' can be either X or XLo in the pattern (and the same for Y') | ||||
1338 | if (match(CrossSum, | ||||
1339 | m_c_Add(m_c_Mul(m_LShr(m_Specific(Y), m_SpecificInt(HalfBits)), | ||||
1340 | m_CombineOr(m_Specific(X), m_Specific(XLo))), | ||||
1341 | m_c_Mul(m_LShr(m_Specific(X), m_SpecificInt(HalfBits)), | ||||
1342 | m_CombineOr(m_Specific(Y), m_Specific(YLo)))))) | ||||
1343 | return BinaryOperator::CreateMul(X, Y); | ||||
1344 | |||||
1345 | return nullptr; | ||||
1346 | } | ||||
1347 | |||||
1348 | Instruction *InstCombinerImpl::visitAdd(BinaryOperator &I) { | ||||
1349 | if (Value *V = simplifyAddInst(I.getOperand(0), I.getOperand(1), | ||||
1350 | I.hasNoSignedWrap(), I.hasNoUnsignedWrap(), | ||||
1351 | SQ.getWithInstruction(&I))) | ||||
1352 | return replaceInstUsesWith(I, V); | ||||
1353 | |||||
1354 | if (SimplifyAssociativeOrCommutative(I)) | ||||
1355 | return &I; | ||||
1356 | |||||
1357 | if (Instruction *X = foldVectorBinop(I)) | ||||
1358 | return X; | ||||
1359 | |||||
1360 | if (Instruction *Phi = foldBinopWithPhiOperands(I)) | ||||
1361 | return Phi; | ||||
1362 | |||||
1363 | // (A*B)+(A*C) -> A*(B+C) etc | ||||
1364 | if (Value *V = foldUsingDistributiveLaws(I)) | ||||
1365 | return replaceInstUsesWith(I, V); | ||||
1366 | |||||
1367 | if (Instruction *R = foldBoxMultiply(I)) | ||||
1368 | return R; | ||||
1369 | |||||
1370 | if (Instruction *R = factorizeMathWithShlOps(I, Builder)) | ||||
1371 | return R; | ||||
1372 | |||||
1373 | if (Instruction *X = foldAddWithConstant(I)) | ||||
1374 | return X; | ||||
1375 | |||||
1376 | if (Instruction *X = foldNoWrapAdd(I, Builder)) | ||||
1377 | return X; | ||||
1378 | |||||
1379 | Value *LHS = I.getOperand(0), *RHS = I.getOperand(1); | ||||
1380 | Type *Ty = I.getType(); | ||||
1381 | if (Ty->isIntOrIntVectorTy(1)) | ||||
1382 | return BinaryOperator::CreateXor(LHS, RHS); | ||||
1383 | |||||
1384 | // X + X --> X << 1 | ||||
1385 | if (LHS == RHS) { | ||||
1386 | auto *Shl = BinaryOperator::CreateShl(LHS, ConstantInt::get(Ty, 1)); | ||||
1387 | Shl->setHasNoSignedWrap(I.hasNoSignedWrap()); | ||||
1388 | Shl->setHasNoUnsignedWrap(I.hasNoUnsignedWrap()); | ||||
1389 | return Shl; | ||||
1390 | } | ||||
1391 | |||||
1392 | Value *A, *B; | ||||
1393 | if (match(LHS, m_Neg(m_Value(A)))) { | ||||
1394 | // -A + -B --> -(A + B) | ||||
1395 | if (match(RHS, m_Neg(m_Value(B)))) | ||||
1396 | return BinaryOperator::CreateNeg(Builder.CreateAdd(A, B)); | ||||
1397 | |||||
1398 | // -A + B --> B - A | ||||
1399 | return BinaryOperator::CreateSub(RHS, A); | ||||
1400 | } | ||||
1401 | |||||
1402 | // A + -B --> A - B | ||||
1403 | if (match(RHS, m_Neg(m_Value(B)))) | ||||
1404 | return BinaryOperator::CreateSub(LHS, B); | ||||
1405 | |||||
1406 | if (Value *V = checkForNegativeOperand(I, Builder)) | ||||
1407 | return replaceInstUsesWith(I, V); | ||||
1408 | |||||
1409 | // (A + 1) + ~B --> A - B | ||||
1410 | // ~B + (A + 1) --> A - B | ||||
1411 | // (~B + A) + 1 --> A - B | ||||
1412 | // (A + ~B) + 1 --> A - B | ||||
1413 | if (match(&I, m_c_BinOp(m_Add(m_Value(A), m_One()), m_Not(m_Value(B)))) || | ||||
1414 | match(&I, m_BinOp(m_c_Add(m_Not(m_Value(B)), m_Value(A)), m_One()))) | ||||
1415 | return BinaryOperator::CreateSub(A, B); | ||||
1416 | |||||
1417 | // (A + RHS) + RHS --> A + (RHS << 1) | ||||
1418 | if (match(LHS, m_OneUse(m_c_Add(m_Value(A), m_Specific(RHS))))) | ||||
1419 | return BinaryOperator::CreateAdd(A, Builder.CreateShl(RHS, 1, "reass.add")); | ||||
1420 | |||||
1421 | // LHS + (A + LHS) --> A + (LHS << 1) | ||||
1422 | if (match(RHS, m_OneUse(m_c_Add(m_Value(A), m_Specific(LHS))))) | ||||
1423 | return BinaryOperator::CreateAdd(A, Builder.CreateShl(LHS, 1, "reass.add")); | ||||
1424 | |||||
1425 | { | ||||
1426 | // (A + C1) + (C2 - B) --> (A - B) + (C1 + C2) | ||||
1427 | Constant *C1, *C2; | ||||
1428 | if (match(&I, m_c_Add(m_Add(m_Value(A), m_ImmConstant(C1)), | ||||
1429 | m_Sub(m_ImmConstant(C2), m_Value(B)))) && | ||||
1430 | (LHS->hasOneUse() || RHS->hasOneUse())) { | ||||
1431 | Value *Sub = Builder.CreateSub(A, B); | ||||
1432 | return BinaryOperator::CreateAdd(Sub, ConstantExpr::getAdd(C1, C2)); | ||||
1433 | } | ||||
1434 | |||||
1435 | // Canonicalize a constant sub operand as an add operand for better folding: | ||||
1436 | // (C1 - A) + B --> (B - A) + C1 | ||||
1437 | if (match(&I, m_c_Add(m_OneUse(m_Sub(m_ImmConstant(C1), m_Value(A))), | ||||
1438 | m_Value(B)))) { | ||||
1439 | Value *Sub = Builder.CreateSub(B, A, "reass.sub"); | ||||
1440 | return BinaryOperator::CreateAdd(Sub, C1); | ||||
1441 | } | ||||
1442 | } | ||||
1443 | |||||
1444 | // X % C0 + (( X / C0 ) % C1) * C0 => X % (C0 * C1) | ||||
1445 | if (Value *V = SimplifyAddWithRemainder(I)) return replaceInstUsesWith(I, V); | ||||
1446 | |||||
1447 | // ((X s/ C1) << C2) + X => X s% -C1 where -C1 is 1 << C2 | ||||
1448 | const APInt *C1, *C2; | ||||
1449 | if (match(LHS, m_Shl(m_SDiv(m_Specific(RHS), m_APInt(C1)), m_APInt(C2)))) { | ||||
1450 | APInt one(C2->getBitWidth(), 1); | ||||
1451 | APInt minusC1 = -(*C1); | ||||
1452 | if (minusC1 == (one << *C2)) { | ||||
1453 | Constant *NewRHS = ConstantInt::get(RHS->getType(), minusC1); | ||||
1454 | return BinaryOperator::CreateSRem(RHS, NewRHS); | ||||
1455 | } | ||||
1456 | } | ||||
1457 | |||||
1458 | // (A & 2^C1) + A => A & (2^C1 - 1) iff bit C1 in A is a sign bit | ||||
1459 | if (match(&I, m_c_Add(m_And(m_Value(A), m_APInt(C1)), m_Deferred(A))) && | ||||
1460 | C1->isPowerOf2() && (ComputeNumSignBits(A) > C1->countl_zero())) { | ||||
1461 | Constant *NewMask = ConstantInt::get(RHS->getType(), *C1 - 1); | ||||
1462 | return BinaryOperator::CreateAnd(A, NewMask); | ||||
1463 | } | ||||
1464 | |||||
1465 | // ZExt (B - A) + ZExt(A) --> ZExt(B) | ||||
1466 | if ((match(RHS, m_ZExt(m_Value(A))) && | ||||
1467 | match(LHS, m_ZExt(m_NUWSub(m_Value(B), m_Specific(A))))) || | ||||
1468 | (match(LHS, m_ZExt(m_Value(A))) && | ||||
1469 | match(RHS, m_ZExt(m_NUWSub(m_Value(B), m_Specific(A)))))) | ||||
1470 | return new ZExtInst(B, LHS->getType()); | ||||
1471 | |||||
1472 | // A+B --> A|B iff A and B have no bits set in common. | ||||
1473 | if (haveNoCommonBitsSet(LHS, RHS, DL, &AC, &I, &DT)) | ||||
1474 | return BinaryOperator::CreateOr(LHS, RHS); | ||||
1475 | |||||
1476 | if (Instruction *Ext = narrowMathIfNoOverflow(I)) | ||||
1477 | return Ext; | ||||
1478 | |||||
1479 | // (add (xor A, B) (and A, B)) --> (or A, B) | ||||
1480 | // (add (and A, B) (xor A, B)) --> (or A, B) | ||||
1481 | if (match(&I, m_c_BinOp(m_Xor(m_Value(A), m_Value(B)), | ||||
1482 | m_c_And(m_Deferred(A), m_Deferred(B))))) | ||||
1483 | return BinaryOperator::CreateOr(A, B); | ||||
1484 | |||||
1485 | // (add (or A, B) (and A, B)) --> (add A, B) | ||||
1486 | // (add (and A, B) (or A, B)) --> (add A, B) | ||||
1487 | if (match(&I, m_c_BinOp(m_Or(m_Value(A), m_Value(B)), | ||||
1488 | m_c_And(m_Deferred(A), m_Deferred(B))))) { | ||||
1489 | // Replacing operands in-place to preserve nuw/nsw flags. | ||||
1490 | replaceOperand(I, 0, A); | ||||
1491 | replaceOperand(I, 1, B); | ||||
1492 | return &I; | ||||
1493 | } | ||||
1494 | |||||
1495 | // (add A (or A, -A)) --> (and (add A, -1) A) | ||||
1496 | // (add A (or -A, A)) --> (and (add A, -1) A) | ||||
1497 | // (add (or A, -A) A) --> (and (add A, -1) A) | ||||
1498 | // (add (or -A, A) A) --> (and (add A, -1) A) | ||||
1499 | if (match(&I, m_c_BinOp(m_Value(A), m_OneUse(m_c_Or(m_Neg(m_Deferred(A)), | ||||
1500 | m_Deferred(A)))))) { | ||||
1501 | Value *Add = | ||||
1502 | Builder.CreateAdd(A, Constant::getAllOnesValue(A->getType()), "", | ||||
1503 | I.hasNoUnsignedWrap(), I.hasNoSignedWrap()); | ||||
1504 | return BinaryOperator::CreateAnd(Add, A); | ||||
1505 | } | ||||
1506 | |||||
1507 | // Canonicalize ((A & -A) - 1) --> ((A - 1) & ~A) | ||||
1508 | // Forms all commutable operations, and simplifies ctpop -> cttz folds. | ||||
1509 | if (match(&I, | ||||
1510 | m_Add(m_OneUse(m_c_And(m_Value(A), m_OneUse(m_Neg(m_Deferred(A))))), | ||||
1511 | m_AllOnes()))) { | ||||
1512 | Constant *AllOnes = ConstantInt::getAllOnesValue(RHS->getType()); | ||||
1513 | Value *Dec = Builder.CreateAdd(A, AllOnes); | ||||
1514 | Value *Not = Builder.CreateXor(A, AllOnes); | ||||
1515 | return BinaryOperator::CreateAnd(Dec, Not); | ||||
1516 | } | ||||
1517 | |||||
1518 | // Disguised reassociation/factorization: | ||||
1519 | // ~(A * C1) + A | ||||
1520 | // ((A * -C1) - 1) + A | ||||
1521 | // ((A * -C1) + A) - 1 | ||||
1522 | // (A * (1 - C1)) - 1 | ||||
1523 | if (match(&I, | ||||
1524 | m_c_Add(m_OneUse(m_Not(m_OneUse(m_Mul(m_Value(A), m_APInt(C1))))), | ||||
1525 | m_Deferred(A)))) { | ||||
1526 | Type *Ty = I.getType(); | ||||
1527 | Constant *NewMulC = ConstantInt::get(Ty, 1 - *C1); | ||||
1528 | Value *NewMul = Builder.CreateMul(A, NewMulC); | ||||
1529 | return BinaryOperator::CreateAdd(NewMul, ConstantInt::getAllOnesValue(Ty)); | ||||
1530 | } | ||||
1531 | |||||
1532 | // (A * -2**C) + B --> B - (A << C) | ||||
1533 | const APInt *NegPow2C; | ||||
1534 | if (match(&I, m_c_Add(m_OneUse(m_Mul(m_Value(A), m_NegatedPower2(NegPow2C))), | ||||
1535 | m_Value(B)))) { | ||||
1536 | Constant *ShiftAmtC = ConstantInt::get(Ty, NegPow2C->countr_zero()); | ||||
1537 | Value *Shl = Builder.CreateShl(A, ShiftAmtC); | ||||
1538 | return BinaryOperator::CreateSub(B, Shl); | ||||
1539 | } | ||||
1540 | |||||
1541 | // Canonicalize signum variant that ends in add: | ||||
1542 | // (A s>> (BW - 1)) + (zext (A s> 0)) --> (A s>> (BW - 1)) | (zext (A != 0)) | ||||
1543 | ICmpInst::Predicate Pred; | ||||
1544 | uint64_t BitWidth = Ty->getScalarSizeInBits(); | ||||
1545 | if (match(LHS, m_AShr(m_Value(A), m_SpecificIntAllowUndef(BitWidth - 1))) && | ||||
1546 | match(RHS, m_OneUse(m_ZExt( | ||||
1547 | m_OneUse(m_ICmp(Pred, m_Specific(A), m_ZeroInt()))))) && | ||||
1548 | Pred == CmpInst::ICMP_SGT) { | ||||
1549 | Value *NotZero = Builder.CreateIsNotNull(A, "isnotnull"); | ||||
1550 | Value *Zext = Builder.CreateZExt(NotZero, Ty, "isnotnull.zext"); | ||||
1551 | return BinaryOperator::CreateOr(LHS, Zext); | ||||
1552 | } | ||||
1553 | |||||
1554 | if (Instruction *Ashr = foldAddToAshr(I)) | ||||
1555 | return Ashr; | ||||
1556 | |||||
1557 | // min(A, B) + max(A, B) => A + B. | ||||
1558 | if (match(&I, m_CombineOr(m_c_Add(m_SMax(m_Value(A), m_Value(B)), | ||||
1559 | m_c_SMin(m_Deferred(A), m_Deferred(B))), | ||||
1560 | m_c_Add(m_UMax(m_Value(A), m_Value(B)), | ||||
1561 | m_c_UMin(m_Deferred(A), m_Deferred(B)))))) | ||||
1562 | return BinaryOperator::CreateWithCopiedFlags(Instruction::Add, A, B, &I); | ||||
1563 | |||||
1564 | // TODO(jingyue): Consider willNotOverflowSignedAdd and | ||||
1565 | // willNotOverflowUnsignedAdd to reduce the number of invocations of | ||||
1566 | // computeKnownBits. | ||||
1567 | bool Changed = false; | ||||
1568 | if (!I.hasNoSignedWrap() && willNotOverflowSignedAdd(LHS, RHS, I)) { | ||||
1569 | Changed = true; | ||||
1570 | I.setHasNoSignedWrap(true); | ||||
1571 | } | ||||
1572 | if (!I.hasNoUnsignedWrap() && willNotOverflowUnsignedAdd(LHS, RHS, I)) { | ||||
1573 | Changed = true; | ||||
1574 | I.setHasNoUnsignedWrap(true); | ||||
1575 | } | ||||
1576 | |||||
1577 | if (Instruction *V = canonicalizeLowbitMask(I, Builder)) | ||||
1578 | return V; | ||||
1579 | |||||
1580 | if (Instruction *V = | ||||
1581 | canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract(I)) | ||||
1582 | return V; | ||||
1583 | |||||
1584 | if (Instruction *SatAdd = foldToUnsignedSaturatedAdd(I)) | ||||
1585 | return SatAdd; | ||||
1586 | |||||
1587 | // usub.sat(A, B) + B => umax(A, B) | ||||
1588 | if (match(&I, m_c_BinOp( | ||||
1589 | m_OneUse(m_Intrinsic<Intrinsic::usub_sat>(m_Value(A), m_Value(B))), | ||||
1590 | m_Deferred(B)))) { | ||||
1591 | return replaceInstUsesWith(I, | ||||
1592 | Builder.CreateIntrinsic(Intrinsic::umax, {I.getType()}, {A, B})); | ||||
1593 | } | ||||
1594 | |||||
1595 | // ctpop(A) + ctpop(B) => ctpop(A | B) if A and B have no bits set in common. | ||||
1596 | if (match(LHS, m_OneUse(m_Intrinsic<Intrinsic::ctpop>(m_Value(A)))) && | ||||
1597 | match(RHS, m_OneUse(m_Intrinsic<Intrinsic::ctpop>(m_Value(B)))) && | ||||
1598 | haveNoCommonBitsSet(A, B, DL, &AC, &I, &DT)) | ||||
1599 | return replaceInstUsesWith( | ||||
1600 | I, Builder.CreateIntrinsic(Intrinsic::ctpop, {I.getType()}, | ||||
1601 | {Builder.CreateOr(A, B)})); | ||||
1602 | |||||
1603 | return Changed ? &I : nullptr; | ||||
1604 | } | ||||
1605 | |||||
1606 | /// Eliminate an op from a linear interpolation (lerp) pattern. | ||||
1607 | static Instruction *factorizeLerp(BinaryOperator &I, | ||||
1608 | InstCombiner::BuilderTy &Builder) { | ||||
1609 | Value *X, *Y, *Z; | ||||
1610 | if (!match(&I, m_c_FAdd(m_OneUse(m_c_FMul(m_Value(Y), | ||||
1611 | m_OneUse(m_FSub(m_FPOne(), | ||||
1612 | m_Value(Z))))), | ||||
1613 | m_OneUse(m_c_FMul(m_Value(X), m_Deferred(Z)))))) | ||||
1614 | return nullptr; | ||||
1615 | |||||
1616 | // (Y * (1.0 - Z)) + (X * Z) --> Y + Z * (X - Y) [8 commuted variants] | ||||
1617 | Value *XY = Builder.CreateFSubFMF(X, Y, &I); | ||||
1618 | Value *MulZ = Builder.CreateFMulFMF(Z, XY, &I); | ||||
1619 | return BinaryOperator::CreateFAddFMF(Y, MulZ, &I); | ||||
1620 | } | ||||
1621 | |||||
1622 | /// Factor a common operand out of fadd/fsub of fmul/fdiv. | ||||
1623 | static Instruction *factorizeFAddFSub(BinaryOperator &I, | ||||
1624 | InstCombiner::BuilderTy &Builder) { | ||||
1625 | assert((I.getOpcode() == Instruction::FAdd ||(static_cast <bool> ((I.getOpcode() == Instruction::FAdd || I.getOpcode() == Instruction::FSub) && "Expecting fadd/fsub" ) ? void (0) : __assert_fail ("(I.getOpcode() == Instruction::FAdd || I.getOpcode() == Instruction::FSub) && \"Expecting fadd/fsub\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1626 , __extension__ __PRETTY_FUNCTION__)) | ||||
1626 | I.getOpcode() == Instruction::FSub) && "Expecting fadd/fsub")(static_cast <bool> ((I.getOpcode() == Instruction::FAdd || I.getOpcode() == Instruction::FSub) && "Expecting fadd/fsub" ) ? void (0) : __assert_fail ("(I.getOpcode() == Instruction::FAdd || I.getOpcode() == Instruction::FSub) && \"Expecting fadd/fsub\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1626 , __extension__ __PRETTY_FUNCTION__)); | ||||
1627 | assert(I.hasAllowReassoc() && I.hasNoSignedZeros() &&(static_cast <bool> (I.hasAllowReassoc() && I.hasNoSignedZeros () && "FP factorization requires FMF") ? void (0) : __assert_fail ("I.hasAllowReassoc() && I.hasNoSignedZeros() && \"FP factorization requires FMF\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1628 , __extension__ __PRETTY_FUNCTION__)) | ||||
1628 | "FP factorization requires FMF")(static_cast <bool> (I.hasAllowReassoc() && I.hasNoSignedZeros () && "FP factorization requires FMF") ? void (0) : __assert_fail ("I.hasAllowReassoc() && I.hasNoSignedZeros() && \"FP factorization requires FMF\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1628 , __extension__ __PRETTY_FUNCTION__)); | ||||
1629 | |||||
1630 | if (Instruction *Lerp = factorizeLerp(I, Builder)) | ||||
1631 | return Lerp; | ||||
1632 | |||||
1633 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||
1634 | if (!Op0->hasOneUse() || !Op1->hasOneUse()) | ||||
1635 | return nullptr; | ||||
1636 | |||||
1637 | Value *X, *Y, *Z; | ||||
1638 | bool IsFMul; | ||||
1639 | if ((match(Op0, m_FMul(m_Value(X), m_Value(Z))) && | ||||
1640 | match(Op1, m_c_FMul(m_Value(Y), m_Specific(Z)))) || | ||||
1641 | (match(Op0, m_FMul(m_Value(Z), m_Value(X))) && | ||||
1642 | match(Op1, m_c_FMul(m_Value(Y), m_Specific(Z))))) | ||||
1643 | IsFMul = true; | ||||
1644 | else if (match(Op0, m_FDiv(m_Value(X), m_Value(Z))) && | ||||
1645 | match(Op1, m_FDiv(m_Value(Y), m_Specific(Z)))) | ||||
1646 | IsFMul = false; | ||||
1647 | else | ||||
1648 | return nullptr; | ||||
1649 | |||||
1650 | // (X * Z) + (Y * Z) --> (X + Y) * Z | ||||
1651 | // (X * Z) - (Y * Z) --> (X - Y) * Z | ||||
1652 | // (X / Z) + (Y / Z) --> (X + Y) / Z | ||||
1653 | // (X / Z) - (Y / Z) --> (X - Y) / Z | ||||
1654 | bool IsFAdd = I.getOpcode() == Instruction::FAdd; | ||||
1655 | Value *XY = IsFAdd ? Builder.CreateFAddFMF(X, Y, &I) | ||||
1656 | : Builder.CreateFSubFMF(X, Y, &I); | ||||
1657 | |||||
1658 | // Bail out if we just created a denormal constant. | ||||
1659 | // TODO: This is copied from a previous implementation. Is it necessary? | ||||
1660 | const APFloat *C; | ||||
1661 | if (match(XY, m_APFloat(C)) && !C->isNormal()) | ||||
1662 | return nullptr; | ||||
1663 | |||||
1664 | return IsFMul ? BinaryOperator::CreateFMulFMF(XY, Z, &I) | ||||
1665 | : BinaryOperator::CreateFDivFMF(XY, Z, &I); | ||||
1666 | } | ||||
1667 | |||||
1668 | Instruction *InstCombinerImpl::visitFAdd(BinaryOperator &I) { | ||||
1669 | if (Value *V = simplifyFAddInst(I.getOperand(0), I.getOperand(1), | ||||
1670 | I.getFastMathFlags(), | ||||
1671 | SQ.getWithInstruction(&I))) | ||||
1672 | return replaceInstUsesWith(I, V); | ||||
1673 | |||||
1674 | if (SimplifyAssociativeOrCommutative(I)) | ||||
1675 | return &I; | ||||
1676 | |||||
1677 | if (Instruction *X = foldVectorBinop(I)) | ||||
1678 | return X; | ||||
1679 | |||||
1680 | if (Instruction *Phi = foldBinopWithPhiOperands(I)) | ||||
1681 | return Phi; | ||||
1682 | |||||
1683 | if (Instruction *FoldedFAdd = foldBinOpIntoSelectOrPhi(I)) | ||||
1684 | return FoldedFAdd; | ||||
1685 | |||||
1686 | // (-X) + Y --> Y - X | ||||
1687 | Value *X, *Y; | ||||
1688 | if (match(&I, m_c_FAdd(m_FNeg(m_Value(X)), m_Value(Y)))) | ||||
1689 | return BinaryOperator::CreateFSubFMF(Y, X, &I); | ||||
1690 | |||||
1691 | // Similar to above, but look through fmul/fdiv for the negated term. | ||||
1692 | // (-X * Y) + Z --> Z - (X * Y) [4 commuted variants] | ||||
1693 | Value *Z; | ||||
1694 | if (match(&I, m_c_FAdd(m_OneUse(m_c_FMul(m_FNeg(m_Value(X)), m_Value(Y))), | ||||
1695 | m_Value(Z)))) { | ||||
1696 | Value *XY = Builder.CreateFMulFMF(X, Y, &I); | ||||
1697 | return BinaryOperator::CreateFSubFMF(Z, XY, &I); | ||||
1698 | } | ||||
1699 | // (-X / Y) + Z --> Z - (X / Y) [2 commuted variants] | ||||
1700 | // (X / -Y) + Z --> Z - (X / Y) [2 commuted variants] | ||||
1701 | if (match(&I, m_c_FAdd(m_OneUse(m_FDiv(m_FNeg(m_Value(X)), m_Value(Y))), | ||||
1702 | m_Value(Z))) || | ||||
1703 | match(&I, m_c_FAdd(m_OneUse(m_FDiv(m_Value(X), m_FNeg(m_Value(Y)))), | ||||
1704 | m_Value(Z)))) { | ||||
1705 | Value *XY = Builder.CreateFDivFMF(X, Y, &I); | ||||
1706 | return BinaryOperator::CreateFSubFMF(Z, XY, &I); | ||||
1707 | } | ||||
1708 | |||||
1709 | // Check for (fadd double (sitofp x), y), see if we can merge this into an | ||||
1710 | // integer add followed by a promotion. | ||||
1711 | Value *LHS = I.getOperand(0), *RHS = I.getOperand(1); | ||||
1712 | if (SIToFPInst *LHSConv = dyn_cast<SIToFPInst>(LHS)) { | ||||
1713 | Value *LHSIntVal = LHSConv->getOperand(0); | ||||
1714 | Type *FPType = LHSConv->getType(); | ||||
1715 | |||||
1716 | // TODO: This check is overly conservative. In many cases known bits | ||||
1717 | // analysis can tell us that the result of the addition has less significant | ||||
1718 | // bits than the integer type can hold. | ||||
1719 | auto IsValidPromotion = [](Type *FTy, Type *ITy) { | ||||
1720 | Type *FScalarTy = FTy->getScalarType(); | ||||
1721 | Type *IScalarTy = ITy->getScalarType(); | ||||
1722 | |||||
1723 | // Do we have enough bits in the significand to represent the result of | ||||
1724 | // the integer addition? | ||||
1725 | unsigned MaxRepresentableBits = | ||||
1726 | APFloat::semanticsPrecision(FScalarTy->getFltSemantics()); | ||||
1727 | return IScalarTy->getIntegerBitWidth() <= MaxRepresentableBits; | ||||
1728 | }; | ||||
1729 | |||||
1730 | // (fadd double (sitofp x), fpcst) --> (sitofp (add int x, intcst)) | ||||
1731 | // ... if the constant fits in the integer value. This is useful for things | ||||
1732 | // like (double)(x & 1234) + 4.0 -> (double)((X & 1234)+4) which no longer | ||||
1733 | // requires a constant pool load, and generally allows the add to be better | ||||
1734 | // instcombined. | ||||
1735 | if (ConstantFP *CFP = dyn_cast<ConstantFP>(RHS)) | ||||
1736 | if (IsValidPromotion(FPType, LHSIntVal->getType())) { | ||||
1737 | Constant *CI = | ||||
1738 | ConstantExpr::getFPToSI(CFP, LHSIntVal->getType()); | ||||
1739 | if (LHSConv->hasOneUse() && | ||||
1740 | ConstantExpr::getSIToFP(CI, I.getType()) == CFP && | ||||
1741 | willNotOverflowSignedAdd(LHSIntVal, CI, I)) { | ||||
1742 | // Insert the new integer add. | ||||
1743 | Value *NewAdd = Builder.CreateNSWAdd(LHSIntVal, CI, "addconv"); | ||||
1744 | return new SIToFPInst(NewAdd, I.getType()); | ||||
1745 | } | ||||
1746 | } | ||||
1747 | |||||
1748 | // (fadd double (sitofp x), (sitofp y)) --> (sitofp (add int x, y)) | ||||
1749 | if (SIToFPInst *RHSConv = dyn_cast<SIToFPInst>(RHS)) { | ||||
1750 | Value *RHSIntVal = RHSConv->getOperand(0); | ||||
1751 | // It's enough to check LHS types only because we require int types to | ||||
1752 | // be the same for this transform. | ||||
1753 | if (IsValidPromotion(FPType, LHSIntVal->getType())) { | ||||
1754 | // Only do this if x/y have the same type, if at least one of them has a | ||||
1755 | // single use (so we don't increase the number of int->fp conversions), | ||||
1756 | // and if the integer add will not overflow. | ||||
1757 | if (LHSIntVal->getType() == RHSIntVal->getType() && | ||||
1758 | (LHSConv->hasOneUse() || RHSConv->hasOneUse()) && | ||||
1759 | willNotOverflowSignedAdd(LHSIntVal, RHSIntVal, I)) { | ||||
1760 | // Insert the new integer add. | ||||
1761 | Value *NewAdd = Builder.CreateNSWAdd(LHSIntVal, RHSIntVal, "addconv"); | ||||
1762 | return new SIToFPInst(NewAdd, I.getType()); | ||||
1763 | } | ||||
1764 | } | ||||
1765 | } | ||||
1766 | } | ||||
1767 | |||||
1768 | // Handle specials cases for FAdd with selects feeding the operation | ||||
1769 | if (Value *V = SimplifySelectsFeedingBinaryOp(I, LHS, RHS)) | ||||
1770 | return replaceInstUsesWith(I, V); | ||||
1771 | |||||
1772 | if (I.hasAllowReassoc() && I.hasNoSignedZeros()) { | ||||
1773 | if (Instruction *F = factorizeFAddFSub(I, Builder)) | ||||
1774 | return F; | ||||
1775 | |||||
1776 | // Try to fold fadd into start value of reduction intrinsic. | ||||
1777 | if (match(&I, m_c_FAdd(m_OneUse(m_Intrinsic<Intrinsic::vector_reduce_fadd>( | ||||
1778 | m_AnyZeroFP(), m_Value(X))), | ||||
1779 | m_Value(Y)))) { | ||||
1780 | // fadd (rdx 0.0, X), Y --> rdx Y, X | ||||
1781 | return replaceInstUsesWith( | ||||
1782 | I, Builder.CreateIntrinsic(Intrinsic::vector_reduce_fadd, | ||||
1783 | {X->getType()}, {Y, X}, &I)); | ||||
1784 | } | ||||
1785 | const APFloat *StartC, *C; | ||||
1786 | if (match(LHS, m_OneUse(m_Intrinsic<Intrinsic::vector_reduce_fadd>( | ||||
1787 | m_APFloat(StartC), m_Value(X)))) && | ||||
1788 | match(RHS, m_APFloat(C))) { | ||||
1789 | // fadd (rdx StartC, X), C --> rdx (C + StartC), X | ||||
1790 | Constant *NewStartC = ConstantFP::get(I.getType(), *C + *StartC); | ||||
1791 | return replaceInstUsesWith( | ||||
1792 | I, Builder.CreateIntrinsic(Intrinsic::vector_reduce_fadd, | ||||
1793 | {X->getType()}, {NewStartC, X}, &I)); | ||||
1794 | } | ||||
1795 | |||||
1796 | // (X * MulC) + X --> X * (MulC + 1.0) | ||||
1797 | Constant *MulC; | ||||
1798 | if (match(&I, m_c_FAdd(m_FMul(m_Value(X), m_ImmConstant(MulC)), | ||||
1799 | m_Deferred(X)))) { | ||||
1800 | if (Constant *NewMulC = ConstantFoldBinaryOpOperands( | ||||
1801 | Instruction::FAdd, MulC, ConstantFP::get(I.getType(), 1.0), DL)) | ||||
1802 | return BinaryOperator::CreateFMulFMF(X, NewMulC, &I); | ||||
1803 | } | ||||
1804 | |||||
1805 | // (-X - Y) + (X + Z) --> Z - Y | ||||
1806 | if (match(&I, m_c_FAdd(m_FSub(m_FNeg(m_Value(X)), m_Value(Y)), | ||||
1807 | m_c_FAdd(m_Deferred(X), m_Value(Z))))) | ||||
1808 | return BinaryOperator::CreateFSubFMF(Z, Y, &I); | ||||
1809 | |||||
1810 | if (Value *V = FAddCombine(Builder).simplify(&I)) | ||||
1811 | return replaceInstUsesWith(I, V); | ||||
1812 | } | ||||
1813 | |||||
1814 | // minumum(X, Y) + maximum(X, Y) => X + Y. | ||||
1815 | if (match(&I, | ||||
1816 | m_c_FAdd(m_Intrinsic<Intrinsic::maximum>(m_Value(X), m_Value(Y)), | ||||
1817 | m_c_Intrinsic<Intrinsic::minimum>(m_Deferred(X), | ||||
1818 | m_Deferred(Y))))) { | ||||
1819 | BinaryOperator *Result = BinaryOperator::CreateFAddFMF(X, Y, &I); | ||||
1820 | // We cannot preserve ninf if nnan flag is not set. | ||||
1821 | // If X is NaN and Y is Inf then in original program we had NaN + NaN, | ||||
1822 | // while in optimized version NaN + Inf and this is a poison with ninf flag. | ||||
1823 | if (!Result->hasNoNaNs()) | ||||
1824 | Result->setHasNoInfs(false); | ||||
1825 | return Result; | ||||
1826 | } | ||||
1827 | |||||
1828 | return nullptr; | ||||
1829 | } | ||||
1830 | |||||
1831 | /// Optimize pointer differences into the same array into a size. Consider: | ||||
1832 | /// &A[10] - &A[0]: we should compile this to "10". LHS/RHS are the pointer | ||||
1833 | /// operands to the ptrtoint instructions for the LHS/RHS of the subtract. | ||||
1834 | Value *InstCombinerImpl::OptimizePointerDifference(Value *LHS, Value *RHS, | ||||
1835 | Type *Ty, bool IsNUW) { | ||||
1836 | // If LHS is a gep based on RHS or RHS is a gep based on LHS, we can optimize | ||||
1837 | // this. | ||||
1838 | bool Swapped = false; | ||||
1839 | GEPOperator *GEP1 = nullptr, *GEP2 = nullptr; | ||||
1840 | if (!isa<GEPOperator>(LHS) && isa<GEPOperator>(RHS)) { | ||||
1841 | std::swap(LHS, RHS); | ||||
1842 | Swapped = true; | ||||
1843 | } | ||||
1844 | |||||
1845 | // Require at least one GEP with a common base pointer on both sides. | ||||
1846 | if (auto *LHSGEP = dyn_cast<GEPOperator>(LHS)) { | ||||
1847 | // (gep X, ...) - X | ||||
1848 | if (LHSGEP->getOperand(0)->stripPointerCasts() == | ||||
1849 | RHS->stripPointerCasts()) { | ||||
1850 | GEP1 = LHSGEP; | ||||
1851 | } else if (auto *RHSGEP = dyn_cast<GEPOperator>(RHS)) { | ||||
1852 | // (gep X, ...) - (gep X, ...) | ||||
1853 | if (LHSGEP->getOperand(0)->stripPointerCasts() == | ||||
1854 | RHSGEP->getOperand(0)->stripPointerCasts()) { | ||||
1855 | GEP1 = LHSGEP; | ||||
1856 | GEP2 = RHSGEP; | ||||
1857 | } | ||||
1858 | } | ||||
1859 | } | ||||
1860 | |||||
1861 | if (!GEP1) | ||||
1862 | return nullptr; | ||||
1863 | |||||
1864 | if (GEP2) { | ||||
1865 | // (gep X, ...) - (gep X, ...) | ||||
1866 | // | ||||
1867 | // Avoid duplicating the arithmetic if there are more than one non-constant | ||||
1868 | // indices between the two GEPs and either GEP has a non-constant index and | ||||
1869 | // multiple users. If zero non-constant index, the result is a constant and | ||||
1870 | // there is no duplication. If one non-constant index, the result is an add | ||||
1871 | // or sub with a constant, which is no larger than the original code, and | ||||
1872 | // there's no duplicated arithmetic, even if either GEP has multiple | ||||
1873 | // users. If more than one non-constant indices combined, as long as the GEP | ||||
1874 | // with at least one non-constant index doesn't have multiple users, there | ||||
1875 | // is no duplication. | ||||
1876 | unsigned NumNonConstantIndices1 = GEP1->countNonConstantIndices(); | ||||
1877 | unsigned NumNonConstantIndices2 = GEP2->countNonConstantIndices(); | ||||
1878 | if (NumNonConstantIndices1 + NumNonConstantIndices2 > 1 && | ||||
1879 | ((NumNonConstantIndices1 > 0 && !GEP1->hasOneUse()) || | ||||
1880 | (NumNonConstantIndices2 > 0 && !GEP2->hasOneUse()))) { | ||||
1881 | return nullptr; | ||||
1882 | } | ||||
1883 | } | ||||
1884 | |||||
1885 | // Emit the offset of the GEP and an intptr_t. | ||||
1886 | Value *Result = EmitGEPOffset(GEP1); | ||||
1887 | |||||
1888 | // If this is a single inbounds GEP and the original sub was nuw, | ||||
1889 | // then the final multiplication is also nuw. | ||||
1890 | if (auto *I = dyn_cast<Instruction>(Result)) | ||||
1891 | if (IsNUW && !GEP2 && !Swapped && GEP1->isInBounds() && | ||||
1892 | I->getOpcode() == Instruction::Mul) | ||||
1893 | I->setHasNoUnsignedWrap(); | ||||
1894 | |||||
1895 | // If we have a 2nd GEP of the same base pointer, subtract the offsets. | ||||
1896 | // If both GEPs are inbounds, then the subtract does not have signed overflow. | ||||
1897 | if (GEP2) { | ||||
1898 | Value *Offset = EmitGEPOffset(GEP2); | ||||
1899 | Result = Builder.CreateSub(Result, Offset, "gepdiff", /* NUW */ false, | ||||
1900 | GEP1->isInBounds() && GEP2->isInBounds()); | ||||
1901 | } | ||||
1902 | |||||
1903 | // If we have p - gep(p, ...) then we have to negate the result. | ||||
1904 | if (Swapped) | ||||
1905 | Result = Builder.CreateNeg(Result, "diff.neg"); | ||||
1906 | |||||
1907 | return Builder.CreateIntCast(Result, Ty, true); | ||||
1908 | } | ||||
1909 | |||||
1910 | static Instruction *foldSubOfMinMax(BinaryOperator &I, | ||||
1911 | InstCombiner::BuilderTy &Builder) { | ||||
1912 | Value *Op0 = I.getOperand(0); | ||||
1913 | Value *Op1 = I.getOperand(1); | ||||
1914 | Type *Ty = I.getType(); | ||||
1915 | auto *MinMax = dyn_cast<MinMaxIntrinsic>(Op1); | ||||
1916 | if (!MinMax) | ||||
1917 | return nullptr; | ||||
1918 | |||||
1919 | // sub(add(X,Y), s/umin(X,Y)) --> s/umax(X,Y) | ||||
1920 | // sub(add(X,Y), s/umax(X,Y)) --> s/umin(X,Y) | ||||
1921 | Value *X = MinMax->getLHS(); | ||||
1922 | Value *Y = MinMax->getRHS(); | ||||
1923 | if (match(Op0, m_c_Add(m_Specific(X), m_Specific(Y))) && | ||||
1924 | (Op0->hasOneUse() || Op1->hasOneUse())) { | ||||
1925 | Intrinsic::ID InvID = getInverseMinMaxIntrinsic(MinMax->getIntrinsicID()); | ||||
1926 | Function *F = Intrinsic::getDeclaration(I.getModule(), InvID, Ty); | ||||
1927 | return CallInst::Create(F, {X, Y}); | ||||
1928 | } | ||||
1929 | |||||
1930 | // sub(add(X,Y),umin(Y,Z)) --> add(X,usub.sat(Y,Z)) | ||||
1931 | // sub(add(X,Z),umin(Y,Z)) --> add(X,usub.sat(Z,Y)) | ||||
1932 | Value *Z; | ||||
1933 | if (match(Op1, m_OneUse(m_UMin(m_Value(Y), m_Value(Z))))) { | ||||
1934 | if (match(Op0, m_OneUse(m_c_Add(m_Specific(Y), m_Value(X))))) { | ||||
1935 | Value *USub = Builder.CreateIntrinsic(Intrinsic::usub_sat, Ty, {Y, Z}); | ||||
1936 | return BinaryOperator::CreateAdd(X, USub); | ||||
1937 | } | ||||
1938 | if (match(Op0, m_OneUse(m_c_Add(m_Specific(Z), m_Value(X))))) { | ||||
1939 | Value *USub = Builder.CreateIntrinsic(Intrinsic::usub_sat, Ty, {Z, Y}); | ||||
1940 | return BinaryOperator::CreateAdd(X, USub); | ||||
1941 | } | ||||
1942 | } | ||||
1943 | |||||
1944 | // sub Op0, smin((sub nsw Op0, Z), 0) --> smax Op0, Z | ||||
1945 | // sub Op0, smax((sub nsw Op0, Z), 0) --> smin Op0, Z | ||||
1946 | if (MinMax->isSigned() && match(Y, m_ZeroInt()) && | ||||
1947 | match(X, m_NSWSub(m_Specific(Op0), m_Value(Z)))) { | ||||
1948 | Intrinsic::ID InvID = getInverseMinMaxIntrinsic(MinMax->getIntrinsicID()); | ||||
1949 | Function *F = Intrinsic::getDeclaration(I.getModule(), InvID, Ty); | ||||
1950 | return CallInst::Create(F, {Op0, Z}); | ||||
1951 | } | ||||
1952 | |||||
1953 | return nullptr; | ||||
1954 | } | ||||
1955 | |||||
1956 | Instruction *InstCombinerImpl::visitSub(BinaryOperator &I) { | ||||
1957 | if (Value *V = simplifySubInst(I.getOperand(0), I.getOperand(1), | ||||
1958 | I.hasNoSignedWrap(), I.hasNoUnsignedWrap(), | ||||
1959 | SQ.getWithInstruction(&I))) | ||||
1960 | return replaceInstUsesWith(I, V); | ||||
1961 | |||||
1962 | if (Instruction *X = foldVectorBinop(I)) | ||||
1963 | return X; | ||||
1964 | |||||
1965 | if (Instruction *Phi = foldBinopWithPhiOperands(I)) | ||||
1966 | return Phi; | ||||
1967 | |||||
1968 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||
1969 | |||||
1970 | // If this is a 'B = x-(-A)', change to B = x+A. | ||||
1971 | // We deal with this without involving Negator to preserve NSW flag. | ||||
1972 | if (Value *V = dyn_castNegVal(Op1)) { | ||||
1973 | BinaryOperator *Res = BinaryOperator::CreateAdd(Op0, V); | ||||
1974 | |||||
1975 | if (const auto *BO = dyn_cast<BinaryOperator>(Op1)) { | ||||
1976 | assert(BO->getOpcode() == Instruction::Sub &&(static_cast <bool> (BO->getOpcode() == Instruction:: Sub && "Expected a subtraction operator!") ? void (0) : __assert_fail ("BO->getOpcode() == Instruction::Sub && \"Expected a subtraction operator!\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1977 , __extension__ __PRETTY_FUNCTION__)) | ||||
1977 | "Expected a subtraction operator!")(static_cast <bool> (BO->getOpcode() == Instruction:: Sub && "Expected a subtraction operator!") ? void (0) : __assert_fail ("BO->getOpcode() == Instruction::Sub && \"Expected a subtraction operator!\"" , "llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp", 1977 , __extension__ __PRETTY_FUNCTION__)); | ||||
1978 | if (BO->hasNoSignedWrap() && I.hasNoSignedWrap()) | ||||
1979 | Res->setHasNoSignedWrap(true); | ||||
1980 | } else { | ||||
1981 | if (cast<Constant>(Op1)->isNotMinSignedValue() && I.hasNoSignedWrap()) | ||||
1982 | Res->setHasNoSignedWrap(true); | ||||
1983 | } | ||||
1984 | |||||
1985 | return Res; | ||||
1986 | } | ||||
1987 | |||||
1988 | // Try this before Negator to preserve NSW flag. | ||||
1989 | if (Instruction *R = factorizeMathWithShlOps(I, Builder)) | ||||
1990 | return R; | ||||
1991 | |||||
1992 | Constant *C; | ||||
1993 | if (match(Op0, m_ImmConstant(C))) { | ||||
1994 | Value *X; | ||||
1995 | Constant *C2; | ||||
1996 | |||||
1997 | // C-(X+C2) --> (C-C2)-X | ||||
1998 | if (match(Op1, m_Add(m_Value(X), m_ImmConstant(C2)))) | ||||
1999 | return BinaryOperator::CreateSub(ConstantExpr::getSub(C, C2), X); | ||||
2000 | } | ||||
2001 | |||||
2002 | auto TryToNarrowDeduceFlags = [this, &I, &Op0, &Op1]() -> Instruction * { | ||||
2003 | if (Instruction *Ext = narrowMathIfNoOverflow(I)) | ||||
2004 | return Ext; | ||||
2005 | |||||
2006 | bool Changed = false; | ||||
2007 | if (!I.hasNoSignedWrap() && willNotOverflowSignedSub(Op0, Op1, I)) { | ||||
2008 | Changed = true; | ||||
2009 | I.setHasNoSignedWrap(true); | ||||
2010 | } | ||||
2011 | if (!I.hasNoUnsignedWrap() && willNotOverflowUnsignedSub(Op0, Op1, I)) { | ||||
2012 | Changed = true; | ||||
2013 | I.setHasNoUnsignedWrap(true); | ||||
2014 | } | ||||
2015 | |||||
2016 | return Changed ? &I : nullptr; | ||||
2017 | }; | ||||
2018 | |||||
2019 | // First, let's try to interpret `sub a, b` as `add a, (sub 0, b)`, | ||||
2020 | // and let's try to sink `(sub 0, b)` into `b` itself. But only if this isn't | ||||
2021 | // a pure negation used by a select that looks like abs/nabs. | ||||
2022 | bool IsNegation = match(Op0, m_ZeroInt()); | ||||
2023 | if (!IsNegation || none_of(I.users(), [&I, Op1](const User *U) { | ||||
2024 | const Instruction *UI = dyn_cast<Instruction>(U); | ||||
2025 | if (!UI) | ||||
2026 | return false; | ||||
2027 | return match(UI, | ||||
2028 | m_Select(m_Value(), m_Specific(Op1), m_Specific(&I))) || | ||||
2029 | match(UI, m_Select(m_Value(), m_Specific(&I), m_Specific(Op1))); | ||||
2030 | })) { | ||||
2031 | if (Value *NegOp1 = Negator::Negate(IsNegation, Op1, *this)) | ||||
2032 | return BinaryOperator::CreateAdd(NegOp1, Op0); | ||||
2033 | } | ||||
2034 | if (IsNegation) | ||||
2035 | return TryToNarrowDeduceFlags(); // Should have been handled in Negator! | ||||
2036 | |||||
2037 | // (A*B)-(A*C) -> A*(B-C) etc | ||||
2038 | if (Value *V = foldUsingDistributiveLaws(I)) | ||||
2039 | return replaceInstUsesWith(I, V); | ||||
2040 | |||||
2041 | if (I.getType()->isIntOrIntVectorTy(1)) | ||||
2042 | return BinaryOperator::CreateXor(Op0, Op1); | ||||
2043 | |||||
2044 | // Replace (-1 - A) with (~A). | ||||
2045 | if (match(Op0, m_AllOnes())) | ||||
2046 | return BinaryOperator::CreateNot(Op1); | ||||
2047 | |||||
2048 | // (X + -1) - Y --> ~Y + X | ||||
2049 | Value *X, *Y; | ||||
2050 | if (match(Op0, m_OneUse(m_Add(m_Value(X), m_AllOnes())))) | ||||
2051 | return BinaryOperator::CreateAdd(Builder.CreateNot(Op1), X); | ||||
2052 | |||||
2053 | // Reassociate sub/add sequences to create more add instructions and | ||||
2054 | // reduce dependency chains: | ||||
2055 | // ((X - Y) + Z) - Op1 --> (X + Z) - (Y + Op1) | ||||
2056 | Value *Z; | ||||
2057 | if (match(Op0, m_OneUse(m_c_Add(m_OneUse(m_Sub(m_Value(X), m_Value(Y))), | ||||
2058 | m_Value(Z))))) { | ||||
2059 | Value *XZ = Builder.CreateAdd(X, Z); | ||||
2060 | Value *YW = Builder.CreateAdd(Y, Op1); | ||||
2061 | return BinaryOperator::CreateSub(XZ, YW); | ||||
2062 | } | ||||
2063 | |||||
2064 | // ((X - Y) - Op1) --> X - (Y + Op1) | ||||
2065 | if (match(Op0, m_OneUse(m_Sub(m_Value(X), m_Value(Y))))) { | ||||
2066 | Value *Add = Builder.CreateAdd(Y, Op1); | ||||
2067 | return BinaryOperator::CreateSub(X, Add); | ||||
2068 | } | ||||
2069 | |||||
2070 | // (~X) - (~Y) --> Y - X | ||||
2071 | // This is placed after the other reassociations and explicitly excludes a | ||||
2072 | // sub-of-sub pattern to avoid infinite looping. | ||||
2073 | if (isFreeToInvert(Op0, Op0->hasOneUse()) && | ||||
2074 | isFreeToInvert(Op1, Op1->hasOneUse()) && | ||||
2075 | !match(Op0, m_Sub(m_ImmConstant(), m_Value()))) { | ||||
2076 | Value *NotOp0 = Builder.CreateNot(Op0); | ||||
2077 | Value *NotOp1 = Builder.CreateNot(Op1); | ||||
2078 | return BinaryOperator::CreateSub(NotOp1, NotOp0); | ||||
2079 | } | ||||
2080 | |||||
2081 | auto m_AddRdx = [](Value *&Vec) { | ||||
2082 | return m_OneUse(m_Intrinsic<Intrinsic::vector_reduce_add>(m_Value(Vec))); | ||||
2083 | }; | ||||
2084 | Value *V0, *V1; | ||||
2085 | if (match(Op0, m_AddRdx(V0)) && match(Op1, m_AddRdx(V1)) && | ||||
2086 | V0->getType() == V1->getType()) { | ||||
2087 | // Difference of sums is sum of differences: | ||||
2088 | // add_rdx(V0) - add_rdx(V1) --> add_rdx(V0 - V1) | ||||
2089 | Value *Sub = Builder.CreateSub(V0, V1); | ||||
2090 | Value *Rdx = Builder.CreateIntrinsic(Intrinsic::vector_reduce_add, | ||||
2091 | {Sub->getType()}, {Sub}); | ||||
2092 | return replaceInstUsesWith(I, Rdx); | ||||
2093 | } | ||||
2094 | |||||
2095 | if (Constant *C = dyn_cast<Constant>(Op0)) { | ||||
2096 | Value *X; | ||||
2097 | if (match(Op1, m_ZExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) | ||||
2098 | // C - (zext bool) --> bool ? C - 1 : C | ||||
2099 | return SelectInst::Create(X, InstCombiner::SubOne(C), C); | ||||
2100 | if (match(Op1, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) | ||||
2101 | // C - (sext bool) --> bool ? C + 1 : C | ||||
2102 | return SelectInst::Create(X, InstCombiner::AddOne(C), C); | ||||
2103 | |||||
2104 | // C - ~X == X + (1+C) | ||||
2105 | if (match(Op1, m_Not(m_Value(X)))) | ||||
2106 | return BinaryOperator::CreateAdd(X, InstCombiner::AddOne(C)); | ||||
2107 | |||||
2108 | // Try to fold constant sub into select arguments. | ||||
2109 | if (SelectInst *SI = dyn_cast<SelectInst>(Op1)) | ||||
2110 | if (Instruction *R = FoldOpIntoSelect(I, SI)) | ||||
2111 | return R; | ||||
2112 | |||||
2113 | // Try to fold constant sub into PHI values. | ||||
2114 | if (PHINode *PN = dyn_cast<PHINode>(Op1)) | ||||
2115 | if (Instruction *R = foldOpIntoPhi(I, PN)) | ||||
2116 | return R; | ||||
2117 | |||||
2118 | Constant *C2; | ||||
2119 | |||||
2120 | // C-(C2-X) --> X+(C-C2) | ||||
2121 | if (match(Op1, m_Sub(m_ImmConstant(C2), m_Value(X)))) | ||||
2122 | return BinaryOperator::CreateAdd(X, ConstantExpr::getSub(C, C2)); | ||||
2123 | } | ||||
2124 | |||||
2125 | const APInt *Op0C; | ||||
2126 | if (match(Op0, m_APInt(Op0C))) { | ||||
2127 | if (Op0C->isMask()) { | ||||
2128 | // Turn this into a xor if LHS is 2^n-1 and the remaining bits are known | ||||
2129 | // zero. | ||||
2130 | KnownBits RHSKnown = computeKnownBits(Op1, 0, &I); | ||||
2131 | if ((*Op0C | RHSKnown.Zero).isAllOnes()) | ||||
2132 | return BinaryOperator::CreateXor(Op1, Op0); | ||||
2133 | } | ||||
2134 | |||||
2135 | // C - ((C3 -nuw X) & C2) --> (C - (C2 & C3)) + (X & C2) when: | ||||
2136 | // (C3 - ((C2 & C3) - 1)) is pow2 | ||||
2137 | // ((C2 + C3) & ((C2 & C3) - 1)) == ((C2 & C3) - 1) | ||||
2138 | // C2 is negative pow2 || sub nuw | ||||
2139 | const APInt *C2, *C3; | ||||
2140 | BinaryOperator *InnerSub; | ||||
2141 | if (match(Op1, m_OneUse(m_And(m_BinOp(InnerSub), m_APInt(C2)))) && | ||||
2142 | match(InnerSub, m_Sub(m_APInt(C3), m_Value(X))) && | ||||
2143 | (InnerSub->hasNoUnsignedWrap() || C2->isNegatedPowerOf2())) { | ||||
2144 | APInt C2AndC3 = *C2 & *C3; | ||||
2145 | APInt C2AndC3Minus1 = C2AndC3 - 1; | ||||
2146 | APInt C2AddC3 = *C2 + *C3; | ||||
2147 | if ((*C3 - C2AndC3Minus1).isPowerOf2() && | ||||
2148 | C2AndC3Minus1.isSubsetOf(C2AddC3)) { | ||||
2149 | Value *And = Builder.CreateAnd(X, ConstantInt::get(I.getType(), *C2)); | ||||
2150 | return BinaryOperator::CreateAdd( | ||||
2151 | And, ConstantInt::get(I.getType(), *Op0C - C2AndC3)); | ||||
2152 | } | ||||
2153 | } | ||||
2154 | } | ||||
2155 | |||||
2156 | { | ||||
2157 | Value *Y; | ||||
2158 | // X-(X+Y) == -Y X-(Y+X) == -Y | ||||
2159 | if (match(Op1, m_c_Add(m_Specific(Op0), m_Value(Y)))) | ||||
2160 | return BinaryOperator::CreateNeg(Y); | ||||
2161 | |||||
2162 | // (X-Y)-X == -Y | ||||
2163 | if (match(Op0, m_Sub(m_Specific(Op1), m_Value(Y)))) | ||||
2164 | return BinaryOperator::CreateNeg(Y); | ||||
2165 | } | ||||
2166 | |||||
2167 | // (sub (or A, B) (and A, B)) --> (xor A, B) | ||||
2168 | { | ||||
2169 | Value *A, *B; | ||||
2170 | if (match(Op1, m_And(m_Value(A), m_Value(B))) && | ||||
2171 | match(Op0, m_c_Or(m_Specific(A), m_Specific(B)))) | ||||
2172 | return BinaryOperator::CreateXor(A, B); | ||||
2173 | } | ||||
2174 | |||||
2175 | // (sub (add A, B) (or A, B)) --> (and A, B) | ||||
2176 | { | ||||
2177 | Value *A, *B; | ||||
2178 | if (match(Op0, m_Add(m_Value(A), m_Value(B))) && | ||||
2179 | match(Op1, m_c_Or(m_Specific(A), m_Specific(B)))) | ||||
2180 | return BinaryOperator::CreateAnd(A, B); | ||||
2181 | } | ||||
2182 | |||||
2183 | // (sub (add A, B) (and A, B)) --> (or A, B) | ||||
2184 | { | ||||
2185 | Value *A, *B; | ||||
2186 | if (match(Op0, m_Add(m_Value(A), m_Value(B))) && | ||||
2187 | match(Op1, m_c_And(m_Specific(A), m_Specific(B)))) | ||||
2188 | return BinaryOperator::CreateOr(A, B); | ||||
2189 | } | ||||
2190 | |||||
2191 | // (sub (and A, B) (or A, B)) --> neg (xor A, B) | ||||
2192 | { | ||||
2193 | Value *A, *B; | ||||
2194 | if (match(Op0, m_And(m_Value(A), m_Value(B))) && | ||||
2195 | match(Op1, m_c_Or(m_Specific(A), m_Specific(B))) && | ||||
2196 | (Op0->hasOneUse() || Op1->hasOneUse())) | ||||
2197 | return BinaryOperator::CreateNeg(Builder.CreateXor(A, B)); | ||||
2198 | } | ||||
2199 | |||||
2200 | // (sub (or A, B), (xor A, B)) --> (and A, B) | ||||
2201 | { | ||||
2202 | Value *A, *B; | ||||
2203 | if (match(Op1, m_Xor(m_Value(A), m_Value(B))) && | ||||
2204 | match(Op0, m_c_Or(m_Specific(A), m_Specific(B)))) | ||||
2205 | return BinaryOperator::CreateAnd(A, B); | ||||
2206 | } | ||||
2207 | |||||
2208 | // (sub (xor A, B) (or A, B)) --> neg (and A, B) | ||||
2209 | { | ||||
2210 | Value *A, *B; | ||||
2211 | if (match(Op0, m_Xor(m_Value(A), m_Value(B))) && | ||||
2212 | match(Op1, m_c_Or(m_Specific(A), m_Specific(B))) && | ||||
2213 | (Op0->hasOneUse() || Op1->hasOneUse())) | ||||
2214 | return BinaryOperator::CreateNeg(Builder.CreateAnd(A, B)); | ||||
2215 | } | ||||
2216 | |||||
2217 | { | ||||
2218 | Value *Y; | ||||
2219 | // ((X | Y) - X) --> (~X & Y) | ||||
2220 | if (match(Op0, m_OneUse(m_c_Or(m_Value(Y), m_Specific(Op1))))) | ||||
2221 | return BinaryOperator::CreateAnd( | ||||
2222 | Y, Builder.CreateNot(Op1, Op1->getName() + ".not")); | ||||
2223 | } | ||||
2224 | |||||
2225 | { | ||||
2226 | // (sub (and Op1, (neg X)), Op1) --> neg (and Op1, (add X, -1)) | ||||
2227 | Value *X; | ||||
2228 | if (match(Op0, m_OneUse(m_c_And(m_Specific(Op1), | ||||
2229 | m_OneUse(m_Neg(m_Value(X))))))) { | ||||
2230 | return BinaryOperator::CreateNeg(Builder.CreateAnd( | ||||
2231 | Op1, Builder.CreateAdd(X, Constant::getAllOnesValue(I.getType())))); | ||||
2232 | } | ||||
2233 | } | ||||
2234 | |||||
2235 | { | ||||
2236 | // (sub (and Op1, C), Op1) --> neg (and Op1, ~C) | ||||
2237 | Constant *C; | ||||
2238 | if (match(Op0, m_OneUse(m_And(m_Specific(Op1), m_Constant(C))))) { | ||||
2239 | return BinaryOperator::CreateNeg( | ||||
2240 | Builder.CreateAnd(Op1, Builder.CreateNot(C))); | ||||
2241 | } | ||||
2242 | } | ||||
2243 | |||||
2244 | if (Instruction *R = foldSubOfMinMax(I, Builder)) | ||||
2245 | return R; | ||||
2246 | |||||
2247 | { | ||||
2248 | // If we have a subtraction between some value and a select between | ||||
2249 | // said value and something else, sink subtraction into select hands, i.e.: | ||||
2250 | // sub (select %Cond, %TrueVal, %FalseVal), %Op1 | ||||
2251 | // -> | ||||
2252 | // select %Cond, (sub %TrueVal, %Op1), (sub %FalseVal, %Op1) | ||||
2253 | // or | ||||
2254 | // sub %Op0, (select %Cond, %TrueVal, %FalseVal) | ||||
2255 | // -> | ||||
2256 | // select %Cond, (sub %Op0, %TrueVal), (sub %Op0, %FalseVal) | ||||
2257 | // This will result in select between new subtraction and 0. | ||||
2258 | auto SinkSubIntoSelect = | ||||
2259 | [Ty = I.getType()](Value *Select, Value *OtherHandOfSub, | ||||
2260 | auto SubBuilder) -> Instruction * { | ||||
2261 | Value *Cond, *TrueVal, *FalseVal; | ||||
2262 | if (!match(Select, m_OneUse(m_Select(m_Value(Cond), m_Value(TrueVal), | ||||
2263 | m_Value(FalseVal))))) | ||||
2264 | return nullptr; | ||||
2265 | if (OtherHandOfSub != TrueVal && OtherHandOfSub != FalseVal) | ||||
2266 | return nullptr; | ||||
2267 | // While it is really tempting to just create two subtractions and let | ||||
2268 | // InstCombine fold one of those to 0, it isn't possible to do so | ||||
2269 | // because of worklist visitation order. So ugly it is. | ||||
2270 | bool OtherHandOfSubIsTrueVal = OtherHandOfSub == TrueVal; | ||||
2271 | Value *NewSub = SubBuilder(OtherHandOfSubIsTrueVal ? FalseVal : TrueVal); | ||||
2272 | Constant *Zero = Constant::getNullValue(Ty); | ||||
2273 | SelectInst *NewSel = | ||||
2274 | SelectInst::Create(Cond, OtherHandOfSubIsTrueVal ? Zero : NewSub, | ||||
2275 | OtherHandOfSubIsTrueVal ? NewSub : Zero); | ||||
2276 | // Preserve prof metadata if any. | ||||
2277 | NewSel->copyMetadata(cast<Instruction>(*Select)); | ||||
2278 | return NewSel; | ||||
2279 | }; | ||||
2280 | if (Instruction *NewSel = SinkSubIntoSelect( | ||||
2281 | /*Select=*/Op0, /*OtherHandOfSub=*/Op1, | ||||
2282 | [Builder = &Builder, Op1](Value *OtherHandOfSelect) { | ||||
2283 | return Builder->CreateSub(OtherHandOfSelect, | ||||
2284 | /*OtherHandOfSub=*/Op1); | ||||
2285 | })) | ||||
2286 | return NewSel; | ||||
2287 | if (Instruction *NewSel = SinkSubIntoSelect( | ||||
2288 | /*Select=*/Op1, /*OtherHandOfSub=*/Op0, | ||||
2289 | [Builder = &Builder, Op0](Value *OtherHandOfSelect) { | ||||
2290 | return Builder->CreateSub(/*OtherHandOfSub=*/Op0, | ||||
2291 | OtherHandOfSelect); | ||||
2292 | })) | ||||
2293 | return NewSel; | ||||
2294 | } | ||||
2295 | |||||
2296 | // (X - (X & Y)) --> (X & ~Y) | ||||
2297 | if (match(Op1, m_c_And(m_Specific(Op0), m_Value(Y))) && | ||||
2298 | (Op1->hasOneUse() || isa<Constant>(Y))) | ||||
2299 | return BinaryOperator::CreateAnd( | ||||
2300 | Op0, Builder.CreateNot(Y, Y->getName() + ".not")); | ||||
2301 | |||||
2302 | // ~X - Min/Max(~X, Y) -> ~Min/Max(X, ~Y) - X | ||||
2303 | // ~X - Min/Max(Y, ~X) -> ~Min/Max(X, ~Y) - X | ||||
2304 | // Min/Max(~X, Y) - ~X -> X - ~Min/Max(X, ~Y) | ||||
2305 | // Min/Max(Y, ~X) - ~X -> X - ~Min/Max(X, ~Y) | ||||
2306 | // As long as Y is freely invertible, this will be neutral or a win. | ||||
2307 | // Note: We don't generate the inverse max/min, just create the 'not' of | ||||
2308 | // it and let other folds do the rest. | ||||
2309 | if (match(Op0, m_Not(m_Value(X))) && | ||||
2310 | match(Op1, m_c_MaxOrMin(m_Specific(Op0), m_Value(Y))) && | ||||
2311 | !Op0->hasNUsesOrMore(3) && isFreeToInvert(Y, Y->hasOneUse())) { | ||||
2312 | Value *Not = Builder.CreateNot(Op1); | ||||
2313 | return BinaryOperator::CreateSub(Not, X); | ||||
2314 | } | ||||
2315 | if (match(Op1, m_Not(m_Value(X))) && | ||||
2316 | match(Op0, m_c_MaxOrMin(m_Specific(Op1), m_Value(Y))) && | ||||
2317 | !Op1->hasNUsesOrMore(3) && isFreeToInvert(Y, Y->hasOneUse())) { | ||||
2318 | Value *Not = Builder.CreateNot(Op0); | ||||
2319 | return BinaryOperator::CreateSub(X, Not); | ||||
2320 | } | ||||
2321 | |||||
2322 | // Optimize pointer differences into the same array into a size. Consider: | ||||
2323 | // &A[10] - &A[0]: we should compile this to "10". | ||||
2324 | Value *LHSOp, *RHSOp; | ||||
2325 | if (match(Op0, m_PtrToInt(m_Value(LHSOp))) && | ||||
2326 | match(Op1, m_PtrToInt(m_Value(RHSOp)))) | ||||
2327 | if (Value *Res = OptimizePointerDifference(LHSOp, RHSOp, I.getType(), | ||||
2328 | I.hasNoUnsignedWrap())) | ||||
2329 | return replaceInstUsesWith(I, Res); | ||||
2330 | |||||
2331 | // trunc(p)-trunc(q) -> trunc(p-q) | ||||
2332 | if (match(Op0, m_Trunc(m_PtrToInt(m_Value(LHSOp)))) && | ||||
2333 | match(Op1, m_Trunc(m_PtrToInt(m_Value(RHSOp))))) | ||||
2334 | if (Value *Res = OptimizePointerDifference(LHSOp, RHSOp, I.getType(), | ||||
2335 | /* IsNUW */ false)) | ||||
2336 | return replaceInstUsesWith(I, Res); | ||||
2337 | |||||
2338 | // Canonicalize a shifty way to code absolute value to the common pattern. | ||||
2339 | // There are 2 potential commuted variants. | ||||
2340 | // We're relying on the fact that we only do this transform when the shift has | ||||
2341 | // exactly 2 uses and the xor has exactly 1 use (otherwise, we might increase | ||||
2342 | // instructions). | ||||
2343 | Value *A; | ||||
2344 | const APInt *ShAmt; | ||||
2345 | Type *Ty = I.getType(); | ||||
2346 | unsigned BitWidth = Ty->getScalarSizeInBits(); | ||||
2347 | if (match(Op1, m_AShr(m_Value(A), m_APInt(ShAmt))) && | ||||
2348 | Op1->hasNUses(2) && *ShAmt == BitWidth - 1 && | ||||
2349 | match(Op0, m_OneUse(m_c_Xor(m_Specific(A), m_Specific(Op1))))) { | ||||
2350 | // B = ashr i32 A, 31 ; smear the sign bit | ||||
2351 | // sub (xor A, B), B ; flip bits if negative and subtract -1 (add 1) | ||||
2352 | // --> (A < 0) ? -A : A | ||||
2353 | Value *IsNeg = Builder.CreateIsNeg(A); | ||||
2354 | // Copy the nuw/nsw flags from the sub to the negate. | ||||
2355 | Value *NegA = Builder.CreateNeg(A, "", I.hasNoUnsignedWrap(), | ||||
2356 | I.hasNoSignedWrap()); | ||||
2357 | return SelectInst::Create(IsNeg, NegA, A); | ||||
2358 | } | ||||
2359 | |||||
2360 | // If we are subtracting a low-bit masked subset of some value from an add | ||||
2361 | // of that same value with no low bits changed, that is clearing some low bits | ||||
2362 | // of the sum: | ||||
2363 | // sub (X + AddC), (X & AndC) --> and (X + AddC), ~AndC | ||||
2364 | const APInt *AddC, *AndC; | ||||
2365 | if (match(Op0, m_Add(m_Value(X), m_APInt(AddC))) && | ||||
2366 | match(Op1, m_And(m_Specific(X), m_APInt(AndC)))) { | ||||
2367 | unsigned Cttz = AddC->countr_zero(); | ||||
2368 | APInt HighMask(APInt::getHighBitsSet(BitWidth, BitWidth - Cttz)); | ||||
2369 | if ((HighMask & *AndC).isZero()) | ||||
2370 | return BinaryOperator::CreateAnd(Op0, ConstantInt::get(Ty, ~(*AndC))); | ||||
2371 | } | ||||
2372 | |||||
2373 | if (Instruction *V = | ||||
2374 | canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract(I)) | ||||
2375 | return V; | ||||
2376 | |||||
2377 | // X - usub.sat(X, Y) => umin(X, Y) | ||||
2378 | if (match(Op1, m_OneUse(m_Intrinsic<Intrinsic::usub_sat>(m_Specific(Op0), | ||||
2379 | m_Value(Y))))) | ||||
2380 | return replaceInstUsesWith( | ||||
2381 | I, Builder.CreateIntrinsic(Intrinsic::umin, {I.getType()}, {Op0, Y})); | ||||
2382 | |||||
2383 | // umax(X, Op1) - Op1 --> usub.sat(X, Op1) | ||||
2384 | // TODO: The one-use restriction is not strictly necessary, but it may | ||||
2385 | // require improving other pattern matching and/or codegen. | ||||
2386 | if (match(Op0, m_OneUse(m_c_UMax(m_Value(X), m_Specific(Op1))))) | ||||
2387 | return replaceInstUsesWith( | ||||
2388 | I, Builder.CreateIntrinsic(Intrinsic::usub_sat, {Ty}, {X, Op1})); | ||||
2389 | |||||
2390 | // Op0 - umin(X, Op0) --> usub.sat(Op0, X) | ||||
2391 | if (match(Op1, m_OneUse(m_c_UMin(m_Value(X), m_Specific(Op0))))) | ||||
2392 | return replaceInstUsesWith( | ||||
2393 | I, Builder.CreateIntrinsic(Intrinsic::usub_sat, {Ty}, {Op0, X})); | ||||
2394 | |||||
2395 | // Op0 - umax(X, Op0) --> 0 - usub.sat(X, Op0) | ||||
2396 | if (match(Op1, m_OneUse(m_c_UMax(m_Value(X), m_Specific(Op0))))) { | ||||
2397 | Value *USub = Builder.CreateIntrinsic(Intrinsic::usub_sat, {Ty}, {X, Op0}); | ||||
2398 | return BinaryOperator::CreateNeg(USub); | ||||
2399 | } | ||||
2400 | |||||
2401 | // umin(X, Op1) - Op1 --> 0 - usub.sat(Op1, X) | ||||
2402 | if (match(Op0, m_OneUse(m_c_UMin(m_Value(X), m_Specific(Op1))))) { | ||||
2403 | Value *USub = Builder.CreateIntrinsic(Intrinsic::usub_sat, {Ty}, {Op1, X}); | ||||
2404 | return BinaryOperator::CreateNeg(USub); | ||||
2405 | } | ||||
2406 | |||||
2407 | // C - ctpop(X) => ctpop(~X) if C is bitwidth | ||||
2408 | if (match(Op0, m_SpecificInt(BitWidth)) && | ||||
2409 | match(Op1, m_OneUse(m_Intrinsic<Intrinsic::ctpop>(m_Value(X))))) | ||||
2410 | return replaceInstUsesWith( | ||||
2411 | I, Builder.CreateIntrinsic(Intrinsic::ctpop, {I.getType()}, | ||||
2412 | {Builder.CreateNot(X)})); | ||||
2413 | |||||
2414 | // Reduce multiplies for difference-of-squares by factoring: | ||||
2415 | // (X * X) - (Y * Y) --> (X + Y) * (X - Y) | ||||
2416 | if (match(Op0, m_OneUse(m_Mul(m_Value(X), m_Deferred(X)))) && | ||||
2417 | match(Op1, m_OneUse(m_Mul(m_Value(Y), m_Deferred(Y))))) { | ||||
2418 | auto *OBO0 = cast<OverflowingBinaryOperator>(Op0); | ||||
2419 | auto *OBO1 = cast<OverflowingBinaryOperator>(Op1); | ||||
2420 | bool PropagateNSW = I.hasNoSignedWrap() && OBO0->hasNoSignedWrap() && | ||||
2421 | OBO1->hasNoSignedWrap() && BitWidth > 2; | ||||
2422 | bool PropagateNUW = I.hasNoUnsignedWrap() && OBO0->hasNoUnsignedWrap() && | ||||
2423 | OBO1->hasNoUnsignedWrap() && BitWidth > 1; | ||||
2424 | Value *Add = Builder.CreateAdd(X, Y, "add", PropagateNUW, PropagateNSW); | ||||
2425 | Value *Sub = Builder.CreateSub(X, Y, "sub", PropagateNUW, PropagateNSW); | ||||
2426 | Value *Mul = Builder.CreateMul(Add, Sub, "", PropagateNUW, PropagateNSW); | ||||
2427 | return replaceInstUsesWith(I, Mul); | ||||
2428 | } | ||||
2429 | |||||
2430 | // max(X,Y) nsw/nuw - min(X,Y) --> abs(X nsw - Y) | ||||
2431 | if (match(Op0, m_OneUse(m_c_SMax(m_Value(X), m_Value(Y)))) && | ||||
2432 | match(Op1, m_OneUse(m_c_SMin(m_Specific(X), m_Specific(Y))))) { | ||||
2433 | if (I.hasNoUnsignedWrap() || I.hasNoSignedWrap()) { | ||||
2434 | Value *Sub = | ||||
2435 | Builder.CreateSub(X, Y, "sub", /*HasNUW=*/false, /*HasNSW=*/true); | ||||
2436 | Value *Call = | ||||
2437 | Builder.CreateBinaryIntrinsic(Intrinsic::abs, Sub, Builder.getTrue()); | ||||
2438 | return replaceInstUsesWith(I, Call); | ||||
2439 | } | ||||
2440 | } | ||||
2441 | |||||
2442 | return TryToNarrowDeduceFlags(); | ||||
2443 | } | ||||
2444 | |||||
2445 | /// This eliminates floating-point negation in either 'fneg(X)' or | ||||
2446 | /// 'fsub(-0.0, X)' form by combining into a constant operand. | ||||
2447 | static Instruction *foldFNegIntoConstant(Instruction &I, const DataLayout &DL) { | ||||
2448 | // This is limited with one-use because fneg is assumed better for | ||||
2449 | // reassociation and cheaper in codegen than fmul/fdiv. | ||||
2450 | // TODO: Should the m_OneUse restriction be removed? | ||||
2451 | Instruction *FNegOp; | ||||
2452 | if (!match(&I, m_FNeg(m_OneUse(m_Instruction(FNegOp))))) | ||||
2453 | return nullptr; | ||||
2454 | |||||
2455 | Value *X; | ||||
2456 | Constant *C; | ||||
2457 | |||||
2458 | // Fold negation into constant operand. | ||||
2459 | // -(X * C) --> X * (-C) | ||||
2460 | if (match(FNegOp, m_FMul(m_Value(X), m_Constant(C)))) | ||||
2461 | if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL)) | ||||
2462 | return BinaryOperator::CreateFMulFMF(X, NegC, &I); | ||||
2463 | // -(X / C) --> X / (-C) | ||||
2464 | if (match(FNegOp, m_FDiv(m_Value(X), m_Constant(C)))) | ||||
2465 | if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL)) | ||||
2466 | return BinaryOperator::CreateFDivFMF(X, NegC, &I); | ||||
2467 | // -(C / X) --> (-C) / X | ||||
2468 | if (match(FNegOp, m_FDiv(m_Constant(C), m_Value(X)))) | ||||
2469 | if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL)) { | ||||
2470 | Instruction *FDiv = BinaryOperator::CreateFDivFMF(NegC, X, &I); | ||||
2471 | |||||
2472 | // Intersect 'nsz' and 'ninf' because those special value exceptions may | ||||
2473 | // not apply to the fdiv. Everything else propagates from the fneg. | ||||
2474 | // TODO: We could propagate nsz/ninf from fdiv alone? | ||||
2475 | FastMathFlags FMF = I.getFastMathFlags(); | ||||
2476 | FastMathFlags OpFMF = FNegOp->getFastMathFlags(); | ||||
2477 | FDiv->setHasNoSignedZeros(FMF.noSignedZeros() && OpFMF.noSignedZeros()); | ||||
2478 | FDiv->setHasNoInfs(FMF.noInfs() && OpFMF.noInfs()); | ||||
2479 | return FDiv; | ||||
2480 | } | ||||
2481 | // With NSZ [ counter-example with -0.0: -(-0.0 + 0.0) != 0.0 + -0.0 ]: | ||||
2482 | // -(X + C) --> -X + -C --> -C - X | ||||
2483 | if (I.hasNoSignedZeros() && match(FNegOp, m_FAdd(m_Value(X), m_Constant(C)))) | ||||
2484 | if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL)) | ||||
2485 | return BinaryOperator::CreateFSubFMF(NegC, X, &I); | ||||
2486 | |||||
2487 | return nullptr; | ||||
2488 | } | ||||
2489 | |||||
2490 | static Instruction *hoistFNegAboveFMulFDiv(Instruction &I, | ||||
2491 | InstCombiner::BuilderTy &Builder) { | ||||
2492 | Value *FNeg; | ||||
2493 | if (!match(&I, m_FNeg(m_Value(FNeg)))) | ||||
2494 | return nullptr; | ||||
2495 | |||||
2496 | Value *X, *Y; | ||||
2497 | if (match(FNeg, m_OneUse(m_FMul(m_Value(X), m_Value(Y))))) | ||||
2498 | return BinaryOperator::CreateFMulFMF(Builder.CreateFNegFMF(X, &I), Y, &I); | ||||
2499 | |||||
2500 | if (match(FNeg, m_OneUse(m_FDiv(m_Value(X), m_Value(Y))))) | ||||
2501 | return BinaryOperator::CreateFDivFMF(Builder.CreateFNegFMF(X, &I), Y, &I); | ||||
2502 | |||||
2503 | return nullptr; | ||||
2504 | } | ||||
2505 | |||||
2506 | Instruction *InstCombinerImpl::visitFNeg(UnaryOperator &I) { | ||||
2507 | Value *Op = I.getOperand(0); | ||||
2508 | |||||
2509 | if (Value *V = simplifyFNegInst(Op, I.getFastMathFlags(), | ||||
2510 | getSimplifyQuery().getWithInstruction(&I))) | ||||
2511 | return replaceInstUsesWith(I, V); | ||||
2512 | |||||
2513 | if (Instruction *X = foldFNegIntoConstant(I, DL)) | ||||
2514 | return X; | ||||
2515 | |||||
2516 | Value *X, *Y; | ||||
2517 | |||||
2518 | // If we can ignore the sign of zeros: -(X - Y) --> (Y - X) | ||||
2519 | if (I.hasNoSignedZeros() && | ||||
2520 | match(Op, m_OneUse(m_FSub(m_Value(X), m_Value(Y))))) | ||||
2521 | return BinaryOperator::CreateFSubFMF(Y, X, &I); | ||||
2522 | |||||
2523 | if (Instruction *R = hoistFNegAboveFMulFDiv(I, Builder)) | ||||
2524 | return R; | ||||
2525 | |||||
2526 | Value *OneUse; | ||||
2527 | if (!match(Op, m_OneUse(m_Value(OneUse)))) | ||||
2528 | return nullptr; | ||||
2529 | |||||
2530 | // Try to eliminate fneg if at least 1 arm of the select is negated. | ||||
2531 | Value *Cond; | ||||
2532 | if (match(OneUse, m_Select(m_Value(Cond), m_Value(X), m_Value(Y)))) { | ||||
2533 | // Unlike most transforms, this one is not safe to propagate nsz unless | ||||
2534 | // it is present on the original select. We union the flags from the select | ||||
2535 | // and fneg and then remove nsz if needed. | ||||
2536 | auto propagateSelectFMF = [&](SelectInst *S, bool CommonOperand) { | ||||
2537 | S->copyFastMathFlags(&I); | ||||
2538 | if (auto *OldSel = dyn_cast<SelectInst>(Op)) { | ||||
2539 | FastMathFlags FMF = I.getFastMathFlags(); | ||||
2540 | FMF |= OldSel->getFastMathFlags(); | ||||
2541 | S->setFastMathFlags(FMF); | ||||
2542 | if (!OldSel->hasNoSignedZeros() && !CommonOperand && | ||||
2543 | !isGuaranteedNotToBeUndefOrPoison(OldSel->getCondition())) | ||||
2544 | S->setHasNoSignedZeros(false); | ||||
2545 | } | ||||
2546 | }; | ||||
2547 | // -(Cond ? -P : Y) --> Cond ? P : -Y | ||||
2548 | Value *P; | ||||
2549 | if (match(X, m_FNeg(m_Value(P)))) { | ||||
2550 | Value *NegY = Builder.CreateFNegFMF(Y, &I, Y->getName() + ".neg"); | ||||
2551 | SelectInst *NewSel = SelectInst::Create(Cond, P, NegY); | ||||
2552 | propagateSelectFMF(NewSel, P == Y); | ||||
2553 | return NewSel; | ||||
2554 | } | ||||
2555 | // -(Cond ? X : -P) --> Cond ? -X : P | ||||
2556 | if (match(Y, m_FNeg(m_Value(P)))) { | ||||
2557 | Value *NegX = Builder.CreateFNegFMF(X, &I, X->getName() + ".neg"); | ||||
2558 | SelectInst *NewSel = SelectInst::Create(Cond, NegX, P); | ||||
2559 | propagateSelectFMF(NewSel, P == X); | ||||
2560 | return NewSel; | ||||
2561 | } | ||||
2562 | } | ||||
2563 | |||||
2564 | // fneg (copysign x, y) -> copysign x, (fneg y) | ||||
2565 | if (match(OneUse, m_CopySign(m_Value(X), m_Value(Y)))) { | ||||
2566 | // The source copysign has an additional value input, so we can't propagate | ||||
2567 | // flags the copysign doesn't also have. | ||||
2568 | FastMathFlags FMF = I.getFastMathFlags(); | ||||
2569 | FMF &= cast<FPMathOperator>(OneUse)->getFastMathFlags(); | ||||
2570 | |||||
2571 | IRBuilder<>::FastMathFlagGuard FMFGuard(Builder); | ||||
2572 | Builder.setFastMathFlags(FMF); | ||||
2573 | |||||
2574 | Value *NegY = Builder.CreateFNeg(Y); | ||||
2575 | Value *NewCopySign = Builder.CreateCopySign(X, NegY); | ||||
2576 | return replaceInstUsesWith(I, NewCopySign); | ||||
2577 | } | ||||
2578 | |||||
2579 | return nullptr; | ||||
2580 | } | ||||
2581 | |||||
2582 | Instruction *InstCombinerImpl::visitFSub(BinaryOperator &I) { | ||||
2583 | if (Value *V = simplifyFSubInst(I.getOperand(0), I.getOperand(1), | ||||
2584 | I.getFastMathFlags(), | ||||
2585 | getSimplifyQuery().getWithInstruction(&I))) | ||||
2586 | return replaceInstUsesWith(I, V); | ||||
2587 | |||||
2588 | if (Instruction *X = foldVectorBinop(I)) | ||||
2589 | return X; | ||||
2590 | |||||
2591 | if (Instruction *Phi = foldBinopWithPhiOperands(I)) | ||||
2592 | return Phi; | ||||
2593 | |||||
2594 | // Subtraction from -0.0 is the canonical form of fneg. | ||||
2595 | // fsub -0.0, X ==> fneg X | ||||
2596 | // fsub nsz 0.0, X ==> fneg nsz X | ||||
2597 | // | ||||
2598 | // FIXME This matcher does not respect FTZ or DAZ yet: | ||||
2599 | // fsub -0.0, Denorm ==> +-0 | ||||
2600 | // fneg Denorm ==> -Denorm | ||||
2601 | Value *Op; | ||||
2602 | if (match(&I, m_FNeg(m_Value(Op)))) | ||||
2603 | return UnaryOperator::CreateFNegFMF(Op, &I); | ||||
2604 | |||||
2605 | if (Instruction *X = foldFNegIntoConstant(I, DL)) | ||||
2606 | return X; | ||||
2607 | |||||
2608 | if (Instruction *R = hoistFNegAboveFMulFDiv(I, Builder)) | ||||
2609 | return R; | ||||
2610 | |||||
2611 | Value *X, *Y; | ||||
2612 | Constant *C; | ||||
2613 | |||||
2614 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||
2615 | // If Op0 is not -0.0 or we can ignore -0.0: Z - (X - Y) --> Z + (Y - X) | ||||
2616 | // Canonicalize to fadd to make analysis easier. | ||||
2617 | // This can also help codegen because fadd is commutative. | ||||
2618 | // Note that if this fsub was really an fneg, the fadd with -0.0 will get | ||||
2619 | // killed later. We still limit that particular transform with 'hasOneUse' | ||||
2620 | // because an fneg is assumed better/cheaper than a generic fsub. | ||||
2621 | if (I.hasNoSignedZeros() || CannotBeNegativeZero(Op0, SQ.TLI)) { | ||||
2622 | if (match(Op1, m_OneUse(m_FSub(m_Value(X), m_Value(Y))))) { | ||||
2623 | Value *NewSub = Builder.CreateFSubFMF(Y, X, &I); | ||||
2624 | return BinaryOperator::CreateFAddFMF(Op0, NewSub, &I); | ||||
2625 | } | ||||
2626 | } | ||||
2627 | |||||
2628 | // (-X) - Op1 --> -(X + Op1) | ||||
2629 | if (I.hasNoSignedZeros() && !isa<ConstantExpr>(Op0) && | ||||
2630 | match(Op0, m_OneUse(m_FNeg(m_Value(X))))) { | ||||
2631 | Value *FAdd = Builder.CreateFAddFMF(X, Op1, &I); | ||||
2632 | return UnaryOperator::CreateFNegFMF(FAdd, &I); | ||||
2633 | } | ||||
2634 | |||||
2635 | if (isa<Constant>(Op0)) | ||||
2636 | if (SelectInst *SI = dyn_cast<SelectInst>(Op1)) | ||||
2637 | if (Instruction *NV = FoldOpIntoSelect(I, SI)) | ||||
2638 | return NV; | ||||
2639 | |||||
2640 | // X - C --> X + (-C) | ||||
2641 | // But don't transform constant expressions because there's an inverse fold | ||||
2642 | // for X + (-Y) --> X - Y. | ||||
2643 | if (match(Op1, m_ImmConstant(C))) | ||||
2644 | if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL)) | ||||
2645 | return BinaryOperator::CreateFAddFMF(Op0, NegC, &I); | ||||
2646 | |||||
2647 | // X - (-Y) --> X + Y | ||||
2648 | if (match(Op1, m_FNeg(m_Value(Y)))) | ||||
2649 | return BinaryOperator::CreateFAddFMF(Op0, Y, &I); | ||||
2650 | |||||
2651 | // Similar to above, but look through a cast of the negated value: | ||||
2652 | // X - (fptrunc(-Y)) --> X + fptrunc(Y) | ||||
2653 | Type *Ty = I.getType(); | ||||
2654 | if (match(Op1, m_OneUse(m_FPTrunc(m_FNeg(m_Value(Y)))))) | ||||
2655 | return BinaryOperator::CreateFAddFMF(Op0, Builder.CreateFPTrunc(Y, Ty), &I); | ||||
2656 | |||||
2657 | // X - (fpext(-Y)) --> X + fpext(Y) | ||||
2658 | if (match(Op1, m_OneUse(m_FPExt(m_FNeg(m_Value(Y)))))) | ||||
2659 | return BinaryOperator::CreateFAddFMF(Op0, Builder.CreateFPExt(Y, Ty), &I); | ||||
2660 | |||||
2661 | // Similar to above, but look through fmul/fdiv of the negated value: | ||||
2662 | // Op0 - (-X * Y) --> Op0 + (X * Y) | ||||
2663 | // Op0 - (Y * -X) --> Op0 + (X * Y) | ||||
2664 | if (match(Op1, m_OneUse(m_c_FMul(m_FNeg(m_Value(X)), m_Value(Y))))) { | ||||
2665 | Value *FMul = Builder.CreateFMulFMF(X, Y, &I); | ||||
2666 | return BinaryOperator::CreateFAddFMF(Op0, FMul, &I); | ||||
2667 | } | ||||
2668 | // Op0 - (-X / Y) --> Op0 + (X / Y) | ||||
2669 | // Op0 - (X / -Y) --> Op0 + (X / Y) | ||||
2670 | if (match(Op1, m_OneUse(m_FDiv(m_FNeg(m_Value(X)), m_Value(Y)))) || | ||||
2671 | match(Op1, m_OneUse(m_FDiv(m_Value(X), m_FNeg(m_Value(Y)))))) { | ||||
2672 | Value *FDiv = Builder.CreateFDivFMF(X, Y, &I); | ||||
2673 | return BinaryOperator::CreateFAddFMF(Op0, FDiv, &I); | ||||
2674 | } | ||||
2675 | |||||
2676 | // Handle special cases for FSub with selects feeding the operation | ||||
2677 | if (Value *V = SimplifySelectsFeedingBinaryOp(I, Op0, Op1)) | ||||
2678 | return replaceInstUsesWith(I, V); | ||||
2679 | |||||
2680 | if (I.hasAllowReassoc() && I.hasNoSignedZeros()) { | ||||
2681 | // (Y - X) - Y --> -X | ||||
2682 | if (match(Op0, m_FSub(m_Specific(Op1), m_Value(X)))) | ||||
2683 | return UnaryOperator::CreateFNegFMF(X, &I); | ||||
2684 | |||||
2685 | // Y - (X + Y) --> -X | ||||
2686 | // Y - (Y + X) --> -X | ||||
2687 | if (match(Op1, m_c_FAdd(m_Specific(Op0), m_Value(X)))) | ||||
2688 | return UnaryOperator::CreateFNegFMF(X, &I); | ||||
2689 | |||||
2690 | // (X * C) - X --> X * (C - 1.0) | ||||
2691 | if (match(Op0, m_FMul(m_Specific(Op1), m_Constant(C)))) { | ||||
2692 | if (Constant *CSubOne = ConstantFoldBinaryOpOperands( | ||||
2693 | Instruction::FSub, C, ConstantFP::get(Ty, 1.0), DL)) | ||||
2694 | return BinaryOperator::CreateFMulFMF(Op1, CSubOne, &I); | ||||
2695 | } | ||||
2696 | // X - (X * C) --> X * (1.0 - C) | ||||
2697 | if (match(Op1, m_FMul(m_Specific(Op0), m_Constant(C)))) { | ||||
2698 | if (Constant *OneSubC = ConstantFoldBinaryOpOperands( | ||||
2699 | Instruction::FSub, ConstantFP::get(Ty, 1.0), C, DL)) | ||||
2700 | return BinaryOperator::CreateFMulFMF(Op0, OneSubC, &I); | ||||
2701 | } | ||||
2702 | |||||
2703 | // Reassociate fsub/fadd sequences to create more fadd instructions and | ||||
2704 | // reduce dependency chains: | ||||
2705 | // ((X - Y) + Z) - Op1 --> (X + Z) - (Y + Op1) | ||||
2706 | Value *Z; | ||||
2707 | if (match(Op0, m_OneUse(m_c_FAdd(m_OneUse(m_FSub(m_Value(X), m_Value(Y))), | ||||
2708 | m_Value(Z))))) { | ||||
2709 | Value *XZ = Builder.CreateFAddFMF(X, Z, &I); | ||||
2710 | Value *YW = Builder.CreateFAddFMF(Y, Op1, &I); | ||||
2711 | return BinaryOperator::CreateFSubFMF(XZ, YW, &I); | ||||
2712 | } | ||||
2713 | |||||
2714 | auto m_FaddRdx = [](Value *&Sum, Value *&Vec) { | ||||
2715 | return m_OneUse(m_Intrinsic<Intrinsic::vector_reduce_fadd>(m_Value(Sum), | ||||
2716 | m_Value(Vec))); | ||||
2717 | }; | ||||
2718 | Value *A0, *A1, *V0, *V1; | ||||
2719 | if (match(Op0, m_FaddRdx(A0, V0)) && match(Op1, m_FaddRdx(A1, V1)) && | ||||
2720 | V0->getType() == V1->getType()) { | ||||
2721 | // Difference of sums is sum of differences: | ||||
2722 | // add_rdx(A0, V0) - add_rdx(A1, V1) --> add_rdx(A0, V0 - V1) - A1 | ||||
2723 | Value *Sub = Builder.CreateFSubFMF(V0, V1, &I); | ||||
2724 | Value *Rdx = Builder.CreateIntrinsic(Intrinsic::vector_reduce_fadd, | ||||
2725 | {Sub->getType()}, {A0, Sub}, &I); | ||||
2726 | return BinaryOperator::CreateFSubFMF(Rdx, A1, &I); | ||||
2727 | } | ||||
2728 | |||||
2729 | if (Instruction *F = factorizeFAddFSub(I, Builder)) | ||||
2730 | return F; | ||||
2731 | |||||
2732 | // TODO: This performs reassociative folds for FP ops. Some fraction of the | ||||
2733 | // functionality has been subsumed by simple pattern matching here and in | ||||
2734 | // InstSimplify. We should let a dedicated reassociation pass handle more | ||||
2735 | // complex pattern matching and remove this from InstCombine. | ||||
2736 | if (Value *V = FAddCombine(Builder).simplify(&I)) | ||||
2737 | return replaceInstUsesWith(I, V); | ||||
2738 | |||||
2739 | // (X - Y) - Op1 --> X - (Y + Op1) | ||||
2740 | if (match(Op0, m_OneUse(m_FSub(m_Value(X), m_Value(Y))))) { | ||||
2741 | Value *FAdd = Builder.CreateFAddFMF(Y, Op1, &I); | ||||
2742 | return BinaryOperator::CreateFSubFMF(X, FAdd, &I); | ||||
2743 | } | ||||
2744 | } | ||||
2745 | |||||
2746 | return nullptr; | ||||
2747 | } |
1 | //===- PatternMatch.h - Match on the LLVM IR --------------------*- C++ -*-===// | ||||
2 | // | ||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||
6 | // | ||||
7 | //===----------------------------------------------------------------------===// | ||||
8 | // | ||||
9 | // This file provides a simple and efficient mechanism for performing general | ||||
10 | // tree-based pattern matches on the LLVM IR. The power of these routines is | ||||
11 | // that it allows you to write concise patterns that are expressive and easy to | ||||
12 | // understand. The other major advantage of this is that it allows you to | ||||
13 | // trivially capture/bind elements in the pattern to variables. For example, | ||||
14 | // you can do something like this: | ||||
15 | // | ||||
16 | // Value *Exp = ... | ||||
17 | // Value *X, *Y; ConstantInt *C1, *C2; // (X & C1) | (Y & C2) | ||||
18 | // if (match(Exp, m_Or(m_And(m_Value(X), m_ConstantInt(C1)), | ||||
19 | // m_And(m_Value(Y), m_ConstantInt(C2))))) { | ||||
20 | // ... Pattern is matched and variables are bound ... | ||||
21 | // } | ||||
22 | // | ||||
23 | // This is primarily useful to things like the instruction combiner, but can | ||||
24 | // also be useful for static analysis tools or code generators. | ||||
25 | // | ||||
26 | //===----------------------------------------------------------------------===// | ||||
27 | |||||
28 | #ifndef LLVM_IR_PATTERNMATCH_H | ||||
29 | #define LLVM_IR_PATTERNMATCH_H | ||||
30 | |||||
31 | #include "llvm/ADT/APFloat.h" | ||||
32 | #include "llvm/ADT/APInt.h" | ||||
33 | #include "llvm/IR/Constant.h" | ||||
34 | #include "llvm/IR/Constants.h" | ||||
35 | #include "llvm/IR/DataLayout.h" | ||||
36 | #include "llvm/IR/InstrTypes.h" | ||||
37 | #include "llvm/IR/Instruction.h" | ||||
38 | #include "llvm/IR/Instructions.h" | ||||
39 | #include "llvm/IR/IntrinsicInst.h" | ||||
40 | #include "llvm/IR/Intrinsics.h" | ||||
41 | #include "llvm/IR/Operator.h" | ||||
42 | #include "llvm/IR/Value.h" | ||||
43 | #include "llvm/Support/Casting.h" | ||||
44 | #include <cstdint> | ||||
45 | |||||
46 | namespace llvm { | ||||
47 | namespace PatternMatch { | ||||
48 | |||||
49 | template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) { | ||||
50 | return const_cast<Pattern &>(P).match(V); | ||||
51 | } | ||||
52 | |||||
53 | template <typename Pattern> bool match(ArrayRef<int> Mask, const Pattern &P) { | ||||
54 | return const_cast<Pattern &>(P).match(Mask); | ||||
55 | } | ||||
56 | |||||
57 | template <typename SubPattern_t> struct OneUse_match { | ||||
58 | SubPattern_t SubPattern; | ||||
59 | |||||
60 | OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {} | ||||
61 | |||||
62 | template <typename OpTy> bool match(OpTy *V) { | ||||
63 | return V->hasOneUse() && SubPattern.match(V); | ||||
64 | } | ||||
65 | }; | ||||
66 | |||||
67 | template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) { | ||||
68 | return SubPattern; | ||||
69 | } | ||||
70 | |||||
71 | template <typename Class> struct class_match { | ||||
72 | template <typename ITy> bool match(ITy *V) { return isa<Class>(V); } | ||||
73 | }; | ||||
74 | |||||
75 | /// Match an arbitrary value and ignore it. | ||||
76 | inline class_match<Value> m_Value() { return class_match<Value>(); } | ||||
77 | |||||
78 | /// Match an arbitrary unary operation and ignore it. | ||||
79 | inline class_match<UnaryOperator> m_UnOp() { | ||||
80 | return class_match<UnaryOperator>(); | ||||
81 | } | ||||
82 | |||||
83 | /// Match an arbitrary binary operation and ignore it. | ||||
84 | inline class_match<BinaryOperator> m_BinOp() { | ||||
85 | return class_match<BinaryOperator>(); | ||||
86 | } | ||||
87 | |||||
88 | /// Matches any compare instruction and ignore it. | ||||
89 | inline class_match<CmpInst> m_Cmp() { return class_match<CmpInst>(); } | ||||
90 | |||||
91 | struct undef_match { | ||||
92 | static bool check(const Value *V) { | ||||
93 | if (isa<UndefValue>(V)) | ||||
94 | return true; | ||||
95 | |||||
96 | const auto *CA = dyn_cast<ConstantAggregate>(V); | ||||
97 | if (!CA) | ||||
98 | return false; | ||||
99 | |||||
100 | SmallPtrSet<const ConstantAggregate *, 8> Seen; | ||||
101 | SmallVector<const ConstantAggregate *, 8> Worklist; | ||||
102 | |||||
103 | // Either UndefValue, PoisonValue, or an aggregate that only contains | ||||
104 | // these is accepted by matcher. | ||||
105 | // CheckValue returns false if CA cannot satisfy this constraint. | ||||
106 | auto CheckValue = [&](const ConstantAggregate *CA) { | ||||
107 | for (const Value *Op : CA->operand_values()) { | ||||
108 | if (isa<UndefValue>(Op)) | ||||
109 | continue; | ||||
110 | |||||
111 | const auto *CA = dyn_cast<ConstantAggregate>(Op); | ||||
112 | if (!CA) | ||||
113 | return false; | ||||
114 | if (Seen.insert(CA).second) | ||||
115 | Worklist.emplace_back(CA); | ||||
116 | } | ||||
117 | |||||
118 | return true; | ||||
119 | }; | ||||
120 | |||||
121 | if (!CheckValue(CA)) | ||||
122 | return false; | ||||
123 | |||||
124 | while (!Worklist.empty()) { | ||||
125 | if (!CheckValue(Worklist.pop_back_val())) | ||||
126 | return false; | ||||
127 | } | ||||
128 | return true; | ||||
129 | } | ||||
130 | template <typename ITy> bool match(ITy *V) { return check(V); } | ||||
131 | }; | ||||
132 | |||||
133 | /// Match an arbitrary undef constant. This matches poison as well. | ||||
134 | /// If this is an aggregate and contains a non-aggregate element that is | ||||
135 | /// neither undef nor poison, the aggregate is not matched. | ||||
136 | inline auto m_Undef() { return undef_match(); } | ||||
137 | |||||
138 | /// Match an arbitrary poison constant. | ||||
139 | inline class_match<PoisonValue> m_Poison() { | ||||
140 | return class_match<PoisonValue>(); | ||||
141 | } | ||||
142 | |||||
143 | /// Match an arbitrary Constant and ignore it. | ||||
144 | inline class_match<Constant> m_Constant() { return class_match<Constant>(); } | ||||
145 | |||||
146 | /// Match an arbitrary ConstantInt and ignore it. | ||||
147 | inline class_match<ConstantInt> m_ConstantInt() { | ||||
148 | return class_match<ConstantInt>(); | ||||
149 | } | ||||
150 | |||||
151 | /// Match an arbitrary ConstantFP and ignore it. | ||||
152 | inline class_match<ConstantFP> m_ConstantFP() { | ||||
153 | return class_match<ConstantFP>(); | ||||
154 | } | ||||
155 | |||||
156 | struct constantexpr_match { | ||||
157 | template <typename ITy> bool match(ITy *V) { | ||||
158 | auto *C = dyn_cast<Constant>(V); | ||||
159 | return C && (isa<ConstantExpr>(C) || C->containsConstantExpression()); | ||||
160 | } | ||||
161 | }; | ||||
162 | |||||
163 | /// Match a constant expression or a constant that contains a constant | ||||
164 | /// expression. | ||||
165 | inline constantexpr_match m_ConstantExpr() { return constantexpr_match(); } | ||||
166 | |||||
167 | /// Match an arbitrary basic block value and ignore it. | ||||
168 | inline class_match<BasicBlock> m_BasicBlock() { | ||||
169 | return class_match<BasicBlock>(); | ||||
170 | } | ||||
171 | |||||
172 | /// Inverting matcher | ||||
173 | template <typename Ty> struct match_unless { | ||||
174 | Ty M; | ||||
175 | |||||
176 | match_unless(const Ty &Matcher) : M(Matcher) {} | ||||
177 | |||||
178 | template <typename ITy> bool match(ITy *V) { return !M.match(V); } | ||||
179 | }; | ||||
180 | |||||
181 | /// Match if the inner matcher does *NOT* match. | ||||
182 | template <typename Ty> inline match_unless<Ty> m_Unless(const Ty &M) { | ||||
183 | return match_unless<Ty>(M); | ||||
184 | } | ||||
185 | |||||
186 | /// Matching combinators | ||||
187 | template <typename LTy, typename RTy> struct match_combine_or { | ||||
188 | LTy L; | ||||
189 | RTy R; | ||||
190 | |||||
191 | match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} | ||||
192 | |||||
193 | template <typename ITy> bool match(ITy *V) { | ||||
194 | if (L.match(V)) | ||||
195 | return true; | ||||
196 | if (R.match(V)) | ||||
197 | return true; | ||||
198 | return false; | ||||
199 | } | ||||
200 | }; | ||||
201 | |||||
202 | template <typename LTy, typename RTy> struct match_combine_and { | ||||
203 | LTy L; | ||||
204 | RTy R; | ||||
205 | |||||
206 | match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} | ||||
207 | |||||
208 | template <typename ITy> bool match(ITy *V) { | ||||
209 | if (L.match(V)) | ||||
210 | if (R.match(V)) | ||||
211 | return true; | ||||
212 | return false; | ||||
213 | } | ||||
214 | }; | ||||
215 | |||||
216 | /// Combine two pattern matchers matching L || R | ||||
217 | template <typename LTy, typename RTy> | ||||
218 | inline match_combine_or<LTy, RTy> m_CombineOr(const LTy &L, const RTy &R) { | ||||
219 | return match_combine_or<LTy, RTy>(L, R); | ||||
220 | } | ||||
221 | |||||
222 | /// Combine two pattern matchers matching L && R | ||||
223 | template <typename LTy, typename RTy> | ||||
224 | inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) { | ||||
225 | return match_combine_and<LTy, RTy>(L, R); | ||||
226 | } | ||||
227 | |||||
228 | struct apint_match { | ||||
229 | const APInt *&Res; | ||||
230 | bool AllowUndef; | ||||
231 | |||||
232 | apint_match(const APInt *&Res, bool AllowUndef) | ||||
233 | : Res(Res), AllowUndef(AllowUndef) {} | ||||
234 | |||||
235 | template <typename ITy> bool match(ITy *V) { | ||||
236 | if (auto *CI = dyn_cast<ConstantInt>(V)) { | ||||
237 | Res = &CI->getValue(); | ||||
238 | return true; | ||||
239 | } | ||||
240 | if (V->getType()->isVectorTy()) | ||||
241 | if (const auto *C = dyn_cast<Constant>(V)) | ||||
242 | if (auto *CI = | ||||
243 | dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndef))) { | ||||
244 | Res = &CI->getValue(); | ||||
245 | return true; | ||||
246 | } | ||||
247 | return false; | ||||
248 | } | ||||
249 | }; | ||||
250 | // Either constexpr if or renaming ConstantFP::getValueAPF to | ||||
251 | // ConstantFP::getValue is needed to do it via single template | ||||
252 | // function for both apint/apfloat. | ||||
253 | struct apfloat_match { | ||||
254 | const APFloat *&Res; | ||||
255 | bool AllowUndef; | ||||
256 | |||||
257 | apfloat_match(const APFloat *&Res, bool AllowUndef) | ||||
258 | : Res(Res), AllowUndef(AllowUndef) {} | ||||
259 | |||||
260 | template <typename ITy> bool match(ITy *V) { | ||||
261 | if (auto *CI = dyn_cast<ConstantFP>(V)) { | ||||
262 | Res = &CI->getValueAPF(); | ||||
263 | return true; | ||||
264 | } | ||||
265 | if (V->getType()->isVectorTy()) | ||||
266 | if (const auto *C = dyn_cast<Constant>(V)) | ||||
267 | if (auto *CI = | ||||
268 | dyn_cast_or_null<ConstantFP>(C->getSplatValue(AllowUndef))) { | ||||
269 | Res = &CI->getValueAPF(); | ||||
270 | return true; | ||||
271 | } | ||||
272 | return false; | ||||
273 | } | ||||
274 | }; | ||||
275 | |||||
276 | /// Match a ConstantInt or splatted ConstantVector, binding the | ||||
277 | /// specified pointer to the contained APInt. | ||||
278 | inline apint_match m_APInt(const APInt *&Res) { | ||||
279 | // Forbid undefs by default to maintain previous behavior. | ||||
280 | return apint_match(Res, /* AllowUndef */ false); | ||||
281 | } | ||||
282 | |||||
283 | /// Match APInt while allowing undefs in splat vector constants. | ||||
284 | inline apint_match m_APIntAllowUndef(const APInt *&Res) { | ||||
285 | return apint_match(Res, /* AllowUndef */ true); | ||||
286 | } | ||||
287 | |||||
288 | /// Match APInt while forbidding undefs in splat vector constants. | ||||
289 | inline apint_match m_APIntForbidUndef(const APInt *&Res) { | ||||
290 | return apint_match(Res, /* AllowUndef */ false); | ||||
291 | } | ||||
292 | |||||
293 | /// Match a ConstantFP or splatted ConstantVector, binding the | ||||
294 | /// specified pointer to the contained APFloat. | ||||
295 | inline apfloat_match m_APFloat(const APFloat *&Res) { | ||||
296 | // Forbid undefs by default to maintain previous behavior. | ||||
297 | return apfloat_match(Res, /* AllowUndef */ false); | ||||
298 | } | ||||
299 | |||||
300 | /// Match APFloat while allowing undefs in splat vector constants. | ||||
301 | inline apfloat_match m_APFloatAllowUndef(const APFloat *&Res) { | ||||
302 | return apfloat_match(Res, /* AllowUndef */ true); | ||||
303 | } | ||||
304 | |||||
305 | /// Match APFloat while forbidding undefs in splat vector constants. | ||||
306 | inline apfloat_match m_APFloatForbidUndef(const APFloat *&Res) { | ||||
307 | return apfloat_match(Res, /* AllowUndef */ false); | ||||
308 | } | ||||
309 | |||||
310 | template <int64_t Val> struct constantint_match { | ||||
311 | template <typename ITy> bool match(ITy *V) { | ||||
312 | if (const auto *CI = dyn_cast<ConstantInt>(V)) { | ||||
313 | const APInt &CIV = CI->getValue(); | ||||
314 | if (Val >= 0) | ||||
315 | return CIV == static_cast<uint64_t>(Val); | ||||
316 | // If Val is negative, and CI is shorter than it, truncate to the right | ||||
317 | // number of bits. If it is larger, then we have to sign extend. Just | ||||
318 | // compare their negated values. | ||||
319 | return -CIV == -Val; | ||||
320 | } | ||||
321 | return false; | ||||
322 | } | ||||
323 | }; | ||||
324 | |||||
325 | /// Match a ConstantInt with a specific value. | ||||
326 | template <int64_t Val> inline constantint_match<Val> m_ConstantInt() { | ||||
327 | return constantint_match<Val>(); | ||||
328 | } | ||||
329 | |||||
330 | /// This helper class is used to match constant scalars, vector splats, | ||||
331 | /// and fixed width vectors that satisfy a specified predicate. | ||||
332 | /// For fixed width vector constants, undefined elements are ignored. | ||||
333 | template <typename Predicate, typename ConstantVal> | ||||
334 | struct cstval_pred_ty : public Predicate { | ||||
335 | template <typename ITy> bool match(ITy *V) { | ||||
336 | if (const auto *CV = dyn_cast<ConstantVal>(V)) | ||||
337 | return this->isValue(CV->getValue()); | ||||
338 | if (const auto *VTy = dyn_cast<VectorType>(V->getType())) { | ||||
339 | if (const auto *C = dyn_cast<Constant>(V)) { | ||||
340 | if (const auto *CV = dyn_cast_or_null<ConstantVal>(C->getSplatValue())) | ||||
341 | return this->isValue(CV->getValue()); | ||||
342 | |||||
343 | // Number of elements of a scalable vector unknown at compile time | ||||
344 | auto *FVTy = dyn_cast<FixedVectorType>(VTy); | ||||
345 | if (!FVTy) | ||||
346 | return false; | ||||
347 | |||||
348 | // Non-splat vector constant: check each element for a match. | ||||
349 | unsigned NumElts = FVTy->getNumElements(); | ||||
350 | assert(NumElts != 0 && "Constant vector with no elements?")(static_cast <bool> (NumElts != 0 && "Constant vector with no elements?" ) ? void (0) : __assert_fail ("NumElts != 0 && \"Constant vector with no elements?\"" , "llvm/include/llvm/IR/PatternMatch.h", 350, __extension__ __PRETTY_FUNCTION__ )); | ||||
351 | bool HasNonUndefElements = false; | ||||
352 | for (unsigned i = 0; i != NumElts; ++i) { | ||||
353 | Constant *Elt = C->getAggregateElement(i); | ||||
354 | if (!Elt) | ||||
355 | return false; | ||||
356 | if (isa<UndefValue>(Elt)) | ||||
357 | continue; | ||||
358 | auto *CV = dyn_cast<ConstantVal>(Elt); | ||||
359 | if (!CV || !this->isValue(CV->getValue())) | ||||
360 | return false; | ||||
361 | HasNonUndefElements = true; | ||||
362 | } | ||||
363 | return HasNonUndefElements; | ||||
364 | } | ||||
365 | } | ||||
366 | return false; | ||||
367 | } | ||||
368 | }; | ||||
369 | |||||
370 | /// specialization of cstval_pred_ty for ConstantInt | ||||
371 | template <typename Predicate> | ||||
372 | using cst_pred_ty = cstval_pred_ty<Predicate, ConstantInt>; | ||||
373 | |||||
374 | /// specialization of cstval_pred_ty for ConstantFP | ||||
375 | template <typename Predicate> | ||||
376 | using cstfp_pred_ty = cstval_pred_ty<Predicate, ConstantFP>; | ||||
377 | |||||
378 | /// This helper class is used to match scalar and vector constants that | ||||
379 | /// satisfy a specified predicate, and bind them to an APInt. | ||||
380 | template <typename Predicate> struct api_pred_ty : public Predicate { | ||||
381 | const APInt *&Res; | ||||
382 | |||||
383 | api_pred_ty(const APInt *&R) : Res(R) {} | ||||
384 | |||||
385 | template <typename ITy> bool match(ITy *V) { | ||||
386 | if (const auto *CI = dyn_cast<ConstantInt>(V)) | ||||
387 | if (this->isValue(CI->getValue())) { | ||||
388 | Res = &CI->getValue(); | ||||
389 | return true; | ||||
390 | } | ||||
391 | if (V->getType()->isVectorTy()) | ||||
392 | if (const auto *C = dyn_cast<Constant>(V)) | ||||
393 | if (auto *CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue())) | ||||
394 | if (this->isValue(CI->getValue())) { | ||||
395 | Res = &CI->getValue(); | ||||
396 | return true; | ||||
397 | } | ||||
398 | |||||
399 | return false; | ||||
400 | } | ||||
401 | }; | ||||
402 | |||||
403 | /// This helper class is used to match scalar and vector constants that | ||||
404 | /// satisfy a specified predicate, and bind them to an APFloat. | ||||
405 | /// Undefs are allowed in splat vector constants. | ||||
406 | template <typename Predicate> struct apf_pred_ty : public Predicate { | ||||
407 | const APFloat *&Res; | ||||
408 | |||||
409 | apf_pred_ty(const APFloat *&R) : Res(R) {} | ||||
410 | |||||
411 | template <typename ITy> bool match(ITy *V) { | ||||
412 | if (const auto *CI = dyn_cast<ConstantFP>(V)) | ||||
413 | if (this->isValue(CI->getValue())) { | ||||
414 | Res = &CI->getValue(); | ||||
415 | return true; | ||||
416 | } | ||||
417 | if (V->getType()->isVectorTy()) | ||||
418 | if (const auto *C = dyn_cast<Constant>(V)) | ||||
419 | if (auto *CI = dyn_cast_or_null<ConstantFP>( | ||||
420 | C->getSplatValue(/* AllowUndef */ true))) | ||||
421 | if (this->isValue(CI->getValue())) { | ||||
422 | Res = &CI->getValue(); | ||||
423 | return true; | ||||
424 | } | ||||
425 | |||||
426 | return false; | ||||
427 | } | ||||
428 | }; | ||||
429 | |||||
430 | /////////////////////////////////////////////////////////////////////////////// | ||||
431 | // | ||||
432 | // Encapsulate constant value queries for use in templated predicate matchers. | ||||
433 | // This allows checking if constants match using compound predicates and works | ||||
434 | // with vector constants, possibly with relaxed constraints. For example, ignore | ||||
435 | // undef values. | ||||
436 | // | ||||
437 | /////////////////////////////////////////////////////////////////////////////// | ||||
438 | |||||
439 | struct is_any_apint { | ||||
440 | bool isValue(const APInt &C) { return true; } | ||||
441 | }; | ||||
442 | /// Match an integer or vector with any integral constant. | ||||
443 | /// For vectors, this includes constants with undefined elements. | ||||
444 | inline cst_pred_ty<is_any_apint> m_AnyIntegralConstant() { | ||||
445 | return cst_pred_ty<is_any_apint>(); | ||||
446 | } | ||||
447 | |||||
448 | struct is_all_ones { | ||||
449 | bool isValue(const APInt &C) { return C.isAllOnes(); } | ||||
450 | }; | ||||
451 | /// Match an integer or vector with all bits set. | ||||
452 | /// For vectors, this includes constants with undefined elements. | ||||
453 | inline cst_pred_ty<is_all_ones> m_AllOnes() { | ||||
454 | return cst_pred_ty<is_all_ones>(); | ||||
455 | } | ||||
456 | |||||
457 | struct is_maxsignedvalue { | ||||
458 | bool isValue(const APInt &C) { return C.isMaxSignedValue(); } | ||||
459 | }; | ||||
460 | /// Match an integer or vector with values having all bits except for the high | ||||
461 | /// bit set (0x7f...). | ||||
462 | /// For vectors, this includes constants with undefined elements. | ||||
463 | inline cst_pred_ty<is_maxsignedvalue> m_MaxSignedValue() { | ||||
464 | return cst_pred_ty<is_maxsignedvalue>(); | ||||
465 | } | ||||
466 | inline api_pred_ty<is_maxsignedvalue> m_MaxSignedValue(const APInt *&V) { | ||||
467 | return V; | ||||
468 | } | ||||
469 | |||||
470 | struct is_negative { | ||||
471 | bool isValue(const APInt &C) { return C.isNegative(); } | ||||
472 | }; | ||||
473 | /// Match an integer or vector of negative values. | ||||
474 | /// For vectors, this includes constants with undefined elements. | ||||
475 | inline cst_pred_ty<is_negative> m_Negative() { | ||||
476 | return cst_pred_ty<is_negative>(); | ||||
477 | } | ||||
478 | inline api_pred_ty<is_negative> m_Negative(const APInt *&V) { return V; } | ||||
479 | |||||
480 | struct is_nonnegative { | ||||
481 | bool isValue(const APInt &C) { return C.isNonNegative(); } | ||||
482 | }; | ||||
483 | /// Match an integer or vector of non-negative values. | ||||
484 | /// For vectors, this includes constants with undefined elements. | ||||
485 | inline cst_pred_ty<is_nonnegative> m_NonNegative() { | ||||
486 | return cst_pred_ty<is_nonnegative>(); | ||||
487 | } | ||||
488 | inline api_pred_ty<is_nonnegative> m_NonNegative(const APInt *&V) { return V; } | ||||
489 | |||||
490 | struct is_strictlypositive { | ||||
491 | bool isValue(const APInt &C) { return C.isStrictlyPositive(); } | ||||
492 | }; | ||||
493 | /// Match an integer or vector of strictly positive values. | ||||
494 | /// For vectors, this includes constants with undefined elements. | ||||
495 | inline cst_pred_ty<is_strictlypositive> m_StrictlyPositive() { | ||||
496 | return cst_pred_ty<is_strictlypositive>(); | ||||
497 | } | ||||
498 | inline api_pred_ty<is_strictlypositive> m_StrictlyPositive(const APInt *&V) { | ||||
499 | return V; | ||||
500 | } | ||||
501 | |||||
502 | struct is_nonpositive { | ||||
503 | bool isValue(const APInt &C) { return C.isNonPositive(); } | ||||
504 | }; | ||||
505 | /// Match an integer or vector of non-positive values. | ||||
506 | /// For vectors, this includes constants with undefined elements. | ||||
507 | inline cst_pred_ty<is_nonpositive> m_NonPositive() { | ||||
508 | return cst_pred_ty<is_nonpositive>(); | ||||
509 | } | ||||
510 | inline api_pred_ty<is_nonpositive> m_NonPositive(const APInt *&V) { return V; } | ||||
511 | |||||
512 | struct is_one { | ||||
513 | bool isValue(const APInt &C) { return C.isOne(); } | ||||
514 | }; | ||||
515 | /// Match an integer 1 or a vector with all elements equal to 1. | ||||
516 | /// For vectors, this includes constants with undefined elements. | ||||
517 | inline cst_pred_ty<is_one> m_One() { return cst_pred_ty<is_one>(); } | ||||
518 | |||||
519 | struct is_zero_int { | ||||
520 | bool isValue(const APInt &C) { return C.isZero(); } | ||||
521 | }; | ||||
522 | /// Match an integer 0 or a vector with all elements equal to 0. | ||||
523 | /// For vectors, this includes constants with undefined elements. | ||||
524 | inline cst_pred_ty<is_zero_int> m_ZeroInt() { | ||||
525 | return cst_pred_ty<is_zero_int>(); | ||||
526 | } | ||||
527 | |||||
528 | struct is_zero { | ||||
529 | template <typename ITy> bool match(ITy *V) { | ||||
530 | auto *C = dyn_cast<Constant>(V); | ||||
531 | // FIXME: this should be able to do something for scalable vectors | ||||
532 | return C && (C->isNullValue() || cst_pred_ty<is_zero_int>().match(C)); | ||||
533 | } | ||||
534 | }; | ||||
535 | /// Match any null constant or a vector with all elements equal to 0. | ||||
536 | /// For vectors, this includes constants with undefined elements. | ||||
537 | inline is_zero m_Zero() { return is_zero(); } | ||||
538 | |||||
539 | struct is_power2 { | ||||
540 | bool isValue(const APInt &C) { return C.isPowerOf2(); } | ||||
541 | }; | ||||
542 | /// Match an integer or vector power-of-2. | ||||
543 | /// For vectors, this includes constants with undefined elements. | ||||
544 | inline cst_pred_ty<is_power2> m_Power2() { return cst_pred_ty<is_power2>(); } | ||||
545 | inline api_pred_ty<is_power2> m_Power2(const APInt *&V) { return V; } | ||||
546 | |||||
547 | struct is_negated_power2 { | ||||
548 | bool isValue(const APInt &C) { return C.isNegatedPowerOf2(); } | ||||
549 | }; | ||||
550 | /// Match a integer or vector negated power-of-2. | ||||
551 | /// For vectors, this includes constants with undefined elements. | ||||
552 | inline cst_pred_ty<is_negated_power2> m_NegatedPower2() { | ||||
553 | return cst_pred_ty<is_negated_power2>(); | ||||
554 | } | ||||
555 | inline api_pred_ty<is_negated_power2> m_NegatedPower2(const APInt *&V) { | ||||
556 | return V; | ||||
557 | } | ||||
558 | |||||
559 | struct is_power2_or_zero { | ||||
560 | bool isValue(const APInt &C) { return !C || C.isPowerOf2(); } | ||||
561 | }; | ||||
562 | /// Match an integer or vector of 0 or power-of-2 values. | ||||
563 | /// For vectors, this includes constants with undefined elements. | ||||
564 | inline cst_pred_ty<is_power2_or_zero> m_Power2OrZero() { | ||||
565 | return cst_pred_ty<is_power2_or_zero>(); | ||||
566 | } | ||||
567 | inline api_pred_ty<is_power2_or_zero> m_Power2OrZero(const APInt *&V) { | ||||
568 | return V; | ||||
569 | } | ||||
570 | |||||
571 | struct is_sign_mask { | ||||
572 | bool isValue(const APInt &C) { return C.isSignMask(); } | ||||
573 | }; | ||||
574 | /// Match an integer or vector with only the sign bit(s) set. | ||||
575 | /// For vectors, this includes constants with undefined elements. | ||||
576 | inline cst_pred_ty<is_sign_mask> m_SignMask() { | ||||
577 | return cst_pred_ty<is_sign_mask>(); | ||||
578 | } | ||||
579 | |||||
580 | struct is_lowbit_mask { | ||||
581 | bool isValue(const APInt &C) { return C.isMask(); } | ||||
582 | }; | ||||
583 | /// Match an integer or vector with only the low bit(s) set. | ||||
584 | /// For vectors, this includes constants with undefined elements. | ||||
585 | inline cst_pred_ty<is_lowbit_mask> m_LowBitMask() { | ||||
586 | return cst_pred_ty<is_lowbit_mask>(); | ||||
587 | } | ||||
588 | inline api_pred_ty<is_lowbit_mask> m_LowBitMask(const APInt *&V) { return V; } | ||||
589 | |||||
590 | struct icmp_pred_with_threshold { | ||||
591 | ICmpInst::Predicate Pred; | ||||
592 | const APInt *Thr; | ||||
593 | bool isValue(const APInt &C) { return ICmpInst::compare(C, *Thr, Pred); } | ||||
594 | }; | ||||
595 | /// Match an integer or vector with every element comparing 'pred' (eg/ne/...) | ||||
596 | /// to Threshold. For vectors, this includes constants with undefined elements. | ||||
597 | inline cst_pred_ty<icmp_pred_with_threshold> | ||||
598 | m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold) { | ||||
599 | cst_pred_ty<icmp_pred_with_threshold> P; | ||||
600 | P.Pred = Predicate; | ||||
601 | P.Thr = &Threshold; | ||||
602 | return P; | ||||
603 | } | ||||
604 | |||||
605 | struct is_nan { | ||||
606 | bool isValue(const APFloat &C) { return C.isNaN(); } | ||||
607 | }; | ||||
608 | /// Match an arbitrary NaN constant. This includes quiet and signalling nans. | ||||
609 | /// For vectors, this includes constants with undefined elements. | ||||
610 | inline cstfp_pred_ty<is_nan> m_NaN() { return cstfp_pred_ty<is_nan>(); } | ||||
611 | |||||
612 | struct is_nonnan { | ||||
613 | bool isValue(const APFloat &C) { return !C.isNaN(); } | ||||
614 | }; | ||||
615 | /// Match a non-NaN FP constant. | ||||
616 | /// For vectors, this includes constants with undefined elements. | ||||
617 | inline cstfp_pred_ty<is_nonnan> m_NonNaN() { | ||||
618 | return cstfp_pred_ty<is_nonnan>(); | ||||
619 | } | ||||
620 | |||||
621 | struct is_inf { | ||||
622 | bool isValue(const APFloat &C) { return C.isInfinity(); } | ||||
623 | }; | ||||
624 | /// Match a positive or negative infinity FP constant. | ||||
625 | /// For vectors, this includes constants with undefined elements. | ||||
626 | inline cstfp_pred_ty<is_inf> m_Inf() { return cstfp_pred_ty<is_inf>(); } | ||||
627 | |||||
628 | struct is_noninf { | ||||
629 | bool isValue(const APFloat &C) { return !C.isInfinity(); } | ||||
630 | }; | ||||
631 | /// Match a non-infinity FP constant, i.e. finite or NaN. | ||||
632 | /// For vectors, this includes constants with undefined elements. | ||||
633 | inline cstfp_pred_ty<is_noninf> m_NonInf() { | ||||
634 | return cstfp_pred_ty<is_noninf>(); | ||||
635 | } | ||||
636 | |||||
637 | struct is_finite { | ||||
638 | bool isValue(const APFloat &C) { return C.isFinite(); } | ||||
639 | }; | ||||
640 | /// Match a finite FP constant, i.e. not infinity or NaN. | ||||
641 | /// For vectors, this includes constants with undefined elements. | ||||
642 | inline cstfp_pred_ty<is_finite> m_Finite() { | ||||
643 | return cstfp_pred_ty<is_finite>(); | ||||
644 | } | ||||
645 | inline apf_pred_ty<is_finite> m_Finite(const APFloat *&V) { return V; } | ||||
646 | |||||
647 | struct is_finitenonzero { | ||||
648 | bool isValue(const APFloat &C) { return C.isFiniteNonZero(); } | ||||
649 | }; | ||||
650 | /// Match a finite non-zero FP constant. | ||||
651 | /// For vectors, this includes constants with undefined elements. | ||||
652 | inline cstfp_pred_ty<is_finitenonzero> m_FiniteNonZero() { | ||||
653 | return cstfp_pred_ty<is_finitenonzero>(); | ||||
654 | } | ||||
655 | inline apf_pred_ty<is_finitenonzero> m_FiniteNonZero(const APFloat *&V) { | ||||
656 | return V; | ||||
657 | } | ||||
658 | |||||
659 | struct is_any_zero_fp { | ||||
660 | bool isValue(const APFloat &C) { return C.isZero(); } | ||||
661 | }; | ||||
662 | /// Match a floating-point negative zero or positive zero. | ||||
663 | /// For vectors, this includes constants with undefined elements. | ||||
664 | inline cstfp_pred_ty<is_any_zero_fp> m_AnyZeroFP() { | ||||
665 | return cstfp_pred_ty<is_any_zero_fp>(); | ||||
666 | } | ||||
667 | |||||
668 | struct is_pos_zero_fp { | ||||
669 | bool isValue(const APFloat &C) { return C.isPosZero(); } | ||||
670 | }; | ||||
671 | /// Match a floating-point positive zero. | ||||
672 | /// For vectors, this includes constants with undefined elements. | ||||
673 | inline cstfp_pred_ty<is_pos_zero_fp> m_PosZeroFP() { | ||||
674 | return cstfp_pred_ty<is_pos_zero_fp>(); | ||||
675 | } | ||||
676 | |||||
677 | struct is_neg_zero_fp { | ||||
678 | bool isValue(const APFloat &C) { return C.isNegZero(); } | ||||
679 | }; | ||||
680 | /// Match a floating-point negative zero. | ||||
681 | /// For vectors, this includes constants with undefined elements. | ||||
682 | inline cstfp_pred_ty<is_neg_zero_fp> m_NegZeroFP() { | ||||
683 | return cstfp_pred_ty<is_neg_zero_fp>(); | ||||
684 | } | ||||
685 | |||||
686 | struct is_non_zero_fp { | ||||
687 | bool isValue(const APFloat &C) { return C.isNonZero(); } | ||||
688 | }; | ||||
689 | /// Match a floating-point non-zero. | ||||
690 | /// For vectors, this includes constants with undefined elements. | ||||
691 | inline cstfp_pred_ty<is_non_zero_fp> m_NonZeroFP() { | ||||
692 | return cstfp_pred_ty<is_non_zero_fp>(); | ||||
693 | } | ||||
694 | |||||
695 | /////////////////////////////////////////////////////////////////////////////// | ||||
696 | |||||
697 | template <typename Class> struct bind_ty { | ||||
698 | Class *&VR; | ||||
699 | |||||
700 | bind_ty(Class *&V) : VR(V) {} | ||||
701 | |||||
702 | template <typename ITy> bool match(ITy *V) { | ||||
703 | if (auto *CV = dyn_cast<Class>(V)) { | ||||
704 | VR = CV; | ||||
705 | return true; | ||||
706 | } | ||||
707 | return false; | ||||
708 | } | ||||
709 | }; | ||||
710 | |||||
711 | /// Match a value, capturing it if we match. | ||||
712 | inline bind_ty<Value> m_Value(Value *&V) { return V; } | ||||
713 | inline bind_ty<const Value> m_Value(const Value *&V) { return V; } | ||||
714 | |||||
715 | /// Match an instruction, capturing it if we match. | ||||
716 | inline bind_ty<Instruction> m_Instruction(Instruction *&I) { return I; } | ||||
717 | /// Match a unary operator, capturing it if we match. | ||||
718 | inline bind_ty<UnaryOperator> m_UnOp(UnaryOperator *&I) { return I; } | ||||
719 | /// Match a binary operator, capturing it if we match. | ||||
720 | inline bind_ty<BinaryOperator> m_BinOp(BinaryOperator *&I) { return I; } | ||||
721 | /// Match a with overflow intrinsic, capturing it if we match. | ||||
722 | inline bind_ty<WithOverflowInst> m_WithOverflowInst(WithOverflowInst *&I) { | ||||
723 | return I; | ||||
724 | } | ||||
725 | inline bind_ty<const WithOverflowInst> | ||||
726 | m_WithOverflowInst(const WithOverflowInst *&I) { | ||||
727 | return I; | ||||
728 | } | ||||
729 | |||||
730 | /// Match a Constant, capturing the value if we match. | ||||
731 | inline bind_ty<Constant> m_Constant(Constant *&C) { return C; } | ||||
732 | |||||
733 | /// Match a ConstantInt, capturing the value if we match. | ||||
734 | inline bind_ty<ConstantInt> m_ConstantInt(ConstantInt *&CI) { return CI; } | ||||
735 | |||||
736 | /// Match a ConstantFP, capturing the value if we match. | ||||
737 | inline bind_ty<ConstantFP> m_ConstantFP(ConstantFP *&C) { return C; } | ||||
738 | |||||
739 | /// Match a ConstantExpr, capturing the value if we match. | ||||
740 | inline bind_ty<ConstantExpr> m_ConstantExpr(ConstantExpr *&C) { return C; } | ||||
741 | |||||
742 | /// Match a basic block value, capturing it if we match. | ||||
743 | inline bind_ty<BasicBlock> m_BasicBlock(BasicBlock *&V) { return V; } | ||||
744 | inline bind_ty<const BasicBlock> m_BasicBlock(const BasicBlock *&V) { | ||||
745 | return V; | ||||
746 | } | ||||
747 | |||||
748 | /// Match an arbitrary immediate Constant and ignore it. | ||||
749 | inline match_combine_and<class_match<Constant>, | ||||
750 | match_unless<constantexpr_match>> | ||||
751 | m_ImmConstant() { | ||||
752 | return m_CombineAnd(m_Constant(), m_Unless(m_ConstantExpr())); | ||||
753 | } | ||||
754 | |||||
755 | /// Match an immediate Constant, capturing the value if we match. | ||||
756 | inline match_combine_and<bind_ty<Constant>, | ||||
757 | match_unless<constantexpr_match>> | ||||
758 | m_ImmConstant(Constant *&C) { | ||||
759 | return m_CombineAnd(m_Constant(C), m_Unless(m_ConstantExpr())); | ||||
760 | } | ||||
761 | |||||
762 | /// Match a specified Value*. | ||||
763 | struct specificval_ty { | ||||
764 | const Value *Val; | ||||
765 | |||||
766 | specificval_ty(const Value *V) : Val(V) {} | ||||
767 | |||||
768 | template <typename ITy> bool match(ITy *V) { return V == Val; } | ||||
769 | }; | ||||
770 | |||||
771 | /// Match if we have a specific specified value. | ||||
772 | inline specificval_ty m_Specific(const Value *V) { return V; } | ||||
773 | |||||
774 | /// Stores a reference to the Value *, not the Value * itself, | ||||
775 | /// thus can be used in commutative matchers. | ||||
776 | template <typename Class> struct deferredval_ty { | ||||
777 | Class *const &Val; | ||||
778 | |||||
779 | deferredval_ty(Class *const &V) : Val(V) {} | ||||
780 | |||||
781 | template <typename ITy> bool match(ITy *const V) { return V == Val; } | ||||
782 | }; | ||||
783 | |||||
784 | /// Like m_Specific(), but works if the specific value to match is determined | ||||
785 | /// as part of the same match() expression. For example: | ||||
786 | /// m_Add(m_Value(X), m_Specific(X)) is incorrect, because m_Specific() will | ||||
787 | /// bind X before the pattern match starts. | ||||
788 | /// m_Add(m_Value(X), m_Deferred(X)) is correct, and will check against | ||||
789 | /// whichever value m_Value(X) populated. | ||||
790 | inline deferredval_ty<Value> m_Deferred(Value *const &V) { return V; } | ||||
791 | inline deferredval_ty<const Value> m_Deferred(const Value *const &V) { | ||||
792 | return V; | ||||
793 | } | ||||
794 | |||||
795 | /// Match a specified floating point value or vector of all elements of | ||||
796 | /// that value. | ||||
797 | struct specific_fpval { | ||||
798 | double Val; | ||||
799 | |||||
800 | specific_fpval(double V) : Val(V) {} | ||||
801 | |||||
802 | template <typename ITy> bool match(ITy *V) { | ||||
803 | if (const auto *CFP = dyn_cast<ConstantFP>(V)) | ||||
804 | return CFP->isExactlyValue(Val); | ||||
805 | if (V->getType()->isVectorTy()) | ||||
806 | if (const auto *C = dyn_cast<Constant>(V)) | ||||
807 | if (auto *CFP = dyn_cast_or_null<ConstantFP>(C->getSplatValue())) | ||||
808 | return CFP->isExactlyValue(Val); | ||||
809 | return false; | ||||
810 | } | ||||
811 | }; | ||||
812 | |||||
813 | /// Match a specific floating point value or vector with all elements | ||||
814 | /// equal to the value. | ||||
815 | inline specific_fpval m_SpecificFP(double V) { return specific_fpval(V); } | ||||
816 | |||||
817 | /// Match a float 1.0 or vector with all elements equal to 1.0. | ||||
818 | inline specific_fpval m_FPOne() { return m_SpecificFP(1.0); } | ||||
819 | |||||
820 | struct bind_const_intval_ty { | ||||
821 | uint64_t &VR; | ||||
822 | |||||
823 | bind_const_intval_ty(uint64_t &V) : VR(V) {} | ||||
824 | |||||
825 | template <typename ITy> bool match(ITy *V) { | ||||
826 | if (const auto *CV = dyn_cast<ConstantInt>(V)) | ||||
827 | if (CV->getValue().ule(UINT64_MAX(18446744073709551615UL))) { | ||||
828 | VR = CV->getZExtValue(); | ||||
829 | return true; | ||||
830 | } | ||||
831 | return false; | ||||
832 | } | ||||
833 | }; | ||||
834 | |||||
835 | /// Match a specified integer value or vector of all elements of that | ||||
836 | /// value. | ||||
837 | template <bool AllowUndefs> struct specific_intval { | ||||
838 | APInt Val; | ||||
839 | |||||
840 | specific_intval(APInt V) : Val(std::move(V)) {} | ||||
841 | |||||
842 | template <typename ITy> bool match(ITy *V) { | ||||
843 | const auto *CI = dyn_cast<ConstantInt>(V); | ||||
844 | if (!CI && V->getType()->isVectorTy()) | ||||
845 | if (const auto *C = dyn_cast<Constant>(V)) | ||||
846 | CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndefs)); | ||||
847 | |||||
848 | return CI && APInt::isSameValue(CI->getValue(), Val); | ||||
849 | } | ||||
850 | }; | ||||
851 | |||||
852 | /// Match a specific integer value or vector with all elements equal to | ||||
853 | /// the value. | ||||
854 | inline specific_intval<false> m_SpecificInt(APInt V) { | ||||
855 | return specific_intval<false>(std::move(V)); | ||||
856 | } | ||||
857 | |||||
858 | inline specific_intval<false> m_SpecificInt(uint64_t V) { | ||||
859 | return m_SpecificInt(APInt(64, V)); | ||||
860 | } | ||||
861 | |||||
862 | inline specific_intval<true> m_SpecificIntAllowUndef(APInt V) { | ||||
863 | return specific_intval<true>(std::move(V)); | ||||
864 | } | ||||
865 | |||||
866 | inline specific_intval<true> m_SpecificIntAllowUndef(uint64_t V) { | ||||
867 | return m_SpecificIntAllowUndef(APInt(64, V)); | ||||
868 | } | ||||
869 | |||||
870 | /// Match a ConstantInt and bind to its value. This does not match | ||||
871 | /// ConstantInts wider than 64-bits. | ||||
872 | inline bind_const_intval_ty m_ConstantInt(uint64_t &V) { return V; } | ||||
873 | |||||
874 | /// Match a specified basic block value. | ||||
875 | struct specific_bbval { | ||||
876 | BasicBlock *Val; | ||||
877 | |||||
878 | specific_bbval(BasicBlock *Val) : Val(Val) {} | ||||
879 | |||||
880 | template <typename ITy> bool match(ITy *V) { | ||||
881 | const auto *BB = dyn_cast<BasicBlock>(V); | ||||
882 | return BB && BB == Val; | ||||
883 | } | ||||
884 | }; | ||||
885 | |||||
886 | /// Match a specific basic block value. | ||||
887 | inline specific_bbval m_SpecificBB(BasicBlock *BB) { | ||||
888 | return specific_bbval(BB); | ||||
889 | } | ||||
890 | |||||
891 | /// A commutative-friendly version of m_Specific(). | ||||
892 | inline deferredval_ty<BasicBlock> m_Deferred(BasicBlock *const &BB) { | ||||
893 | return BB; | ||||
894 | } | ||||
895 | inline deferredval_ty<const BasicBlock> | ||||
896 | m_Deferred(const BasicBlock *const &BB) { | ||||
897 | return BB; | ||||
898 | } | ||||
899 | |||||
900 | //===----------------------------------------------------------------------===// | ||||
901 | // Matcher for any binary operator. | ||||
902 | // | ||||
903 | template <typename LHS_t, typename RHS_t, bool Commutable = false> | ||||
904 | struct AnyBinaryOp_match { | ||||
905 | LHS_t L; | ||||
906 | RHS_t R; | ||||
907 | |||||
908 | // The evaluation order is always stable, regardless of Commutability. | ||||
909 | // The LHS is always matched first. | ||||
910 | AnyBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} | ||||
911 | |||||
912 | template <typename OpTy> bool match(OpTy *V) { | ||||
913 | if (auto *I
| ||||
914 | return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) || | ||||
915 | (Commutable && L.match(I->getOperand(1)) && | ||||
916 | R.match(I->getOperand(0))); | ||||
917 | return false; | ||||
918 | } | ||||
919 | }; | ||||
920 | |||||
921 | template <typename LHS, typename RHS> | ||||
922 | inline AnyBinaryOp_match<LHS, RHS> m_BinOp(const LHS &L, const RHS &R) { | ||||
923 | return AnyBinaryOp_match<LHS, RHS>(L, R); | ||||
924 | } | ||||
925 | |||||
926 | //===----------------------------------------------------------------------===// | ||||
927 | // Matcher for any unary operator. | ||||
928 | // TODO fuse unary, binary matcher into n-ary matcher | ||||
929 | // | ||||
930 | template <typename OP_t> struct AnyUnaryOp_match { | ||||
931 | OP_t X; | ||||
932 | |||||
933 | AnyUnaryOp_match(const OP_t &X) : X(X) {} | ||||
934 | |||||
935 | template <typename OpTy> bool match(OpTy *V) { | ||||
936 | if (auto *I = dyn_cast<UnaryOperator>(V)) | ||||
937 | return X.match(I->getOperand(0)); | ||||
938 | return false; | ||||
939 | } | ||||
940 | }; | ||||
941 | |||||
942 | template <typename OP_t> inline AnyUnaryOp_match<OP_t> m_UnOp(const OP_t &X) { | ||||
943 | return AnyUnaryOp_match<OP_t>(X); | ||||
944 | } | ||||
945 | |||||
946 | //===----------------------------------------------------------------------===// | ||||
947 | // Matchers for specific binary operators. | ||||
948 | // | ||||
949 | |||||
950 | template <typename LHS_t, typename RHS_t, unsigned Opcode, | ||||
951 | bool Commutable = false> | ||||
952 | struct BinaryOp_match { | ||||
953 | LHS_t L; | ||||
954 | RHS_t R; | ||||
955 | |||||
956 | // The evaluation order is always stable, regardless of Commutability. | ||||
957 | // The LHS is always matched first. | ||||
958 | BinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} | ||||
959 | |||||
960 | template <typename OpTy> inline bool match(unsigned Opc, OpTy *V) { | ||||
961 | if (V->getValueID() == Value::InstructionVal + Opc) { | ||||
962 | auto *I = cast<BinaryOperator>(V); | ||||
963 | return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) || | ||||
964 | (Commutable && L.match(I->getOperand(1)) && | ||||
965 | R.match(I->getOperand(0))); | ||||
966 | } | ||||
967 | if (auto *CE = dyn_cast<ConstantExpr>(V)) | ||||
968 | return CE->getOpcode() == Opc && | ||||
969 | ((L.match(CE->getOperand(0)) && R.match(CE->getOperand(1))) || | ||||
970 | (Commutable && L.match(CE->getOperand(1)) && | ||||
971 | R.match(CE->getOperand(0)))); | ||||
972 | return false; | ||||
973 | } | ||||
974 | |||||
975 | template <typename OpTy> bool match(OpTy *V) { return match(Opcode, V); } | ||||
976 | }; | ||||
977 | |||||
978 | template <typename LHS, typename RHS> | ||||
979 | inline BinaryOp_match<LHS, RHS, Instruction::Add> m_Add(const LHS &L, | ||||
980 | const RHS &R) { | ||||
981 | return BinaryOp_match<LHS, RHS, Instruction::Add>(L, R); | ||||
982 | } | ||||
983 | |||||
984 | template <typename LHS, typename RHS> | ||||
985 | inline BinaryOp_match<LHS, RHS, Instruction::FAdd> m_FAdd(const LHS &L, | ||||
986 | const RHS &R) { | ||||
987 | return BinaryOp_match<LHS, RHS, Instruction::FAdd>(L, R); | ||||
988 | } | ||||
989 | |||||
990 | template <typename LHS, typename RHS> | ||||
991 | inline BinaryOp_match<LHS, RHS, Instruction::Sub> m_Sub(const LHS &L, | ||||
992 | const RHS &R) { | ||||
993 | return BinaryOp_match<LHS, RHS, Instruction::Sub>(L, R); | ||||
994 | } | ||||
995 | |||||
996 | template <typename LHS, typename RHS> | ||||
997 | inline BinaryOp_match<LHS, RHS, Instruction::FSub> m_FSub(const LHS &L, | ||||
998 | const RHS &R) { | ||||
999 | return BinaryOp_match<LHS, RHS, Instruction::FSub>(L, R); | ||||
1000 | } | ||||
1001 | |||||
1002 | template <typename Op_t> struct FNeg_match { | ||||
1003 | Op_t X; | ||||
1004 | |||||
1005 | FNeg_match(const Op_t &Op) : X(Op) {} | ||||
1006 | template <typename OpTy> bool match(OpTy *V) { | ||||
1007 | auto *FPMO = dyn_cast<FPMathOperator>(V); | ||||
1008 | if (!FPMO) | ||||
1009 | return false; | ||||
1010 | |||||
1011 | if (FPMO->getOpcode() == Instruction::FNeg) | ||||
1012 | return X.match(FPMO->getOperand(0)); | ||||
1013 | |||||
1014 | if (FPMO->getOpcode() == Instruction::FSub) { | ||||
1015 | if (FPMO->hasNoSignedZeros()) { | ||||
1016 | // With 'nsz', any zero goes. | ||||
1017 | if (!cstfp_pred_ty<is_any_zero_fp>().match(FPMO->getOperand(0))) | ||||
1018 | return false; | ||||
1019 | } else { | ||||
1020 | // Without 'nsz', we need fsub -0.0, X exactly. | ||||
1021 | if (!cstfp_pred_ty<is_neg_zero_fp>().match(FPMO->getOperand(0))) | ||||
1022 | return false; | ||||
1023 | } | ||||
1024 | |||||
1025 | return X.match(FPMO->getOperand(1)); | ||||
1026 | } | ||||
1027 | |||||
1028 | return false; | ||||
1029 | } | ||||
1030 | }; | ||||
1031 | |||||
1032 | /// Match 'fneg X' as 'fsub -0.0, X'. | ||||
1033 | template <typename OpTy> inline FNeg_match<OpTy> m_FNeg(const OpTy &X) { | ||||
1034 | return FNeg_match<OpTy>(X); | ||||
1035 | } | ||||
1036 | |||||
1037 | /// Match 'fneg X' as 'fsub +-0.0, X'. | ||||
1038 | template <typename RHS> | ||||
1039 | inline BinaryOp_match<cstfp_pred_ty<is_any_zero_fp>, RHS, Instruction::FSub> | ||||
1040 | m_FNegNSZ(const RHS &X) { | ||||
1041 | return m_FSub(m_AnyZeroFP(), X); | ||||
1042 | } | ||||
1043 | |||||
1044 | template <typename LHS, typename RHS> | ||||
1045 | inline BinaryOp_match<LHS, RHS, Instruction::Mul> m_Mul(const LHS &L, | ||||
1046 | const RHS &R) { | ||||
1047 | return BinaryOp_match<LHS, RHS, Instruction::Mul>(L, R); | ||||
1048 | } | ||||
1049 | |||||
1050 | template <typename LHS, typename RHS> | ||||
1051 | inline BinaryOp_match<LHS, RHS, Instruction::FMul> m_FMul(const LHS &L, | ||||
1052 | const RHS &R) { | ||||
1053 | return BinaryOp_match<LHS, RHS, Instruction::FMul>(L, R); | ||||
1054 | } | ||||
1055 | |||||
1056 | template <typename LHS, typename RHS> | ||||
1057 | inline BinaryOp_match<LHS, RHS, Instruction::UDiv> m_UDiv(const LHS &L, | ||||
1058 | const RHS &R) { | ||||
1059 | return BinaryOp_match<LHS, RHS, Instruction::UDiv>(L, R); | ||||
1060 | } | ||||
1061 | |||||
1062 | template <typename LHS, typename RHS> | ||||
1063 | inline BinaryOp_match<LHS, RHS, Instruction::SDiv> m_SDiv(const LHS &L, | ||||
1064 | const RHS &R) { | ||||
1065 | return BinaryOp_match<LHS, RHS, Instruction::SDiv>(L, R); | ||||
1066 | } | ||||
1067 | |||||
1068 | template <typename LHS, typename RHS> | ||||
1069 | inline BinaryOp_match<LHS, RHS, Instruction::FDiv> m_FDiv(const LHS &L, | ||||
1070 | const RHS &R) { | ||||
1071 | return BinaryOp_match<LHS, RHS, Instruction::FDiv>(L, R); | ||||
1072 | } | ||||
1073 | |||||
1074 | template <typename LHS, typename RHS> | ||||
1075 | inline BinaryOp_match<LHS, RHS, Instruction::URem> m_URem(const LHS &L, | ||||
1076 | const RHS &R) { | ||||
1077 | return BinaryOp_match<LHS, RHS, Instruction::URem>(L, R); | ||||
1078 | } | ||||
1079 | |||||
1080 | template <typename LHS, typename RHS> | ||||
1081 | inline BinaryOp_match<LHS, RHS, Instruction::SRem> m_SRem(const LHS &L, | ||||
1082 | const RHS &R) { | ||||
1083 | return BinaryOp_match<LHS, RHS, Instruction::SRem>(L, R); | ||||
1084 | } | ||||
1085 | |||||
1086 | template <typename LHS, typename RHS> | ||||
1087 | inline BinaryOp_match<LHS, RHS, Instruction::FRem> m_FRem(const LHS &L, | ||||
1088 | const RHS &R) { | ||||
1089 | return BinaryOp_match<LHS, RHS, Instruction::FRem>(L, R); | ||||
1090 | } | ||||
1091 | |||||
1092 | template <typename LHS, typename RHS> | ||||
1093 | inline BinaryOp_match<LHS, RHS, Instruction::And> m_And(const LHS &L, | ||||
1094 | const RHS &R) { | ||||
1095 | return BinaryOp_match<LHS, RHS, Instruction::And>(L, R); | ||||
1096 | } | ||||
1097 | |||||
1098 | template <typename LHS, typename RHS> | ||||
1099 | inline BinaryOp_match<LHS, RHS, Instruction::Or> m_Or(const LHS &L, | ||||
1100 | const RHS &R) { | ||||
1101 | return BinaryOp_match<LHS, RHS, Instruction::Or>(L, R); | ||||
1102 | } | ||||
1103 | |||||
1104 | template <typename LHS, typename RHS> | ||||
1105 | inline BinaryOp_match<LHS, RHS, Instruction::Xor> m_Xor(const LHS &L, | ||||
1106 | const RHS &R) { | ||||
1107 | return BinaryOp_match<LHS, RHS, Instruction::Xor>(L, R); | ||||
1108 | } | ||||
1109 | |||||
1110 | template <typename LHS, typename RHS> | ||||
1111 | inline BinaryOp_match<LHS, RHS, Instruction::Shl> m_Shl(const LHS &L, | ||||
1112 | const RHS &R) { | ||||
1113 | return BinaryOp_match<LHS, RHS, Instruction::Shl>(L, R); | ||||
1114 | } | ||||
1115 | |||||
1116 | template <typename LHS, typename RHS> | ||||
1117 | inline BinaryOp_match<LHS, RHS, Instruction::LShr> m_LShr(const LHS &L, | ||||
1118 | const RHS &R) { | ||||
1119 | return BinaryOp_match<LHS, RHS, Instruction::LShr>(L, R); | ||||
1120 | } | ||||
1121 | |||||
1122 | template <typename LHS, typename RHS> | ||||
1123 | inline BinaryOp_match<LHS, RHS, Instruction::AShr> m_AShr(const LHS &L, | ||||
1124 | const RHS &R) { | ||||
1125 | return BinaryOp_match<LHS, RHS, Instruction::AShr>(L, R); | ||||
1126 | } | ||||
1127 | |||||
1128 | template <typename LHS_t, typename RHS_t, unsigned Opcode, | ||||
1129 | unsigned WrapFlags = 0> | ||||
1130 | struct OverflowingBinaryOp_match { | ||||
1131 | LHS_t L; | ||||
1132 | RHS_t R; | ||||
1133 | |||||
1134 | OverflowingBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) | ||||
1135 | : L(LHS), R(RHS) {} | ||||
1136 | |||||
1137 | template <typename OpTy> bool match(OpTy *V) { | ||||
1138 | if (auto *Op = dyn_cast<OverflowingBinaryOperator>(V)) { | ||||
1139 | if (Op->getOpcode() != Opcode) | ||||
1140 | return false; | ||||
1141 | if ((WrapFlags & OverflowingBinaryOperator::NoUnsignedWrap) && | ||||
1142 | !Op->hasNoUnsignedWrap()) | ||||
1143 | return false; | ||||
1144 | if ((WrapFlags & OverflowingBinaryOperator::NoSignedWrap) && | ||||
1145 | !Op->hasNoSignedWrap()) | ||||
1146 | return false; | ||||
1147 | return L.match(Op->getOperand(0)) && R.match(Op->getOperand(1)); | ||||
1148 | } | ||||
1149 | return false; | ||||
1150 | } | ||||
1151 | }; | ||||
1152 | |||||
1153 | template <typename LHS, typename RHS> | ||||
1154 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, | ||||
1155 | OverflowingBinaryOperator::NoSignedWrap> | ||||
1156 | m_NSWAdd(const LHS &L, const RHS &R) { | ||||
1157 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, | ||||
1158 | OverflowingBinaryOperator::NoSignedWrap>(L, | ||||
1159 | R); | ||||
1160 | } | ||||
1161 | template <typename LHS, typename RHS> | ||||
1162 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, | ||||
1163 | OverflowingBinaryOperator::NoSignedWrap> | ||||
1164 | m_NSWSub(const LHS &L, const RHS &R) { | ||||
1165 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, | ||||
1166 | OverflowingBinaryOperator::NoSignedWrap>(L, | ||||
1167 | R); | ||||
1168 | } | ||||
1169 | template <typename LHS, typename RHS> | ||||
1170 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, | ||||
1171 | OverflowingBinaryOperator::NoSignedWrap> | ||||
1172 | m_NSWMul(const LHS &L, const RHS &R) { | ||||
1173 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, | ||||
1174 | OverflowingBinaryOperator::NoSignedWrap>(L, | ||||
1175 | R); | ||||
1176 | } | ||||
1177 | template <typename LHS, typename RHS> | ||||
1178 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, | ||||
1179 | OverflowingBinaryOperator::NoSignedWrap> | ||||
1180 | m_NSWShl(const LHS &L, const RHS &R) { | ||||
1181 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, | ||||
1182 | OverflowingBinaryOperator::NoSignedWrap>(L, | ||||
1183 | R); | ||||
1184 | } | ||||
1185 | |||||
1186 | template <typename LHS, typename RHS> | ||||
1187 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, | ||||
1188 | OverflowingBinaryOperator::NoUnsignedWrap> | ||||
1189 | m_NUWAdd(const LHS &L, const RHS &R) { | ||||
1190 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, | ||||
1191 | OverflowingBinaryOperator::NoUnsignedWrap>( | ||||
1192 | L, R); | ||||
1193 | } | ||||
1194 | template <typename LHS, typename RHS> | ||||
1195 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, | ||||
1196 | OverflowingBinaryOperator::NoUnsignedWrap> | ||||
1197 | m_NUWSub(const LHS &L, const RHS &R) { | ||||
1198 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, | ||||
1199 | OverflowingBinaryOperator::NoUnsignedWrap>( | ||||
1200 | L, R); | ||||
1201 | } | ||||
1202 | template <typename LHS, typename RHS> | ||||
1203 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, | ||||
1204 | OverflowingBinaryOperator::NoUnsignedWrap> | ||||
1205 | m_NUWMul(const LHS &L, const RHS &R) { | ||||
1206 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, | ||||
1207 | OverflowingBinaryOperator::NoUnsignedWrap>( | ||||
1208 | L, R); | ||||
1209 | } | ||||
1210 | template <typename LHS, typename RHS> | ||||
1211 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, | ||||
1212 | OverflowingBinaryOperator::NoUnsignedWrap> | ||||
1213 | m_NUWShl(const LHS &L, const RHS &R) { | ||||
1214 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, | ||||
1215 | OverflowingBinaryOperator::NoUnsignedWrap>( | ||||
1216 | L, R); | ||||
1217 | } | ||||
1218 | |||||
1219 | template <typename LHS_t, typename RHS_t, bool Commutable = false> | ||||
1220 | struct SpecificBinaryOp_match | ||||
1221 | : public BinaryOp_match<LHS_t, RHS_t, 0, Commutable> { | ||||
1222 | unsigned Opcode; | ||||
1223 | |||||
1224 | SpecificBinaryOp_match(unsigned Opcode, const LHS_t &LHS, const RHS_t &RHS) | ||||
1225 | : BinaryOp_match<LHS_t, RHS_t, 0, Commutable>(LHS, RHS), Opcode(Opcode) {} | ||||
1226 | |||||
1227 | template <typename OpTy> bool match(OpTy *V) { | ||||
1228 | return BinaryOp_match<LHS_t, RHS_t, 0, Commutable>::match(Opcode, V); | ||||
1229 | } | ||||
1230 | }; | ||||
1231 | |||||
1232 | /// Matches a specific opcode. | ||||
1233 | template <typename LHS, typename RHS> | ||||
1234 | inline SpecificBinaryOp_match<LHS, RHS> m_BinOp(unsigned Opcode, const LHS &L, | ||||
1235 | const RHS &R) { | ||||
1236 | return SpecificBinaryOp_match<LHS, RHS>(Opcode, L, R); | ||||
1237 | } | ||||
1238 | |||||
1239 | //===----------------------------------------------------------------------===// | ||||
1240 | // Class that matches a group of binary opcodes. | ||||
1241 | // | ||||
1242 | template <typename LHS_t, typename RHS_t, typename Predicate> | ||||
1243 | struct BinOpPred_match : Predicate { | ||||
1244 | LHS_t L; | ||||
1245 | RHS_t R; | ||||
1246 | |||||
1247 | BinOpPred_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} | ||||
1248 | |||||
1249 | template <typename OpTy> bool match(OpTy *V) { | ||||
1250 | if (auto *I = dyn_cast<Instruction>(V)) | ||||
1251 | return this->isOpType(I->getOpcode()) && L.match(I->getOperand(0)) && | ||||
1252 | R.match(I->getOperand(1)); | ||||
1253 | if (auto *CE = dyn_cast<ConstantExpr>(V)) | ||||
1254 | return this->isOpType(CE->getOpcode()) && L.match(CE->getOperand(0)) && | ||||
1255 | R.match(CE->getOperand(1)); | ||||
1256 | return false; | ||||
1257 | } | ||||
1258 | }; | ||||
1259 | |||||
1260 | struct is_shift_op { | ||||
1261 | bool isOpType(unsigned Opcode) { return Instruction::isShift(Opcode); } | ||||
1262 | }; | ||||
1263 | |||||
1264 | struct is_right_shift_op { | ||||
1265 | bool isOpType(unsigned Opcode) { | ||||
1266 | return Opcode == Instruction::LShr || Opcode == Instruction::AShr; | ||||
1267 | } | ||||
1268 | }; | ||||
1269 | |||||
1270 | struct is_logical_shift_op { | ||||
1271 | bool isOpType(unsigned Opcode) { | ||||
1272 | return Opcode == Instruction::LShr || Opcode == Instruction::Shl; | ||||
1273 | } | ||||
1274 | }; | ||||
1275 | |||||
1276 | struct is_bitwiselogic_op { | ||||
1277 | bool isOpType(unsigned Opcode) { | ||||
1278 | return Instruction::isBitwiseLogicOp(Opcode); | ||||
1279 | } | ||||
1280 | }; | ||||
1281 | |||||
1282 | struct is_idiv_op { | ||||
1283 | bool isOpType(unsigned Opcode) { | ||||
1284 | return Opcode == Instruction::SDiv || Opcode == Instruction::UDiv; | ||||
1285 | } | ||||
1286 | }; | ||||
1287 | |||||
1288 | struct is_irem_op { | ||||
1289 | bool isOpType(unsigned Opcode) { | ||||
1290 | return Opcode == Instruction::SRem || Opcode == Instruction::URem; | ||||
1291 | } | ||||
1292 | }; | ||||
1293 | |||||
1294 | /// Matches shift operations. | ||||
1295 | template <typename LHS, typename RHS> | ||||
1296 | inline BinOpPred_match<LHS, RHS, is_shift_op> m_Shift(const LHS &L, | ||||
1297 | const RHS &R) { | ||||
1298 | return BinOpPred_match<LHS, RHS, is_shift_op>(L, R); | ||||
1299 | } | ||||
1300 | |||||
1301 | /// Matches logical shift operations. | ||||
1302 | template <typename LHS, typename RHS> | ||||
1303 | inline BinOpPred_match<LHS, RHS, is_right_shift_op> m_Shr(const LHS &L, | ||||
1304 | const RHS &R) { | ||||
1305 | return BinOpPred_match<LHS, RHS, is_right_shift_op>(L, R); | ||||
1306 | } | ||||
1307 | |||||
1308 | /// Matches logical shift operations. | ||||
1309 | template <typename LHS, typename RHS> | ||||
1310 | inline BinOpPred_match<LHS, RHS, is_logical_shift_op> | ||||
1311 | m_LogicalShift(const LHS &L, const RHS &R) { | ||||
1312 | return BinOpPred_match<LHS, RHS, is_logical_shift_op>(L, R); | ||||
1313 | } | ||||
1314 | |||||
1315 | /// Matches bitwise logic operations. | ||||
1316 | template <typename LHS, typename RHS> | ||||
1317 | inline BinOpPred_match<LHS, RHS, is_bitwiselogic_op> | ||||
1318 | m_BitwiseLogic(const LHS &L, const RHS &R) { | ||||
1319 | return BinOpPred_match<LHS, RHS, is_bitwiselogic_op>(L, R); | ||||
1320 | } | ||||
1321 | |||||
1322 | /// Matches integer division operations. | ||||
1323 | template <typename LHS, typename RHS> | ||||
1324 | inline BinOpPred_match<LHS, RHS, is_idiv_op> m_IDiv(const LHS &L, | ||||
1325 | const RHS &R) { | ||||
1326 | return BinOpPred_match<LHS, RHS, is_idiv_op>(L, R); | ||||
1327 | } | ||||
1328 | |||||
1329 | /// Matches integer remainder operations. | ||||
1330 | template <typename LHS, typename RHS> | ||||
1331 | inline BinOpPred_match<LHS, RHS, is_irem_op> m_IRem(const LHS &L, | ||||
1332 | const RHS &R) { | ||||
1333 | return BinOpPred_match<LHS, RHS, is_irem_op>(L, R); | ||||
1334 | } | ||||
1335 | |||||
1336 | //===----------------------------------------------------------------------===// | ||||
1337 | // Class that matches exact binary ops. | ||||
1338 | // | ||||
1339 | template <typename SubPattern_t> struct Exact_match { | ||||
1340 | SubPattern_t SubPattern; | ||||
1341 | |||||
1342 | Exact_match(const SubPattern_t &SP) : SubPattern(SP) {} | ||||
1343 | |||||
1344 | template <typename OpTy> bool match(OpTy *V) { | ||||
1345 | if (auto *PEO = dyn_cast<PossiblyExactOperator>(V)) | ||||
1346 | return PEO->isExact() && SubPattern.match(V); | ||||
1347 | return false; | ||||
1348 | } | ||||
1349 | }; | ||||
1350 | |||||
1351 | template <typename T> inline Exact_match<T> m_Exact(const T &SubPattern) { | ||||
1352 | return SubPattern; | ||||
1353 | } | ||||
1354 | |||||
1355 | //===----------------------------------------------------------------------===// | ||||
1356 | // Matchers for CmpInst classes | ||||
1357 | // | ||||
1358 | |||||
1359 | template <typename LHS_t, typename RHS_t, typename Class, typename PredicateTy, | ||||
1360 | bool Commutable = false> | ||||
1361 | struct CmpClass_match { | ||||
1362 | PredicateTy &Predicate; | ||||
1363 | LHS_t L; | ||||
1364 | RHS_t R; | ||||
1365 | |||||
1366 | // The evaluation order is always stable, regardless of Commutability. | ||||
1367 | // The LHS is always matched first. | ||||
1368 | CmpClass_match(PredicateTy &Pred, const LHS_t &LHS, const RHS_t &RHS) | ||||
1369 | : Predicate(Pred), L(LHS), R(RHS) {} | ||||
1370 | |||||
1371 | template <typename OpTy> bool match(OpTy *V) { | ||||
1372 | if (auto *I = dyn_cast<Class>(V)) { | ||||
1373 | if (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) { | ||||
1374 | Predicate = I->getPredicate(); | ||||
1375 | return true; | ||||
1376 | } else if (Commutable && L.match(I->getOperand(1)) && | ||||
1377 | R.match(I->getOperand(0))) { | ||||
1378 | Predicate = I->getSwappedPredicate(); | ||||
1379 | return true; | ||||
1380 | } | ||||
1381 | } | ||||
1382 | return false; | ||||
1383 | } | ||||
1384 | }; | ||||
1385 | |||||
1386 | template <typename LHS, typename RHS> | ||||
1387 | inline CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate> | ||||
1388 | m_Cmp(CmpInst::Predicate &Pred, const LHS &L, const RHS &R) { | ||||
1389 | return CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate>(Pred, L, R); | ||||
1390 | } | ||||
1391 | |||||
1392 | template <typename LHS, typename RHS> | ||||
1393 | inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate> | ||||
1394 | m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { | ||||
1395 | return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate>(Pred, L, R); | ||||
1396 | } | ||||
1397 | |||||
1398 | template <typename LHS, typename RHS> | ||||
1399 | inline CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate> | ||||
1400 | m_FCmp(FCmpInst::Predicate &Pred, const LHS &L, const RHS &R) { | ||||
1401 | return CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate>(Pred, L, R); | ||||
1402 | } | ||||
1403 | |||||
1404 | //===----------------------------------------------------------------------===// | ||||
1405 | // Matchers for instructions with a given opcode and number of operands. | ||||
1406 | // | ||||
1407 | |||||
1408 | /// Matches instructions with Opcode and three operands. | ||||
1409 | template <typename T0, unsigned Opcode> struct OneOps_match { | ||||
1410 | T0 Op1; | ||||
1411 | |||||
1412 | OneOps_match(const T0 &Op1) : Op1(Op1) {} | ||||
1413 | |||||
1414 | template <typename OpTy> bool match(OpTy *V) { | ||||
1415 | if (V->getValueID() == Value::InstructionVal + Opcode) { | ||||
1416 | auto *I = cast<Instruction>(V); | ||||
1417 | return Op1.match(I->getOperand(0)); | ||||
1418 | } | ||||
1419 | return false; | ||||
1420 | } | ||||
1421 | }; | ||||
1422 | |||||
1423 | /// Matches instructions with Opcode and three operands. | ||||
1424 | template <typename T0, typename T1, unsigned Opcode> struct TwoOps_match { | ||||
1425 | T0 Op1; | ||||
1426 | T1 Op2; | ||||
1427 | |||||
1428 | TwoOps_match(const T0 &Op1, const T1 &Op2) : Op1(Op1), Op2(Op2) {} | ||||
1429 | |||||
1430 | template <typename OpTy> bool match(OpTy *V) { | ||||
1431 | if (V->getValueID() == Value::InstructionVal + Opcode) { | ||||
1432 | auto *I = cast<Instruction>(V); | ||||
1433 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)); | ||||
1434 | } | ||||
1435 | return false; | ||||
1436 | } | ||||
1437 | }; | ||||
1438 | |||||
1439 | /// Matches instructions with Opcode and three operands. | ||||
1440 | template <typename T0, typename T1, typename T2, unsigned Opcode> | ||||
1441 | struct ThreeOps_match { | ||||
1442 | T0 Op1; | ||||
1443 | T1 Op2; | ||||
1444 | T2 Op3; | ||||
1445 | |||||
1446 | ThreeOps_match(const T0 &Op1, const T1 &Op2, const T2 &Op3) | ||||
1447 | : Op1(Op1), Op2(Op2), Op3(Op3) {} | ||||
1448 | |||||
1449 | template <typename OpTy> bool match(OpTy *V) { | ||||
1450 | if (V->getValueID() == Value::InstructionVal + Opcode) { | ||||
| |||||
1451 | auto *I = cast<Instruction>(V); | ||||
1452 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) && | ||||
1453 | Op3.match(I->getOperand(2)); | ||||
1454 | } | ||||
1455 | return false; | ||||
1456 | } | ||||
1457 | }; | ||||
1458 | |||||
1459 | /// Matches SelectInst. | ||||
1460 | template <typename Cond, typename LHS, typename RHS> | ||||
1461 | inline ThreeOps_match<Cond, LHS, RHS, Instruction::Select> | ||||
1462 | m_Select(const Cond &C, const LHS &L, const RHS &R) { | ||||
1463 | return ThreeOps_match<Cond, LHS, RHS, Instruction::Select>(C, L, R); | ||||
1464 | } | ||||
1465 | |||||
1466 | /// This matches a select of two constants, e.g.: | ||||
1467 | /// m_SelectCst<-1, 0>(m_Value(V)) | ||||
1468 | template <int64_t L, int64_t R, typename Cond> | ||||
1469 | inline ThreeOps_match<Cond, constantint_match<L>, constantint_match<R>, | ||||
1470 | Instruction::Select> | ||||
1471 | m_SelectCst(const Cond &C) { | ||||
1472 | return m_Select(C, m_ConstantInt<L>(), m_ConstantInt<R>()); | ||||
1473 | } | ||||
1474 | |||||
1475 | /// Matches FreezeInst. | ||||
1476 | template <typename OpTy> | ||||
1477 | inline OneOps_match<OpTy, Instruction::Freeze> m_Freeze(const OpTy &Op) { | ||||
1478 | return OneOps_match<OpTy, Instruction::Freeze>(Op); | ||||
1479 | } | ||||
1480 | |||||
1481 | /// Matches InsertElementInst. | ||||
1482 | template <typename Val_t, typename Elt_t, typename Idx_t> | ||||
1483 | inline ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement> | ||||
1484 | m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx) { | ||||
1485 | return ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement>( | ||||
1486 | Val, Elt, Idx); | ||||
1487 | } | ||||
1488 | |||||
1489 | /// Matches ExtractElementInst. | ||||
1490 | template <typename Val_t, typename Idx_t> | ||||
1491 | inline TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement> | ||||
1492 | m_ExtractElt(const Val_t &Val, const Idx_t &Idx) { | ||||
1493 | return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx); | ||||
1494 | } | ||||
1495 | |||||
1496 | /// Matches shuffle. | ||||
1497 | template <typename T0, typename T1, typename T2> struct Shuffle_match { | ||||
1498 | T0 Op1; | ||||
1499 | T1 Op2; | ||||
1500 | T2 Mask; | ||||
1501 | |||||
1502 | Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask) | ||||
1503 | : Op1(Op1), Op2(Op2), Mask(Mask) {} | ||||
1504 | |||||
1505 | template <typename OpTy> bool match(OpTy *V) { | ||||
1506 | if (auto *I = dyn_cast<ShuffleVectorInst>(V)) { | ||||
1507 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) && | ||||
1508 | Mask.match(I->getShuffleMask()); | ||||
1509 | } | ||||
1510 | return false; | ||||
1511 | } | ||||
1512 | }; | ||||
1513 | |||||
1514 | struct m_Mask { | ||||
1515 | ArrayRef<int> &MaskRef; | ||||
1516 | m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {} | ||||
1517 | bool match(ArrayRef<int> Mask) { | ||||
1518 | MaskRef = Mask; | ||||
1519 | return true; | ||||
1520 | } | ||||
1521 | }; | ||||
1522 | |||||
1523 | struct m_ZeroMask { | ||||
1524 | bool match(ArrayRef<int> Mask) { | ||||
1525 | return all_of(Mask, [](int Elem) { return Elem == 0 || Elem == -1; }); | ||||
1526 | } | ||||
1527 | }; | ||||
1528 | |||||
1529 | struct m_SpecificMask { | ||||
1530 | ArrayRef<int> &MaskRef; | ||||
1531 | m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {} | ||||
1532 | bool match(ArrayRef<int> Mask) { return MaskRef == Mask; } | ||||
1533 | }; | ||||
1534 | |||||
1535 | struct m_SplatOrUndefMask { | ||||
1536 | int &SplatIndex; | ||||
1537 | m_SplatOrUndefMask(int &SplatIndex) : SplatIndex(SplatIndex) {} | ||||
1538 | bool match(ArrayRef<int> Mask) { | ||||
1539 | const auto *First = find_if(Mask, [](int Elem) { return Elem != -1; }); | ||||
1540 | if (First == Mask.end()) | ||||
1541 | return false; | ||||
1542 | SplatIndex = *First; | ||||
1543 | return all_of(Mask, | ||||
1544 | [First](int Elem) { return Elem == *First || Elem == -1; }); | ||||
1545 | } | ||||
1546 | }; | ||||
1547 | |||||
1548 | /// Matches ShuffleVectorInst independently of mask value. | ||||
1549 | template <typename V1_t, typename V2_t> | ||||
1550 | inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector> | ||||
1551 | m_Shuffle(const V1_t &v1, const V2_t &v2) { | ||||
1552 | return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2); | ||||
1553 | } | ||||
1554 | |||||
1555 | template <typename V1_t, typename V2_t, typename Mask_t> | ||||
1556 | inline Shuffle_match<V1_t, V2_t, Mask_t> | ||||
1557 | m_Shuffle(const V1_t &v1, const V2_t &v2, const Mask_t &mask) { | ||||
1558 | return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask); | ||||
1559 | } | ||||
1560 | |||||
1561 | /// Matches LoadInst. | ||||
1562 | template <typename OpTy> | ||||
1563 | inline OneOps_match<OpTy, Instruction::Load> m_Load(const OpTy &Op) { | ||||
1564 | return OneOps_match<OpTy, Instruction::Load>(Op); | ||||
1565 | } | ||||
1566 | |||||
1567 | /// Matches StoreInst. | ||||
1568 | template <typename ValueOpTy, typename PointerOpTy> | ||||
1569 | inline TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store> | ||||
1570 | m_Store(const ValueOpTy &ValueOp, const PointerOpTy &PointerOp) { | ||||
1571 | return TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store>(ValueOp, | ||||
1572 | PointerOp); | ||||
1573 | } | ||||
1574 | |||||
1575 | //===----------------------------------------------------------------------===// | ||||
1576 | // Matchers for CastInst classes | ||||
1577 | // | ||||
1578 | |||||
1579 | template <typename Op_t, unsigned Opcode> struct CastClass_match { | ||||
1580 | Op_t Op; | ||||
1581 | |||||
1582 | CastClass_match(const Op_t &OpMatch) : Op(OpMatch) {} | ||||
1583 | |||||
1584 | template <typename OpTy> bool match(OpTy *V) { | ||||
1585 | if (auto *O = dyn_cast<Operator>(V)) | ||||
1586 | return O->getOpcode() == Opcode && Op.match(O->getOperand(0)); | ||||
1587 | return false; | ||||
1588 | } | ||||
1589 | }; | ||||
1590 | |||||
1591 | /// Matches BitCast. | ||||
1592 | template <typename OpTy> | ||||
1593 | inline CastClass_match<OpTy, Instruction::BitCast> m_BitCast(const OpTy &Op) { | ||||
1594 | return CastClass_match<OpTy, Instruction::BitCast>(Op); | ||||
1595 | } | ||||
1596 | |||||
1597 | /// Matches PtrToInt. | ||||
1598 | template <typename OpTy> | ||||
1599 | inline CastClass_match<OpTy, Instruction::PtrToInt> m_PtrToInt(const OpTy &Op) { | ||||
1600 | return CastClass_match<OpTy, Instruction::PtrToInt>(Op); | ||||
1601 | } | ||||
1602 | |||||
1603 | /// Matches IntToPtr. | ||||
1604 | template <typename OpTy> | ||||
1605 | inline CastClass_match<OpTy, Instruction::IntToPtr> m_IntToPtr(const OpTy &Op) { | ||||
1606 | return CastClass_match<OpTy, Instruction::IntToPtr>(Op); | ||||
1607 | } | ||||
1608 | |||||
1609 | /// Matches Trunc. | ||||
1610 | template <typename OpTy> | ||||
1611 | inline CastClass_match<OpTy, Instruction::Trunc> m_Trunc(const OpTy &Op) { | ||||
1612 | return CastClass_match<OpTy, Instruction::Trunc>(Op); | ||||
1613 | } | ||||
1614 | |||||
1615 | template <typename OpTy> | ||||
1616 | inline match_combine_or<CastClass_match<OpTy, Instruction::Trunc>, OpTy> | ||||
1617 | m_TruncOrSelf(const OpTy &Op) { | ||||
1618 | return m_CombineOr(m_Trunc(Op), Op); | ||||
1619 | } | ||||
1620 | |||||
1621 | /// Matches SExt. | ||||
1622 | template <typename OpTy> | ||||
1623 | inline CastClass_match<OpTy, Instruction::SExt> m_SExt(const OpTy &Op) { | ||||
1624 | return CastClass_match<OpTy, Instruction::SExt>(Op); | ||||
1625 | } | ||||
1626 | |||||
1627 | /// Matches ZExt. | ||||
1628 | template <typename OpTy> | ||||
1629 | inline CastClass_match<OpTy, Instruction::ZExt> m_ZExt(const OpTy &Op) { | ||||
1630 | return CastClass_match<OpTy, Instruction::ZExt>(Op); | ||||
1631 | } | ||||
1632 | |||||
1633 | template <typename OpTy> | ||||
1634 | inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, OpTy> | ||||
1635 | m_ZExtOrSelf(const OpTy &Op) { | ||||
1636 | return m_CombineOr(m_ZExt(Op), Op); | ||||
1637 | } | ||||
1638 | |||||
1639 | template <typename OpTy> | ||||
1640 | inline match_combine_or<CastClass_match<OpTy, Instruction::SExt>, OpTy> | ||||
1641 | m_SExtOrSelf(const OpTy &Op) { | ||||
1642 | return m_CombineOr(m_SExt(Op), Op); | ||||
1643 | } | ||||
1644 | |||||
1645 | template <typename OpTy> | ||||
1646 | inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, | ||||
1647 | CastClass_match<OpTy, Instruction::SExt>> | ||||
1648 | m_ZExtOrSExt(const OpTy &Op) { | ||||
1649 | return m_CombineOr(m_ZExt(Op), m_SExt(Op)); | ||||
1650 | } | ||||
1651 | |||||
1652 | template <typename OpTy> | ||||
1653 | inline match_combine_or< | ||||
1654 | match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, | ||||
1655 | CastClass_match<OpTy, Instruction::SExt>>, | ||||
1656 | OpTy> | ||||
1657 | m_ZExtOrSExtOrSelf(const OpTy &Op) { | ||||
1658 | return m_CombineOr(m_ZExtOrSExt(Op), Op); | ||||
1659 | } | ||||
1660 | |||||
1661 | template <typename OpTy> | ||||
1662 | inline CastClass_match<OpTy, Instruction::UIToFP> m_UIToFP(const OpTy &Op) { | ||||
1663 | return CastClass_match<OpTy, Instruction::UIToFP>(Op); | ||||
1664 | } | ||||
1665 | |||||
1666 | template <typename OpTy> | ||||
1667 | inline CastClass_match<OpTy, Instruction::SIToFP> m_SIToFP(const OpTy &Op) { | ||||
1668 | return CastClass_match<OpTy, Instruction::SIToFP>(Op); | ||||
1669 | } | ||||
1670 | |||||
1671 | template <typename OpTy> | ||||
1672 | inline CastClass_match<OpTy, Instruction::FPToUI> m_FPToUI(const OpTy &Op) { | ||||
1673 | return CastClass_match<OpTy, Instruction::FPToUI>(Op); | ||||
1674 | } | ||||
1675 | |||||
1676 | template <typename OpTy> | ||||
1677 | inline CastClass_match<OpTy, Instruction::FPToSI> m_FPToSI(const OpTy &Op) { | ||||
1678 | return CastClass_match<OpTy, Instruction::FPToSI>(Op); | ||||
1679 | } | ||||
1680 | |||||
1681 | template <typename OpTy> | ||||
1682 | inline CastClass_match<OpTy, Instruction::FPTrunc> m_FPTrunc(const OpTy &Op) { | ||||
1683 | return CastClass_match<OpTy, Instruction::FPTrunc>(Op); | ||||
1684 | } | ||||
1685 | |||||
1686 | template <typename OpTy> | ||||
1687 | inline CastClass_match<OpTy, Instruction::FPExt> m_FPExt(const OpTy &Op) { | ||||
1688 | return CastClass_match<OpTy, Instruction::FPExt>(Op); | ||||
1689 | } | ||||
1690 | |||||
1691 | //===----------------------------------------------------------------------===// | ||||
1692 | // Matchers for control flow. | ||||
1693 | // | ||||
1694 | |||||
1695 | struct br_match { | ||||
1696 | BasicBlock *&Succ; | ||||
1697 | |||||
1698 | br_match(BasicBlock *&Succ) : Succ(Succ) {} | ||||
1699 | |||||
1700 | template <typename OpTy> bool match(OpTy *V) { | ||||
1701 | if (auto *BI = dyn_cast<BranchInst>(V)) | ||||
1702 | if (BI->isUnconditional()) { | ||||
1703 | Succ = BI->getSuccessor(0); | ||||
1704 | return true; | ||||
1705 | } | ||||
1706 | return false; | ||||
1707 | } | ||||
1708 | }; | ||||
1709 | |||||
1710 | inline br_match m_UnconditionalBr(BasicBlock *&Succ) { return br_match(Succ); } | ||||
1711 | |||||
1712 | template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> | ||||
1713 | struct brc_match { | ||||
1714 | Cond_t Cond; | ||||
1715 | TrueBlock_t T; | ||||
1716 | FalseBlock_t F; | ||||
1717 | |||||
1718 | brc_match(const Cond_t &C, const TrueBlock_t &t, const FalseBlock_t &f) | ||||
1719 | : Cond(C), T(t), F(f) {} | ||||
1720 | |||||
1721 | template <typename OpTy> bool match(OpTy *V) { | ||||
1722 | if (auto *BI = dyn_cast<BranchInst>(V)) | ||||
1723 | if (BI->isConditional() && Cond.match(BI->getCondition())) | ||||
1724 | return T.match(BI->getSuccessor(0)) && F.match(BI->getSuccessor(1)); | ||||
1725 | return false; | ||||
1726 | } | ||||
1727 | }; | ||||
1728 | |||||
1729 | template <typename Cond_t> | ||||
1730 | inline brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>> | ||||
1731 | m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) { | ||||
1732 | return brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>( | ||||
1733 | C, m_BasicBlock(T), m_BasicBlock(F)); | ||||
1734 | } | ||||
1735 | |||||
1736 | template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> | ||||
1737 | inline brc_match<Cond_t, TrueBlock_t, FalseBlock_t> | ||||
1738 | m_Br(const Cond_t &C, const TrueBlock_t &T, const FalseBlock_t &F) { | ||||
1739 | return brc_match<Cond_t, TrueBlock_t, FalseBlock_t>(C, T, F); | ||||
1740 | } | ||||
1741 | |||||
1742 | //===----------------------------------------------------------------------===// | ||||
1743 | // Matchers for max/min idioms, eg: "select (sgt x, y), x, y" -> smax(x,y). | ||||
1744 | // | ||||
1745 | |||||
1746 | template <typename CmpInst_t, typename LHS_t, typename RHS_t, typename Pred_t, | ||||
1747 | bool Commutable = false> | ||||
1748 | struct MaxMin_match { | ||||
1749 | using PredType = Pred_t; | ||||
1750 | LHS_t L; | ||||
1751 | RHS_t R; | ||||
1752 | |||||
1753 | // The evaluation order is always stable, regardless of Commutability. | ||||
1754 | // The LHS is always matched first. | ||||
1755 | MaxMin_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} | ||||
1756 | |||||
1757 | template <typename OpTy> bool match(OpTy *V) { | ||||
1758 | if (auto *II = dyn_cast<IntrinsicInst>(V)) { | ||||
1759 | Intrinsic::ID IID = II->getIntrinsicID(); | ||||
1760 | if ((IID == Intrinsic::smax && Pred_t::match(ICmpInst::ICMP_SGT)) || | ||||
1761 | (IID == Intrinsic::smin && Pred_t::match(ICmpInst::ICMP_SLT)) || | ||||
1762 | (IID == Intrinsic::umax && Pred_t::match(ICmpInst::ICMP_UGT)) || | ||||
1763 | (IID == Intrinsic::umin && Pred_t::match(ICmpInst::ICMP_ULT))) { | ||||
1764 | Value *LHS = II->getOperand(0), *RHS = II->getOperand(1); | ||||
1765 | return (L.match(LHS) && R.match(RHS)) || | ||||
1766 | (Commutable && L.match(RHS) && R.match(LHS)); | ||||
1767 | } | ||||
1768 | } | ||||
1769 | // Look for "(x pred y) ? x : y" or "(x pred y) ? y : x". | ||||
1770 | auto *SI = dyn_cast<SelectInst>(V); | ||||
1771 | if (!SI) | ||||
1772 | return false; | ||||
1773 | auto *Cmp = dyn_cast<CmpInst_t>(SI->getCondition()); | ||||
1774 | if (!Cmp) | ||||
1775 | return false; | ||||
1776 | // At this point we have a select conditioned on a comparison. Check that | ||||
1777 | // it is the values returned by the select that are being compared. | ||||
1778 | auto *TrueVal = SI->getTrueValue(); | ||||
1779 | auto *FalseVal = SI->getFalseValue(); | ||||
1780 | auto *LHS = Cmp->getOperand(0); | ||||
1781 | auto *RHS = Cmp->getOperand(1); | ||||
1782 | if ((TrueVal != LHS || FalseVal != RHS) && | ||||
1783 | (TrueVal != RHS || FalseVal != LHS)) | ||||
1784 | return false; | ||||
1785 | typename CmpInst_t::Predicate Pred = | ||||
1786 | LHS == TrueVal ? Cmp->getPredicate() : Cmp->getInversePredicate(); | ||||
1787 | // Does "(x pred y) ? x : y" represent the desired max/min operation? | ||||
1788 | if (!Pred_t::match(Pred)) | ||||
1789 | return false; | ||||
1790 | // It does! Bind the operands. | ||||
1791 | return (L.match(LHS) && R.match(RHS)) || | ||||
1792 | (Commutable && L.match(RHS) && R.match(LHS)); | ||||
1793 | } | ||||
1794 | }; | ||||
1795 | |||||
1796 | /// Helper class for identifying signed max predicates. | ||||
1797 | struct smax_pred_ty { | ||||
1798 | static bool match(ICmpInst::Predicate Pred) { | ||||
1799 | return Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SGE; | ||||
1800 | } | ||||
1801 | }; | ||||
1802 | |||||
1803 | /// Helper class for identifying signed min predicates. | ||||
1804 | struct smin_pred_ty { | ||||
1805 | static bool match(ICmpInst::Predicate Pred) { | ||||
1806 | return Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SLE; | ||||
1807 | } | ||||
1808 | }; | ||||
1809 | |||||
1810 | /// Helper class for identifying unsigned max predicates. | ||||
1811 | struct umax_pred_ty { | ||||
1812 | static bool match(ICmpInst::Predicate Pred) { | ||||
1813 | return Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_UGE; | ||||
1814 | } | ||||
1815 | }; | ||||
1816 | |||||
1817 | /// Helper class for identifying unsigned min predicates. | ||||
1818 | struct umin_pred_ty { | ||||
1819 | static bool match(ICmpInst::Predicate Pred) { | ||||
1820 | return Pred == CmpInst::ICMP_ULT || Pred == CmpInst::ICMP_ULE; | ||||
1821 | } | ||||
1822 | }; | ||||
1823 | |||||
1824 | /// Helper class for identifying ordered max predicates. | ||||
1825 | struct ofmax_pred_ty { | ||||
1826 | static bool match(FCmpInst::Predicate Pred) { | ||||
1827 | return Pred == CmpInst::FCMP_OGT || Pred == CmpInst::FCMP_OGE; | ||||
1828 | } | ||||
1829 | }; | ||||
1830 | |||||
1831 | /// Helper class for identifying ordered min predicates. | ||||
1832 | struct ofmin_pred_ty { | ||||
1833 | static bool match(FCmpInst::Predicate Pred) { | ||||
1834 | return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE; | ||||
1835 | } | ||||
1836 | }; | ||||
1837 | |||||
1838 | /// Helper class for identifying unordered max predicates. | ||||
1839 | struct ufmax_pred_ty { | ||||
1840 | static bool match(FCmpInst::Predicate Pred) { | ||||
1841 | return Pred == CmpInst::FCMP_UGT || Pred == CmpInst::FCMP_UGE; | ||||
1842 | } | ||||
1843 | }; | ||||
1844 | |||||
1845 | /// Helper class for identifying unordered min predicates. | ||||
1846 | struct ufmin_pred_ty { | ||||
1847 | static bool match(FCmpInst::Predicate Pred) { | ||||
1848 | return Pred == CmpInst::FCMP_ULT || Pred == CmpInst::FCMP_ULE; | ||||
1849 | } | ||||
1850 | }; | ||||
1851 | |||||
1852 | template <typename LHS, typename RHS> | ||||
1853 | inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty> m_SMax(const LHS &L, | ||||
1854 | const RHS &R) { | ||||
1855 | return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>(L, R); | ||||
1856 | } | ||||
1857 | |||||
1858 | template <typename LHS, typename RHS> | ||||
1859 | inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty> m_SMin(const LHS &L, | ||||
1860 | const RHS &R) { | ||||
1861 | return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>(L, R); | ||||
1862 | } | ||||
1863 | |||||
1864 | template <typename LHS, typename RHS> | ||||
1865 | inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty> m_UMax(const LHS &L, | ||||
1866 | const RHS &R) { | ||||
1867 | return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>(L, R); | ||||
1868 | } | ||||
1869 | |||||
1870 | template <typename LHS, typename RHS> | ||||
1871 | inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty> m_UMin(const LHS &L, | ||||
1872 | const RHS &R) { | ||||
1873 | return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>(L, R); | ||||
1874 | } | ||||
1875 | |||||
1876 | template <typename LHS, typename RHS> | ||||
1877 | inline match_combine_or< | ||||
1878 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>, | ||||
1879 | MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>>, | ||||
1880 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>, | ||||
1881 | MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>>> | ||||
1882 | m_MaxOrMin(const LHS &L, const RHS &R) { | ||||
1883 | return m_CombineOr(m_CombineOr(m_SMax(L, R), m_SMin(L, R)), | ||||
1884 | m_CombineOr(m_UMax(L, R), m_UMin(L, R))); | ||||
1885 | } | ||||
1886 | |||||
1887 | /// Match an 'ordered' floating point maximum function. | ||||
1888 | /// Floating point has one special value 'NaN'. Therefore, there is no total | ||||
1889 | /// order. However, if we can ignore the 'NaN' value (for example, because of a | ||||
1890 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum' | ||||
1891 | /// semantics. In the presence of 'NaN' we have to preserve the original | ||||
1892 | /// select(fcmp(ogt/ge, L, R), L, R) semantics matched by this predicate. | ||||
1893 | /// | ||||
1894 | /// max(L, R) iff L and R are not NaN | ||||
1895 | /// m_OrdFMax(L, R) = R iff L or R are NaN | ||||
1896 | template <typename LHS, typename RHS> | ||||
1897 | inline MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty> m_OrdFMax(const LHS &L, | ||||
1898 | const RHS &R) { | ||||
1899 | return MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty>(L, R); | ||||
1900 | } | ||||
1901 | |||||
1902 | /// Match an 'ordered' floating point minimum function. | ||||
1903 | /// Floating point has one special value 'NaN'. Therefore, there is no total | ||||
1904 | /// order. However, if we can ignore the 'NaN' value (for example, because of a | ||||
1905 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum' | ||||
1906 | /// semantics. In the presence of 'NaN' we have to preserve the original | ||||
1907 | /// select(fcmp(olt/le, L, R), L, R) semantics matched by this predicate. | ||||
1908 | /// | ||||
1909 | /// min(L, R) iff L and R are not NaN | ||||
1910 | /// m_OrdFMin(L, R) = R iff L or R are NaN | ||||
1911 | template <typename LHS, typename RHS> | ||||
1912 | inline MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty> m_OrdFMin(const LHS &L, | ||||
1913 | const RHS &R) { | ||||
1914 | return MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty>(L, R); | ||||
1915 | } | ||||
1916 | |||||
1917 | /// Match an 'unordered' floating point maximum function. | ||||
1918 | /// Floating point has one special value 'NaN'. Therefore, there is no total | ||||
1919 | /// order. However, if we can ignore the 'NaN' value (for example, because of a | ||||
1920 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum' | ||||
1921 | /// semantics. In the presence of 'NaN' we have to preserve the original | ||||
1922 | /// select(fcmp(ugt/ge, L, R), L, R) semantics matched by this predicate. | ||||
1923 | /// | ||||
1924 | /// max(L, R) iff L and R are not NaN | ||||
1925 | /// m_UnordFMax(L, R) = L iff L or R are NaN | ||||
1926 | template <typename LHS, typename RHS> | ||||
1927 | inline MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty> | ||||
1928 | m_UnordFMax(const LHS &L, const RHS &R) { | ||||
1929 | return MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>(L, R); | ||||
1930 | } | ||||
1931 | |||||
1932 | /// Match an 'unordered' floating point minimum function. | ||||
1933 | /// Floating point has one special value 'NaN'. Therefore, there is no total | ||||
1934 | /// order. However, if we can ignore the 'NaN' value (for example, because of a | ||||
1935 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum' | ||||
1936 | /// semantics. In the presence of 'NaN' we have to preserve the original | ||||
1937 | /// select(fcmp(ult/le, L, R), L, R) semantics matched by this predicate. | ||||
1938 | /// | ||||
1939 | /// min(L, R) iff L and R are not NaN | ||||
1940 | /// m_UnordFMin(L, R) = L iff L or R are NaN | ||||
1941 | template <typename LHS, typename RHS> | ||||
1942 | inline MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty> | ||||
1943 | m_UnordFMin(const LHS &L, const RHS &R) { | ||||
1944 | return MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>(L, R); | ||||
1945 | } | ||||
1946 | |||||
1947 | //===----------------------------------------------------------------------===// | ||||
1948 | // Matchers for overflow check patterns: e.g. (a + b) u< a, (a ^ -1) <u b | ||||
1949 | // Note that S might be matched to other instructions than AddInst. | ||||
1950 | // | ||||
1951 | |||||
1952 | template <typename LHS_t, typename RHS_t, typename Sum_t> | ||||
1953 | struct UAddWithOverflow_match { | ||||
1954 | LHS_t L; | ||||
1955 | RHS_t R; | ||||
1956 | Sum_t S; | ||||
1957 | |||||
1958 | UAddWithOverflow_match(const LHS_t &L, const RHS_t &R, const Sum_t &S) | ||||
1959 | : L(L), R(R), S(S) {} | ||||
1960 | |||||
1961 | template <typename OpTy> bool match(OpTy *V) { | ||||
1962 | Value *ICmpLHS, *ICmpRHS; | ||||
1963 | ICmpInst::Predicate Pred; | ||||
1964 | if (!m_ICmp(Pred, m_Value(ICmpLHS), m_Value(ICmpRHS)).match(V)) | ||||
1965 | return false; | ||||
1966 | |||||
1967 | Value *AddLHS, *AddRHS; | ||||
1968 | auto AddExpr = m_Add(m_Value(AddLHS), m_Value(AddRHS)); | ||||
1969 | |||||
1970 | // (a + b) u< a, (a + b) u< b | ||||
1971 | if (Pred == ICmpInst::ICMP_ULT) | ||||
1972 | if (AddExpr.match(ICmpLHS) && (ICmpRHS == AddLHS || ICmpRHS == AddRHS)) | ||||
1973 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS); | ||||
1974 | |||||
1975 | // a >u (a + b), b >u (a + b) | ||||
1976 | if (Pred == ICmpInst::ICMP_UGT) | ||||
1977 | if (AddExpr.match(ICmpRHS) && (ICmpLHS == AddLHS || ICmpLHS == AddRHS)) | ||||
1978 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); | ||||
1979 | |||||
1980 | Value *Op1; | ||||
1981 | auto XorExpr = m_OneUse(m_Xor(m_Value(Op1), m_AllOnes())); | ||||
1982 | // (a ^ -1) <u b | ||||
1983 | if (Pred == ICmpInst::ICMP_ULT) { | ||||
1984 | if (XorExpr.match(ICmpLHS)) | ||||
1985 | return L.match(Op1) && R.match(ICmpRHS) && S.match(ICmpLHS); | ||||
1986 | } | ||||
1987 | // b > u (a ^ -1) | ||||
1988 | if (Pred == ICmpInst::ICMP_UGT) { | ||||
1989 | if (XorExpr.match(ICmpRHS)) | ||||
1990 | return L.match(Op1) && R.match(ICmpLHS) && S.match(ICmpRHS); | ||||
1991 | } | ||||
1992 | |||||
1993 | // Match special-case for increment-by-1. | ||||
1994 | if (Pred == ICmpInst::ICMP_EQ) { | ||||
1995 | // (a + 1) == 0 | ||||
1996 | // (1 + a) == 0 | ||||
1997 | if (AddExpr.match(ICmpLHS) && m_ZeroInt().match(ICmpRHS) && | ||||
1998 | (m_One().match(AddLHS) || m_One().match(AddRHS))) | ||||
1999 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS); | ||||
2000 | // 0 == (a + 1) | ||||
2001 | // 0 == (1 + a) | ||||
2002 | if (m_ZeroInt().match(ICmpLHS) && AddExpr.match(ICmpRHS) && | ||||
2003 | (m_One().match(AddLHS) || m_One().match(AddRHS))) | ||||
2004 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); | ||||
2005 | } | ||||
2006 | |||||
2007 | return false; | ||||
2008 | } | ||||
2009 | }; | ||||
2010 | |||||
2011 | /// Match an icmp instruction checking for unsigned overflow on addition. | ||||
2012 | /// | ||||
2013 | /// S is matched to the addition whose result is being checked for overflow, and | ||||
2014 | /// L and R are matched to the LHS and RHS of S. | ||||
2015 | template <typename LHS_t, typename RHS_t, typename Sum_t> | ||||
2016 | UAddWithOverflow_match<LHS_t, RHS_t, Sum_t> | ||||
2017 | m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S) { | ||||
2018 | return UAddWithOverflow_match<LHS_t, RHS_t, Sum_t>(L, R, S); | ||||
2019 | } | ||||
2020 | |||||
2021 | template <typename Opnd_t> struct Argument_match { | ||||
2022 | unsigned OpI; | ||||
2023 | Opnd_t Val; | ||||
2024 | |||||
2025 | Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {} | ||||
2026 | |||||
2027 | template <typename OpTy> bool match(OpTy *V) { | ||||
2028 | // FIXME: Should likely be switched to use `CallBase`. | ||||
2029 | if (const auto *CI = dyn_cast<CallInst>(V)) | ||||
2030 | return Val.match(CI->getArgOperand(OpI)); | ||||
2031 | return false; | ||||
2032 | } | ||||
2033 | }; | ||||
2034 | |||||
2035 | /// Match an argument. | ||||
2036 | template <unsigned OpI, typename Opnd_t> | ||||
2037 | inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) { | ||||
2038 | return Argument_match<Opnd_t>(OpI, Op); | ||||
2039 | } | ||||
2040 | |||||
2041 | /// Intrinsic matchers. | ||||
2042 | struct IntrinsicID_match { | ||||
2043 | unsigned ID; | ||||
2044 | |||||
2045 | IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {} | ||||
2046 | |||||
2047 | template <typename OpTy> bool match(OpTy *V) { | ||||
2048 | if (const auto *CI = dyn_cast<CallInst>(V)) | ||||
2049 | if (const auto *F = CI->getCalledFunction()) | ||||
2050 | return F->getIntrinsicID() == ID; | ||||
2051 | return false; | ||||
2052 | } | ||||
2053 | }; | ||||
2054 | |||||
2055 | /// Intrinsic matches are combinations of ID matchers, and argument | ||||
2056 | /// matchers. Higher arity matcher are defined recursively in terms of and-ing | ||||
2057 | /// them with lower arity matchers. Here's some convenient typedefs for up to | ||||
2058 | /// several arguments, and more can be added as needed | ||||
2059 | template <typename T0 = void, typename T1 = void, typename T2 = void, | ||||
2060 | typename T3 = void, typename T4 = void, typename T5 = void, | ||||
2061 | typename T6 = void, typename T7 = void, typename T8 = void, | ||||
2062 | typename T9 = void, typename T10 = void> | ||||
2063 | struct m_Intrinsic_Ty; | ||||
2064 | template <typename T0> struct m_Intrinsic_Ty<T0> { | ||||
2065 | using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>; | ||||
2066 | }; | ||||
2067 | template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> { | ||||
2068 | using Ty = | ||||
2069 | match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>; | ||||
2070 | }; | ||||
2071 | template <typename T0, typename T1, typename T2> | ||||
2072 | struct m_Intrinsic_Ty<T0, T1, T2> { | ||||
2073 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty, | ||||
2074 | Argument_match<T2>>; | ||||
2075 | }; | ||||
2076 | template <typename T0, typename T1, typename T2, typename T3> | ||||
2077 | struct m_Intrinsic_Ty<T0, T1, T2, T3> { | ||||
2078 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty, | ||||
2079 | Argument_match<T3>>; | ||||
2080 | }; | ||||
2081 | |||||
2082 | template <typename T0, typename T1, typename T2, typename T3, typename T4> | ||||
2083 | struct m_Intrinsic_Ty<T0, T1, T2, T3, T4> { | ||||
2084 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty, | ||||
2085 | Argument_match<T4>>; | ||||
2086 | }; | ||||
2087 | |||||
2088 | template <typename T0, typename T1, typename T2, typename T3, typename T4, | ||||
2089 | typename T5> | ||||
2090 | struct m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5> { | ||||
2091 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty, | ||||
2092 | Argument_match<T5>>; | ||||
2093 | }; | ||||
2094 | |||||
2095 | /// Match intrinsic calls like this: | ||||
2096 | /// m_Intrinsic<Intrinsic::fabs>(m_Value(X)) | ||||
2097 | template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() { | ||||
2098 | return IntrinsicID_match(IntrID); | ||||
2099 | } | ||||
2100 | |||||
2101 | /// Matches MaskedLoad Intrinsic. | ||||
2102 | template <typename Opnd0, typename Opnd1, typename Opnd2, typename Opnd3> | ||||
2103 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2, Opnd3>::Ty | ||||
2104 | m_MaskedLoad(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2, | ||||
2105 | const Opnd3 &Op3) { | ||||
2106 | return m_Intrinsic<Intrinsic::masked_load>(Op0, Op1, Op2, Op3); | ||||
2107 | } | ||||
2108 | |||||
2109 | /// Matches MaskedGather Intrinsic. | ||||
2110 | template <typename Opnd0, typename Opnd1, typename Opnd2, typename Opnd3> | ||||
2111 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2, Opnd3>::Ty | ||||
2112 | m_MaskedGather(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2, | ||||
2113 | const Opnd3 &Op3) { | ||||
2114 | return m_Intrinsic<Intrinsic::masked_gather>(Op0, Op1, Op2, Op3); | ||||
2115 | } | ||||
2116 | |||||
2117 | template <Intrinsic::ID IntrID, typename T0> | ||||
2118 | inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) { | ||||
2119 | return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0)); | ||||
2120 | } | ||||
2121 | |||||
2122 | template <Intrinsic::ID IntrID, typename T0, typename T1> | ||||
2123 | inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0, | ||||
2124 | const T1 &Op1) { | ||||
2125 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1)); | ||||
2126 | } | ||||
2127 | |||||
2128 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2> | ||||
2129 | inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty | ||||
2130 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) { | ||||
2131 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2)); | ||||
2132 | } | ||||
2133 | |||||
2134 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, | ||||
2135 | typename T3> | ||||
2136 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty | ||||
2137 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) { | ||||
2138 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3)); | ||||
2139 | } | ||||
2140 | |||||
2141 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, | ||||
2142 | typename T3, typename T4> | ||||
2143 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty | ||||
2144 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3, | ||||
2145 | const T4 &Op4) { | ||||
2146 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3), | ||||
2147 | m_Argument<4>(Op4)); | ||||
2148 | } | ||||
2149 | |||||
2150 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, | ||||
2151 | typename T3, typename T4, typename T5> | ||||
2152 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5>::Ty | ||||
2153 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3, | ||||
2154 | const T4 &Op4, const T5 &Op5) { | ||||
2155 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3, Op4), | ||||
2156 | m_Argument<5>(Op5)); | ||||
2157 | } | ||||
2158 | |||||
2159 | // Helper intrinsic matching specializations. | ||||
2160 | template <typename Opnd0> | ||||
2161 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BitReverse(const Opnd0 &Op0) { | ||||
2162 | return m_Intrinsic<Intrinsic::bitreverse>(Op0); | ||||
2163 | } | ||||
2164 | |||||
2165 | template <typename Opnd0> | ||||
2166 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BSwap(const Opnd0 &Op0) { | ||||
2167 | return m_Intrinsic<Intrinsic::bswap>(Op0); | ||||
2168 | } | ||||
2169 | |||||
2170 | template <typename Opnd0> | ||||
2171 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FAbs(const Opnd0 &Op0) { | ||||
2172 | return m_Intrinsic<Intrinsic::fabs>(Op0); | ||||
2173 | } | ||||
2174 | |||||
2175 | template <typename Opnd0> | ||||
2176 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FCanonicalize(const Opnd0 &Op0) { | ||||
2177 | return m_Intrinsic<Intrinsic::canonicalize>(Op0); | ||||
2178 | } | ||||
2179 | |||||
2180 | template <typename Opnd0, typename Opnd1> | ||||
2181 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMin(const Opnd0 &Op0, | ||||
2182 | const Opnd1 &Op1) { | ||||
2183 | return m_Intrinsic<Intrinsic::minnum>(Op0, Op1); | ||||
2184 | } | ||||
2185 | |||||
2186 | template <typename Opnd0, typename Opnd1> | ||||
2187 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMax(const Opnd0 &Op0, | ||||
2188 | const Opnd1 &Op1) { | ||||
2189 | return m_Intrinsic<Intrinsic::maxnum>(Op0, Op1); | ||||
2190 | } | ||||
2191 | |||||
2192 | template <typename Opnd0, typename Opnd1, typename Opnd2> | ||||
2193 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty | ||||
2194 | m_FShl(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) { | ||||
2195 | return m_Intrinsic<Intrinsic::fshl>(Op0, Op1, Op2); | ||||
2196 | } | ||||
2197 | |||||
2198 | template <typename Opnd0, typename Opnd1, typename Opnd2> | ||||
2199 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty | ||||
2200 | m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) { | ||||
2201 | return m_Intrinsic<Intrinsic::fshr>(Op0, Op1, Op2); | ||||
2202 | } | ||||
2203 | |||||
2204 | template <typename Opnd0> | ||||
2205 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_Sqrt(const Opnd0 &Op0) { | ||||
2206 | return m_Intrinsic<Intrinsic::sqrt>(Op0); | ||||
2207 | } | ||||
2208 | |||||
2209 | template <typename Opnd0, typename Opnd1> | ||||
2210 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_CopySign(const Opnd0 &Op0, | ||||
2211 | const Opnd1 &Op1) { | ||||
2212 | return m_Intrinsic<Intrinsic::copysign>(Op0, Op1); | ||||
2213 | } | ||||
2214 | |||||
2215 | template <typename Opnd0> | ||||
2216 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_VecReverse(const Opnd0 &Op0) { | ||||
2217 | return m_Intrinsic<Intrinsic::experimental_vector_reverse>(Op0); | ||||
2218 | } | ||||
2219 | |||||
2220 | //===----------------------------------------------------------------------===// | ||||
2221 | // Matchers for two-operands operators with the operators in either order | ||||
2222 | // | ||||
2223 | |||||
2224 | /// Matches a BinaryOperator with LHS and RHS in either order. | ||||
2225 | template <typename LHS, typename RHS> | ||||
2226 | inline AnyBinaryOp_match<LHS, RHS, true> m_c_BinOp(const LHS &L, const RHS &R) { | ||||
2227 | return AnyBinaryOp_match<LHS, RHS, true>(L, R); | ||||
2228 | } | ||||
2229 | |||||
2230 | /// Matches an ICmp with a predicate over LHS and RHS in either order. | ||||
2231 | /// Swaps the predicate if operands are commuted. | ||||
2232 | template <typename LHS, typename RHS> | ||||
2233 | inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true> | ||||
2234 | m_c_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { | ||||
2235 | return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true>(Pred, L, | ||||
2236 | R); | ||||
2237 | } | ||||
2238 | |||||
2239 | /// Matches a specific opcode with LHS and RHS in either order. | ||||
2240 | template <typename LHS, typename RHS> | ||||
2241 | inline SpecificBinaryOp_match<LHS, RHS, true> | ||||
2242 | m_c_BinOp(unsigned Opcode, const LHS &L, const RHS &R) { | ||||
2243 | return SpecificBinaryOp_match<LHS, RHS, true>(Opcode, L, R); | ||||
2244 | } | ||||
2245 | |||||
2246 | /// Matches a Add with LHS and RHS in either order. | ||||
2247 | template <typename LHS, typename RHS> | ||||
2248 | inline BinaryOp_match<LHS, RHS, Instruction::Add, true> m_c_Add(const LHS &L, | ||||
2249 | const RHS &R) { | ||||
2250 | return BinaryOp_match<LHS, RHS, Instruction::Add, true>(L, R); | ||||
2251 | } | ||||
2252 | |||||
2253 | /// Matches a Mul with LHS and RHS in either order. | ||||
2254 | template <typename LHS, typename RHS> | ||||
2255 | inline BinaryOp_match<LHS, RHS, Instruction::Mul, true> m_c_Mul(const LHS &L, | ||||
2256 | const RHS &R) { | ||||
2257 | return BinaryOp_match<LHS, RHS, Instruction::Mul, true>(L, R); | ||||
2258 | } | ||||
2259 | |||||
2260 | /// Matches an And with LHS and RHS in either order. | ||||
2261 | template <typename LHS, typename RHS> | ||||
2262 | inline BinaryOp_match<LHS, RHS, Instruction::And, true> m_c_And(const LHS &L, | ||||
2263 | const RHS &R) { | ||||
2264 | return BinaryOp_match<LHS, RHS, Instruction::And, true>(L, R); | ||||
2265 | } | ||||
2266 | |||||
2267 | /// Matches an Or with LHS and RHS in either order. | ||||
2268 | template <typename LHS, typename RHS> | ||||
2269 | inline BinaryOp_match<LHS, RHS, Instruction::Or, true> m_c_Or(const LHS &L, | ||||
2270 | const RHS &R) { | ||||
2271 | return BinaryOp_match<LHS, RHS, Instruction::Or, true>(L, R); | ||||
2272 | } | ||||
2273 | |||||
2274 | /// Matches an Xor with LHS and RHS in either order. | ||||
2275 | template <typename LHS, typename RHS> | ||||
2276 | inline BinaryOp_match<LHS, RHS, Instruction::Xor, true> m_c_Xor(const LHS &L, | ||||
2277 | const RHS &R) { | ||||
2278 | return BinaryOp_match<LHS, RHS, Instruction::Xor, true>(L, R); | ||||
2279 | } | ||||
2280 | |||||
2281 | /// Matches a 'Neg' as 'sub 0, V'. | ||||
2282 | template <typename ValTy> | ||||
2283 | inline BinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, Instruction::Sub> | ||||
2284 | m_Neg(const ValTy &V) { | ||||
2285 | return m_Sub(m_ZeroInt(), V); | ||||
2286 | } | ||||
2287 | |||||
2288 | /// Matches a 'Neg' as 'sub nsw 0, V'. | ||||
2289 | template <typename ValTy> | ||||
2290 | inline OverflowingBinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, | ||||
2291 | Instruction::Sub, | ||||
2292 | OverflowingBinaryOperator::NoSignedWrap> | ||||
2293 | m_NSWNeg(const ValTy &V) { | ||||
2294 | return m_NSWSub(m_ZeroInt(), V); | ||||
2295 | } | ||||
2296 | |||||
2297 | /// Matches a 'Not' as 'xor V, -1' or 'xor -1, V'. | ||||
2298 | /// NOTE: we first match the 'Not' (by matching '-1'), | ||||
2299 | /// and only then match the inner matcher! | ||||
2300 | template <typename ValTy> | ||||
2301 | inline BinaryOp_match<cst_pred_ty<is_all_ones>, ValTy, Instruction::Xor, true> | ||||
2302 | m_Not(const ValTy &V) { | ||||
2303 | return m_c_Xor(m_AllOnes(), V); | ||||
2304 | } | ||||
2305 | |||||
2306 | template <typename ValTy> struct NotForbidUndef_match { | ||||
2307 | ValTy Val; | ||||
2308 | NotForbidUndef_match(const ValTy &V) : Val(V) {} | ||||
2309 | |||||
2310 | template <typename OpTy> bool match(OpTy *V) { | ||||
2311 | // We do not use m_c_Xor because that could match an arbitrary APInt that is | ||||
2312 | // not -1 as C and then fail to match the other operand if it is -1. | ||||
2313 | // This code should still work even when both operands are constants. | ||||
2314 | Value *X; | ||||
2315 | const APInt *C; | ||||
2316 | if (m_Xor(m_Value(X), m_APIntForbidUndef(C)).match(V) && C->isAllOnes()) | ||||
2317 | return Val.match(X); | ||||
2318 | if (m_Xor(m_APIntForbidUndef(C), m_Value(X)).match(V) && C->isAllOnes()) | ||||
2319 | return Val.match(X); | ||||
2320 | return false; | ||||
2321 | } | ||||
2322 | }; | ||||
2323 | |||||
2324 | /// Matches a bitwise 'not' as 'xor V, -1' or 'xor -1, V'. For vectors, the | ||||
2325 | /// constant value must be composed of only -1 scalar elements. | ||||
2326 | template <typename ValTy> | ||||
2327 | inline NotForbidUndef_match<ValTy> m_NotForbidUndef(const ValTy &V) { | ||||
2328 | return NotForbidUndef_match<ValTy>(V); | ||||
2329 | } | ||||
2330 | |||||
2331 | /// Matches an SMin with LHS and RHS in either order. | ||||
2332 | template <typename LHS, typename RHS> | ||||
2333 | inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true> | ||||
2334 | m_c_SMin(const LHS &L, const RHS &R) { | ||||
2335 | return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>(L, R); | ||||
2336 | } | ||||
2337 | /// Matches an SMax with LHS and RHS in either order. | ||||
2338 | template <typename LHS, typename RHS> | ||||
2339 | inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true> | ||||
2340 | m_c_SMax(const LHS &L, const RHS &R) { | ||||
2341 | return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>(L, R); | ||||
2342 | } | ||||
2343 | /// Matches a UMin with LHS and RHS in either order. | ||||
2344 | template <typename LHS, typename RHS> | ||||
2345 | inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true> | ||||
2346 | m_c_UMin(const LHS &L, const RHS &R) { | ||||
2347 | return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>(L, R); | ||||
2348 | } | ||||
2349 | /// Matches a UMax with LHS and RHS in either order. | ||||
2350 | template <typename LHS, typename RHS> | ||||
2351 | inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true> | ||||
2352 | m_c_UMax(const LHS &L, const RHS &R) { | ||||
2353 | return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>(L, R); | ||||
2354 | } | ||||
2355 | |||||
2356 | template <typename LHS, typename RHS> | ||||
2357 | inline match_combine_or< | ||||
2358 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>, | ||||
2359 | MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>>, | ||||
2360 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>, | ||||
2361 | MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>>> | ||||
2362 | m_c_MaxOrMin(const LHS &L, const RHS &R) { | ||||
2363 | return m_CombineOr(m_CombineOr(m_c_SMax(L, R), m_c_SMin(L, R)), | ||||
2364 | m_CombineOr(m_c_UMax(L, R), m_c_UMin(L, R))); | ||||
2365 | } | ||||
2366 | |||||
2367 | template <Intrinsic::ID IntrID, typename T0, typename T1> | ||||
2368 | inline match_combine_or<typename m_Intrinsic_Ty<T0, T1>::Ty, | ||||
2369 | typename m_Intrinsic_Ty<T1, T0>::Ty> | ||||
2370 | m_c_Intrinsic(const T0 &Op0, const T1 &Op1) { | ||||
2371 | return m_CombineOr(m_Intrinsic<IntrID>(Op0, Op1), | ||||
2372 | m_Intrinsic<IntrID>(Op1, Op0)); | ||||
2373 | } | ||||
2374 | |||||
2375 | /// Matches FAdd with LHS and RHS in either order. | ||||
2376 | template <typename LHS, typename RHS> | ||||
2377 | inline BinaryOp_match<LHS, RHS, Instruction::FAdd, true> | ||||
2378 | m_c_FAdd(const LHS &L, const RHS &R) { | ||||
2379 | return BinaryOp_match<LHS, RHS, Instruction::FAdd, true>(L, R); | ||||
2380 | } | ||||
2381 | |||||
2382 | /// Matches FMul with LHS and RHS in either order. | ||||
2383 | template <typename LHS, typename RHS> | ||||
2384 | inline BinaryOp_match<LHS, RHS, Instruction::FMul, true> | ||||
2385 | m_c_FMul(const LHS &L, const RHS &R) { | ||||
2386 | return BinaryOp_match<LHS, RHS, Instruction::FMul, true>(L, R); | ||||
2387 | } | ||||
2388 | |||||
2389 | template <typename Opnd_t> struct Signum_match { | ||||
2390 | Opnd_t Val; | ||||
2391 | Signum_match(const Opnd_t &V) : Val(V) {} | ||||
2392 | |||||
2393 | template <typename OpTy> bool match(OpTy *V) { | ||||
2394 | unsigned TypeSize = V->getType()->getScalarSizeInBits(); | ||||
2395 | if (TypeSize == 0) | ||||
2396 | return false; | ||||
2397 | |||||
2398 | unsigned ShiftWidth = TypeSize - 1; | ||||
2399 | Value *OpL = nullptr, *OpR = nullptr; | ||||
2400 | |||||
2401 | // This is the representation of signum we match: | ||||
2402 | // | ||||
2403 | // signum(x) == (x >> 63) | (-x >>u 63) | ||||
2404 | // | ||||
2405 | // An i1 value is its own signum, so it's correct to match | ||||
2406 | // | ||||
2407 | // signum(x) == (x >> 0) | (-x >>u 0) | ||||
2408 | // | ||||
2409 | // for i1 values. | ||||
2410 | |||||
2411 | auto LHS = m_AShr(m_Value(OpL), m_SpecificInt(ShiftWidth)); | ||||
2412 | auto RHS = m_LShr(m_Neg(m_Value(OpR)), m_SpecificInt(ShiftWidth)); | ||||
2413 | auto Signum = m_Or(LHS, RHS); | ||||
2414 | |||||
2415 | return Signum.match(V) && OpL == OpR && Val.match(OpL); | ||||
2416 | } | ||||
2417 | }; | ||||
2418 | |||||
2419 | /// Matches a signum pattern. | ||||
2420 | /// | ||||
2421 | /// signum(x) = | ||||
2422 | /// x > 0 -> 1 | ||||
2423 | /// x == 0 -> 0 | ||||
2424 | /// x < 0 -> -1 | ||||
2425 | template <typename Val_t> inline Signum_match<Val_t> m_Signum(const Val_t &V) { | ||||
2426 | return Signum_match<Val_t>(V); | ||||
2427 | } | ||||
2428 | |||||
2429 | template <int Ind, typename Opnd_t> struct ExtractValue_match { | ||||
2430 | Opnd_t Val; | ||||
2431 | ExtractValue_match(const Opnd_t &V) : Val(V) {} | ||||
2432 | |||||
2433 | template <typename OpTy> bool match(OpTy *V) { | ||||
2434 | if (auto *I = dyn_cast<ExtractValueInst>(V)) { | ||||
2435 | // If Ind is -1, don't inspect indices | ||||
2436 | if (Ind != -1 && | ||||
2437 | !(I->getNumIndices() == 1 && I->getIndices()[0] == (unsigned)Ind)) | ||||
2438 | return false; | ||||
2439 | return Val.match(I->getAggregateOperand()); | ||||
2440 | } | ||||
2441 | return false; | ||||
2442 | } | ||||
2443 | }; | ||||
2444 | |||||
2445 | /// Match a single index ExtractValue instruction. | ||||
2446 | /// For example m_ExtractValue<1>(...) | ||||
2447 | template <int Ind, typename Val_t> | ||||
2448 | inline ExtractValue_match<Ind, Val_t> m_ExtractValue(const Val_t &V) { | ||||
2449 | return ExtractValue_match<Ind, Val_t>(V); | ||||
2450 | } | ||||
2451 | |||||
2452 | /// Match an ExtractValue instruction with any index. | ||||
2453 | /// For example m_ExtractValue(...) | ||||
2454 | template <typename Val_t> | ||||
2455 | inline ExtractValue_match<-1, Val_t> m_ExtractValue(const Val_t &V) { | ||||
2456 | return ExtractValue_match<-1, Val_t>(V); | ||||
2457 | } | ||||
2458 | |||||
2459 | /// Matcher for a single index InsertValue instruction. | ||||
2460 | template <int Ind, typename T0, typename T1> struct InsertValue_match { | ||||
2461 | T0 Op0; | ||||
2462 | T1 Op1; | ||||
2463 | |||||
2464 | InsertValue_match(const T0 &Op0, const T1 &Op1) : Op0(Op0), Op1(Op1) {} | ||||
2465 | |||||
2466 | template <typename OpTy> bool match(OpTy *V) { | ||||
2467 | if (auto *I = dyn_cast<InsertValueInst>(V)) { | ||||
2468 | return Op0.match(I->getOperand(0)) && Op1.match(I->getOperand(1)) && | ||||
2469 | I->getNumIndices() == 1 && Ind == I->getIndices()[0]; | ||||
2470 | } | ||||
2471 | return false; | ||||
2472 | } | ||||
2473 | }; | ||||
2474 | |||||
2475 | /// Matches a single index InsertValue instruction. | ||||
2476 | template <int Ind, typename Val_t, typename Elt_t> | ||||
2477 | inline InsertValue_match<Ind, Val_t, Elt_t> m_InsertValue(const Val_t &Val, | ||||
2478 | const Elt_t &Elt) { | ||||
2479 | return InsertValue_match<Ind, Val_t, Elt_t>(Val, Elt); | ||||
2480 | } | ||||
2481 | |||||
2482 | /// Matches patterns for `vscale`. This can either be a call to `llvm.vscale` or | ||||
2483 | /// the constant expression | ||||
2484 | /// `ptrtoint(gep <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1>` | ||||
2485 | /// under the right conditions determined by DataLayout. | ||||
2486 | struct VScaleVal_match { | ||||
2487 | template <typename ITy> bool match(ITy *V) { | ||||
2488 | if (m_Intrinsic<Intrinsic::vscale>().match(V)) | ||||
2489 | return true; | ||||
2490 | |||||
2491 | Value *Ptr; | ||||
2492 | if (m_PtrToInt(m_Value(Ptr)).match(V)) { | ||||
2493 | if (auto *GEP = dyn_cast<GEPOperator>(Ptr)) { | ||||
2494 | auto *DerefTy = | ||||
2495 | dyn_cast<ScalableVectorType>(GEP->getSourceElementType()); | ||||
2496 | if (GEP->getNumIndices() == 1 && DerefTy && | ||||
2497 | DerefTy->getElementType()->isIntegerTy(8) && | ||||
2498 | m_Zero().match(GEP->getPointerOperand()) && | ||||
2499 | m_SpecificInt(1).match(GEP->idx_begin()->get())) | ||||
2500 | return true; | ||||
2501 | } | ||||
2502 | } | ||||
2503 | |||||
2504 | return false; | ||||
2505 | } | ||||
2506 | }; | ||||
2507 | |||||
2508 | inline VScaleVal_match m_VScale() { | ||||
2509 | return VScaleVal_match(); | ||||
2510 | } | ||||
2511 | |||||
2512 | template <typename LHS, typename RHS, unsigned Opcode, bool Commutable = false> | ||||
2513 | struct LogicalOp_match { | ||||
2514 | LHS L; | ||||
2515 | RHS R; | ||||
2516 | |||||
2517 | LogicalOp_match(const LHS &L, const RHS &R) : L(L), R(R) {} | ||||
2518 | |||||
2519 | template <typename T> bool match(T *V) { | ||||
2520 | auto *I = dyn_cast<Instruction>(V); | ||||
2521 | if (!I || !I->getType()->isIntOrIntVectorTy(1)) | ||||
2522 | return false; | ||||
2523 | |||||
2524 | if (I->getOpcode() == Opcode) { | ||||
2525 | auto *Op0 = I->getOperand(0); | ||||
2526 | auto *Op1 = I->getOperand(1); | ||||
2527 | return (L.match(Op0) && R.match(Op1)) || | ||||
2528 | (Commutable && L.match(Op1) && R.match(Op0)); | ||||
2529 | } | ||||
2530 | |||||
2531 | if (auto *Select = dyn_cast<SelectInst>(I)) { | ||||
2532 | auto *Cond = Select->getCondition(); | ||||
2533 | auto *TVal = Select->getTrueValue(); | ||||
2534 | auto *FVal = Select->getFalseValue(); | ||||
2535 | |||||
2536 | // Don't match a scalar select of bool vectors. | ||||
2537 | // Transforms expect a single type for operands if this matches. | ||||
2538 | if (Cond->getType() != Select->getType()) | ||||
2539 | return false; | ||||
2540 | |||||
2541 | if (Opcode == Instruction::And) { | ||||
2542 | auto *C = dyn_cast<Constant>(FVal); | ||||
2543 | if (C && C->isNullValue()) | ||||
2544 | return (L.match(Cond) && R.match(TVal)) || | ||||
2545 | (Commutable && L.match(TVal) && R.match(Cond)); | ||||
2546 | } else { | ||||
2547 | assert(Opcode == Instruction::Or)(static_cast <bool> (Opcode == Instruction::Or) ? void ( 0) : __assert_fail ("Opcode == Instruction::Or", "llvm/include/llvm/IR/PatternMatch.h" , 2547, __extension__ __PRETTY_FUNCTION__)); | ||||
2548 | auto *C = dyn_cast<Constant>(TVal); | ||||
2549 | if (C && C->isOneValue()) | ||||
2550 | return (L.match(Cond) && R.match(FVal)) || | ||||
2551 | (Commutable && L.match(FVal) && R.match(Cond)); | ||||
2552 | } | ||||
2553 | } | ||||
2554 | |||||
2555 | return false; | ||||
2556 | } | ||||
2557 | }; | ||||
2558 | |||||
2559 | /// Matches L && R either in the form of L & R or L ? R : false. | ||||
2560 | /// Note that the latter form is poison-blocking. | ||||
2561 | template <typename LHS, typename RHS> | ||||
2562 | inline LogicalOp_match<LHS, RHS, Instruction::And> m_LogicalAnd(const LHS &L, | ||||
2563 | const RHS &R) { | ||||
2564 | return LogicalOp_match<LHS, RHS, Instruction::And>(L, R); | ||||
2565 | } | ||||
2566 | |||||
2567 | /// Matches L && R where L and R are arbitrary values. | ||||
2568 | inline auto m_LogicalAnd() { return m_LogicalAnd(m_Value(), m_Value()); } | ||||
2569 | |||||
2570 | /// Matches L && R with LHS and RHS in either order. | ||||
2571 | template <typename LHS, typename RHS> | ||||
2572 | inline LogicalOp_match<LHS, RHS, Instruction::And, true> | ||||
2573 | m_c_LogicalAnd(const LHS &L, const RHS &R) { | ||||
2574 | return LogicalOp_match<LHS, RHS, Instruction::And, true>(L, R); | ||||
2575 | } | ||||
2576 | |||||
2577 | /// Matches L || R either in the form of L | R or L ? true : R. | ||||
2578 | /// Note that the latter form is poison-blocking. | ||||
2579 | template <typename LHS, typename RHS> | ||||
2580 | inline LogicalOp_match<LHS, RHS, Instruction::Or> m_LogicalOr(const LHS &L, | ||||
2581 | const RHS &R) { | ||||
2582 | return LogicalOp_match<LHS, RHS, Instruction::Or>(L, R); | ||||
2583 | } | ||||
2584 | |||||
2585 | /// Matches L || R where L and R are arbitrary values. | ||||
2586 | inline auto m_LogicalOr() { return m_LogicalOr(m_Value(), m_Value()); } | ||||
2587 | |||||
2588 | /// Matches L || R with LHS and RHS in either order. | ||||
2589 | template <typename LHS, typename RHS> | ||||
2590 | inline LogicalOp_match<LHS, RHS, Instruction::Or, true> | ||||
2591 | m_c_LogicalOr(const LHS &L, const RHS &R) { | ||||
2592 | return LogicalOp_match<LHS, RHS, Instruction::Or, true>(L, R); | ||||
2593 | } | ||||
2594 | |||||
2595 | /// Matches either L && R or L || R, | ||||
2596 | /// either one being in the either binary or logical form. | ||||
2597 | /// Note that the latter form is poison-blocking. | ||||
2598 | template <typename LHS, typename RHS, bool Commutable = false> | ||||
2599 | inline auto m_LogicalOp(const LHS &L, const RHS &R) { | ||||
2600 | return m_CombineOr( | ||||
2601 | LogicalOp_match<LHS, RHS, Instruction::And, Commutable>(L, R), | ||||
2602 | LogicalOp_match<LHS, RHS, Instruction::Or, Commutable>(L, R)); | ||||
2603 | } | ||||
2604 | |||||
2605 | /// Matches either L && R or L || R where L and R are arbitrary values. | ||||
2606 | inline auto m_LogicalOp() { return m_LogicalOp(m_Value(), m_Value()); } | ||||
2607 | |||||
2608 | /// Matches either L && R or L || R with LHS and RHS in either order. | ||||
2609 | template <typename LHS, typename RHS> | ||||
2610 | inline auto m_c_LogicalOp(const LHS &L, const RHS &R) { | ||||
2611 | return m_LogicalOp<LHS, RHS, /*Commutable=*/true>(L, R); | ||||
2612 | } | ||||
2613 | |||||
2614 | } // end namespace PatternMatch | ||||
2615 | } // end namespace llvm | ||||
2616 | |||||
2617 | #endif // LLVM_IR_PATTERNMATCH_H |