LLVM 24.0.0git
InstCombineAndOrXor.cpp
Go to the documentation of this file.
1//===- InstCombineAndOrXor.cpp --------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visitAnd, visitOr, and visitXor functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
21#include "llvm/IR/Intrinsics.h"
26
27using namespace llvm;
28using namespace PatternMatch;
29
30#define DEBUG_TYPE "instcombine"
31
32namespace llvm {
34}
35
36/// This is the complement of getICmpCode, which turns an opcode and two
37/// operands into either a constant true or false, or a brand new ICmp
38/// instruction. The sign is passed in to determine which kind of predicate to
39/// use in the new icmp instruction.
40static Value *getNewICmpValue(unsigned Code, bool Sign, Value *LHS, Value *RHS,
41 InstCombiner::BuilderTy &Builder) {
42 ICmpInst::Predicate NewPred;
43 if (Constant *TorF = getPredForICmpCode(Code, Sign, LHS->getType(), NewPred))
44 return TorF;
45 return Builder.CreateICmp(NewPred, LHS, RHS);
46}
47
48/// This is the complement of getFCmpCode, which turns an opcode and two
49/// operands into either a FCmp instruction, or a true/false constant.
50static Value *getFCmpValue(unsigned Code, Value *LHS, Value *RHS,
51 InstCombiner::BuilderTy &Builder, FMFSource FMF) {
52 FCmpInst::Predicate NewPred;
53 if (Constant *TorF = getPredForFCmpCode(Code, LHS->getType(), NewPred))
54 return TorF;
55 return Builder.CreateFCmpFMF(NewPred, LHS, RHS, FMF);
56}
57
58/// Emit a computation of: (V >= Lo && V < Hi) if Inside is true, otherwise
59/// (V < Lo || V >= Hi). This method expects that Lo < Hi. IsSigned indicates
60/// whether to treat V, Lo, and Hi as signed or not.
62 const APInt &Hi, bool isSigned,
63 bool Inside) {
64 assert((isSigned ? Lo.slt(Hi) : Lo.ult(Hi)) &&
65 "Lo is not < Hi in range emission code!");
66
67 Type *Ty = V->getType();
68
69 // V >= Min && V < Hi --> V < Hi
70 // V < Min || V >= Hi --> V >= Hi
72 if (isSigned ? Lo.isMinSignedValue() : Lo.isMinValue()) {
73 Pred = isSigned ? ICmpInst::getSignedPredicate(Pred) : Pred;
74 return Builder.CreateICmp(Pred, V, ConstantInt::get(Ty, Hi));
75 }
76
77 // V >= Lo && V < Hi --> V - Lo u< Hi - Lo
78 // V < Lo || V >= Hi --> V - Lo u>= Hi - Lo
79 Value *VMinusLo =
80 Builder.CreateSub(V, ConstantInt::get(Ty, Lo), V->getName() + ".off");
81 Constant *HiMinusLo = ConstantInt::get(Ty, Hi - Lo);
82 return Builder.CreateICmp(Pred, VMinusLo, HiMinusLo);
83}
84
85/// Classify (icmp eq (A & B), C) and (icmp ne (A & B), C) as matching patterns
86/// that can be simplified.
87/// One of A and B is considered the mask. The other is the value. This is
88/// described as the "AMask" or "BMask" part of the enum. If the enum contains
89/// only "Mask", then both A and B can be considered masks. If A is the mask,
90/// then it was proven that (A & C) == C. This is trivial if C == A or C == 0.
91/// If both A and C are constants, this proof is also easy.
92/// For the following explanations, we assume that A is the mask.
93///
94/// "AllOnes" declares that the comparison is true only if (A & B) == A or all
95/// bits of A are set in B.
96/// Example: (icmp eq (A & 3), 3) -> AMask_AllOnes
97///
98/// "AllZeros" declares that the comparison is true only if (A & B) == 0 or all
99/// bits of A are cleared in B.
100/// Example: (icmp eq (A & 3), 0) -> Mask_AllZeroes
101///
102/// "Mixed" declares that (A & B) == C and C might or might not contain any
103/// number of one bits and zero bits.
104/// Example: (icmp eq (A & 3), 1) -> AMask_Mixed
105///
106/// "Not" means that in above descriptions "==" should be replaced by "!=".
107/// Example: (icmp ne (A & 3), 3) -> AMask_NotAllOnes
108///
109/// If the mask A contains a single bit, then the following is equivalent:
110/// (icmp eq (A & B), A) equals (icmp ne (A & B), 0)
111/// (icmp ne (A & B), A) equals (icmp eq (A & B), 0)
124
125/// Return the set of patterns (from MaskedICmpType) that (icmp SCC (A & B), C)
126/// satisfies.
127static unsigned getMaskedICmpType(Value *A, Value *B, Value *C,
129 const APInt *ConstA = nullptr, *ConstB = nullptr, *ConstC = nullptr;
130 match(A, m_APInt(ConstA));
131 match(B, m_APInt(ConstB));
132 match(C, m_APInt(ConstC));
133 bool IsEq = (Pred == ICmpInst::ICMP_EQ);
134 bool IsAPow2 = ConstA && ConstA->isPowerOf2();
135 bool IsBPow2 = ConstB && ConstB->isPowerOf2();
136 unsigned MaskVal = 0;
137 if (ConstC && ConstC->isZero()) {
138 // if C is zero, then both A and B qualify as mask
139 MaskVal |= (IsEq ? (Mask_AllZeros | AMask_Mixed | BMask_Mixed)
141 if (IsAPow2)
142 MaskVal |= (IsEq ? (AMask_NotAllOnes | AMask_NotMixed)
144 if (IsBPow2)
145 MaskVal |= (IsEq ? (BMask_NotAllOnes | BMask_NotMixed)
147 return MaskVal;
148 }
149
150 if (A == C) {
151 MaskVal |= (IsEq ? (AMask_AllOnes | AMask_Mixed)
153 if (IsAPow2)
154 MaskVal |= (IsEq ? (Mask_NotAllZeros | AMask_NotMixed)
156 } else if (ConstA && ConstC && ConstC->isSubsetOf(*ConstA)) {
157 MaskVal |= (IsEq ? AMask_Mixed : AMask_NotMixed);
158 }
159
160 if (B == C) {
161 MaskVal |= (IsEq ? (BMask_AllOnes | BMask_Mixed)
163 if (IsBPow2)
164 MaskVal |= (IsEq ? (Mask_NotAllZeros | BMask_NotMixed)
166 } else if (ConstB && ConstC && ConstC->isSubsetOf(*ConstB)) {
167 MaskVal |= (IsEq ? BMask_Mixed : BMask_NotMixed);
168 }
169
170 return MaskVal;
171}
172
173/// Convert an analysis of a masked ICmp into its equivalent if all boolean
174/// operations had the opposite sense. Since each "NotXXX" flag (recording !=)
175/// is adjacent to the corresponding normal flag (recording ==), this just
176/// involves swapping those bits over.
177static unsigned conjugateICmpMask(unsigned Mask) {
178 unsigned NewMask;
179 NewMask = (Mask & (AMask_AllOnes | BMask_AllOnes | Mask_AllZeros |
181 << 1;
182
183 NewMask |= (Mask & (AMask_NotAllOnes | BMask_NotAllOnes | Mask_NotAllZeros |
185 >> 1;
186
187 return NewMask;
188}
189
190// Adapts the external decomposeBitTest for local use.
192 Value *&Y, Value *&Z) {
193 auto Res =
194 llvm::decomposeBitTest(Cond, /*LookThroughTrunc=*/true,
195 /*AllowNonZeroC=*/true, /*DecomposeAnd=*/true);
196 if (!Res)
197 return false;
198
199 Pred = Res->Pred;
200 X = Res->X;
201 Y = ConstantInt::get(X->getType(), Res->Mask);
202 Z = ConstantInt::get(X->getType(), Res->C);
203 return true;
204}
205
206/// Handle (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E).
207/// Return the pattern classes (from MaskedICmpType) for the left hand side and
208/// the right hand side as a pair.
209/// LHS and RHS are the left hand side and the right hand side ICmps and PredL
210/// and PredR are their predicates, respectively.
211static std::optional<std::pair<unsigned, unsigned>>
214 ICmpInst::Predicate &PredR) {
215
216 // Here comes the tricky part:
217 // LHS might be of the form L11 & L12 == X, X == L21 & L22,
218 // and L11 & L12 == L21 & L22. The same goes for RHS.
219 // Now we must find those components L** and R**, that are equal, so
220 // that we can extract the parameters A, B, C, D, and E for the canonical
221 // above.
222
223 // Check whether the icmp can be decomposed into a bit test.
224 Value *L1, *L11, *L12, *L2, *L21, *L22;
225 if (decomposeBitTest(LHS, PredL, L11, L12, L2)) {
226 L21 = L22 = L1 = nullptr;
227 } else {
228 auto *LHSCMP = dyn_cast<ICmpInst>(LHS);
229 if (!LHSCMP)
230 return std::nullopt;
231
232 // Don't allow pointers. Splat vectors are fine.
233 if (!LHSCMP->getOperand(0)->getType()->isIntOrIntVectorTy())
234 return std::nullopt;
235
236 PredL = LHSCMP->getPredicate();
237 L1 = LHSCMP->getOperand(0);
238 L2 = LHSCMP->getOperand(1);
239 // Look for ANDs in the LHS icmp.
240 if (!match(L1, m_And(m_Value(L11), m_Value(L12)))) {
241 // Any icmp can be viewed as being trivially masked; if it allows us to
242 // remove one, it's worth it.
243 L11 = L1;
245 }
246
247 if (!match(L2, m_And(m_Value(L21), m_Value(L22)))) {
248 L21 = L2;
250 }
251 }
252
253 // Bail if LHS was a icmp that can't be decomposed into an equality.
254 if (!ICmpInst::isEquality(PredL))
255 return std::nullopt;
256
257 Value *R11, *R12, *R2;
258 if (decomposeBitTest(RHS, PredR, R11, R12, R2)) {
259 if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) {
260 A = R11;
261 D = R12;
262 } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) {
263 A = R12;
264 D = R11;
265 } else {
266 return std::nullopt;
267 }
268 E = R2;
269 } else {
270 auto *RHSCMP = dyn_cast<ICmpInst>(RHS);
271 if (!RHSCMP)
272 return std::nullopt;
273 // Don't allow pointers. Splat vectors are fine.
274 if (!RHSCMP->getOperand(0)->getType()->isIntOrIntVectorTy())
275 return std::nullopt;
276
277 PredR = RHSCMP->getPredicate();
278
279 Value *R1 = RHSCMP->getOperand(0);
280 R2 = RHSCMP->getOperand(1);
281 bool Ok = false;
282 if (!match(R1, m_And(m_Value(R11), m_Value(R12)))) {
283 // As before, model no mask as a trivial mask if it'll let us do an
284 // optimization.
285 R11 = R1;
287 }
288
289 if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) {
290 A = R11;
291 D = R12;
292 E = R2;
293 Ok = true;
294 } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) {
295 A = R12;
296 D = R11;
297 E = R2;
298 Ok = true;
299 }
300
301 // Avoid matching against the -1 value we created for unmasked operand.
302 if (Ok && match(A, m_AllOnes()))
303 Ok = false;
304
305 // Look for ANDs on the right side of the RHS icmp.
306 if (!Ok) {
307 if (!match(R2, m_And(m_Value(R11), m_Value(R12)))) {
308 R11 = R2;
309 R12 = Constant::getAllOnesValue(R2->getType());
310 }
311
312 if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) {
313 A = R11;
314 D = R12;
315 E = R1;
316 } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) {
317 A = R12;
318 D = R11;
319 E = R1;
320 } else {
321 return std::nullopt;
322 }
323 }
324 }
325
326 // Bail if RHS was a icmp that can't be decomposed into an equality.
327 if (!ICmpInst::isEquality(PredR))
328 return std::nullopt;
329
330 if (L11 == A) {
331 B = L12;
332 C = L2;
333 } else if (L12 == A) {
334 B = L11;
335 C = L2;
336 } else if (L21 == A) {
337 B = L22;
338 C = L1;
339 } else if (L22 == A) {
340 B = L21;
341 C = L1;
342 }
343
344 unsigned LeftType = getMaskedICmpType(A, B, C, PredL);
345 unsigned RightType = getMaskedICmpType(A, D, E, PredR);
346 return std::optional<std::pair<unsigned, unsigned>>(
347 std::make_pair(LeftType, RightType));
348}
349
350/// When foldLogOpOfMaskedICmps returns RHS for a logical and/or, we need to
351/// make sure that RHS is not more poisonous, e.g. as a result of poison flags
352/// on icmp or trunc, or via poison elements in vector constants. Using freeze
353/// is insufficient here, because we need the constants to match those the
354/// analysis was performed on.
355/// D and E are expected to be poison-free constants here.
356static Value *getReconstructedMaskedICmp(IRBuilderBase &Builder, bool IsLogical,
357 Value *RHS, Value *A, Value *D,
358 Value *E, ICmpInst::Predicate PredR) {
359 if (!IsLogical)
360 return RHS;
361
362 Value *And = Builder.CreateAnd(A, D);
363 return Builder.CreateICmp(PredR, And, E);
364}
365
366/// Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E) into a single
367/// (icmp(A & X) ==/!= Y), where the left-hand side is of type Mask_NotAllZeros
368/// and the right hand side is of type BMask_Mixed. For example,
369/// (icmp (A & 12) != 0) & (icmp (A & 15) == 8) -> (icmp (A & 15) == 8).
370/// Also used for logical and/or, must be poison safe.
372 Value *LHS, Value *RHS, bool IsAnd, bool IsLogical, Value *A, Value *B,
374 InstCombiner::BuilderTy &Builder) {
375 // We are given the canonical form:
376 // (icmp ne (A & B), 0) & (icmp eq (A & D), E).
377 // where D & E == E.
378 //
379 // If IsAnd is false, we get it in negated form:
380 // (icmp eq (A & B), 0) | (icmp ne (A & D), E) ->
381 // !((icmp ne (A & B), 0) & (icmp eq (A & D), E)).
382 //
383 // We currently handle the case of B, C, D, E are constant.
384 //
385 const APInt *BCst, *DCst, *OrigECst;
386 if (!match(B, m_APInt(BCst)) || !match(D, m_APInt(DCst)) ||
387 !match(E, m_APInt(OrigECst)))
388 return nullptr;
389
391
392 // Update E to the canonical form when D is a power of two and RHS is
393 // canonicalized as,
394 // (icmp ne (A & D), 0) -> (icmp eq (A & D), D) or
395 // (icmp ne (A & D), D) -> (icmp eq (A & D), 0).
396 APInt ECst = *OrigECst;
397 if (PredR != NewCC)
398 ECst ^= *DCst;
399
400 // If B or D is zero, skip because if LHS or RHS can be trivially folded by
401 // other folding rules and this pattern won't apply any more.
402 if (*BCst == 0 || *DCst == 0)
403 return nullptr;
404
405 // If B and D don't intersect, ie. (B & D) == 0, try to fold isNaN idiom:
406 // (icmp ne (A & FractionBits), 0) & (icmp eq (A & ExpBits), ExpBits)
407 // -> isNaN(A)
408 // Otherwise, we cannot deduce anything from it.
409 if (!BCst->intersects(*DCst)) {
410 Value *Src;
411 if (*DCst == ECst && match(A, m_ElementWiseBitCast(m_Value(Src))) &&
412 !Builder.GetInsertBlock()->getParent()->hasFnAttribute(
413 Attribute::StrictFP)) {
414 Type *Ty = Src->getType()->getScalarType();
415 if (!Ty->isIEEELikeFPTy())
416 return nullptr;
417
418 APInt ExpBits = APFloat::getInf(Ty->getFltSemantics()).bitcastToAPInt();
419 if (ECst != ExpBits)
420 return nullptr;
421 APInt FractionBits = ~ExpBits;
422 FractionBits.clearSignBit();
423 if (*BCst != FractionBits)
424 return nullptr;
425
426 return Builder.CreateFCmp(IsAnd ? FCmpInst::FCMP_UNO : FCmpInst::FCMP_ORD,
427 Src, ConstantFP::getZero(Src->getType()));
428 }
429 return nullptr;
430 }
431
432 // If the following two conditions are met:
433 //
434 // 1. mask B covers only a single bit that's not covered by mask D, that is,
435 // (B & (B ^ D)) is a power of 2 (in other words, B minus the intersection of
436 // B and D has only one bit set) and,
437 //
438 // 2. RHS (and E) indicates that the rest of B's bits are zero (in other
439 // words, the intersection of B and D is zero), that is, ((B & D) & E) == 0
440 //
441 // then that single bit in B must be one and thus the whole expression can be
442 // folded to
443 // (A & (B | D)) == (B & (B ^ D)) | E.
444 //
445 // For example,
446 // (icmp ne (A & 12), 0) & (icmp eq (A & 7), 1) -> (icmp eq (A & 15), 9)
447 // (icmp ne (A & 15), 0) & (icmp eq (A & 7), 0) -> (icmp eq (A & 15), 8)
448 if ((((*BCst & *DCst) & ECst) == 0) &&
449 (*BCst & (*BCst ^ *DCst)).isPowerOf2()) {
450 APInt BorD = *BCst | *DCst;
451 APInt BandBxorDorE = (*BCst & (*BCst ^ *DCst)) | ECst;
452 Value *NewMask = ConstantInt::get(A->getType(), BorD);
453 Value *NewMaskedValue = ConstantInt::get(A->getType(), BandBxorDorE);
454 Value *NewAnd = Builder.CreateAnd(A, NewMask);
455 return Builder.CreateICmp(NewCC, NewAnd, NewMaskedValue);
456 }
457
458 auto IsSubSetOrEqual = [](const APInt *C1, const APInt *C2) {
459 return (*C1 & *C2) == *C1;
460 };
461 auto IsSuperSetOrEqual = [](const APInt *C1, const APInt *C2) {
462 return (*C1 & *C2) == *C2;
463 };
464
465 // In the following, we consider only the cases where B is a superset of D, B
466 // is a subset of D, or B == D because otherwise there's at least one bit
467 // covered by B but not D, in which case we can't deduce much from it, so
468 // no folding (aside from the single must-be-one bit case right above.)
469 // For example,
470 // (icmp ne (A & 14), 0) & (icmp eq (A & 3), 1) -> no folding.
471 if (!IsSubSetOrEqual(BCst, DCst) && !IsSuperSetOrEqual(BCst, DCst))
472 return nullptr;
473
474 // At this point, either B is a superset of D, B is a subset of D or B == D.
475
476 // If E is zero, if B is a subset of (or equal to) D, LHS and RHS contradict
477 // and the whole expression becomes false (or true if negated), otherwise, no
478 // folding.
479 // For example,
480 // (icmp ne (A & 3), 0) & (icmp eq (A & 7), 0) -> false.
481 // (icmp ne (A & 15), 0) & (icmp eq (A & 3), 0) -> no folding.
482 if (ECst.isZero()) {
483 if (IsSubSetOrEqual(BCst, DCst))
484 return ConstantInt::get(LHS->getType(), !IsAnd);
485 return nullptr;
486 }
487
488 // At this point, B, D, E aren't zero and (B & D) == B, (B & D) == D or B ==
489 // D. If B is a superset of (or equal to) D, since E is not zero, LHS is
490 // subsumed by RHS (RHS implies LHS.) So the whole expression becomes
491 // RHS. For example,
492 // (icmp ne (A & 255), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8).
493 // (icmp ne (A & 15), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8).
494 if (IsSuperSetOrEqual(BCst, DCst))
495 return getReconstructedMaskedICmp(Builder, IsLogical, RHS, A, D, E, PredR);
496
497 // Otherwise, B is a subset of D. If B and E have a common bit set,
498 // ie. (B & E) != 0, then LHS is subsumed by RHS. For example.
499 // (icmp ne (A & 12), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8).
500 assert(IsSubSetOrEqual(BCst, DCst) && "Precondition due to above code");
501 if ((*BCst & ECst) != 0)
502 return getReconstructedMaskedICmp(Builder, IsLogical, RHS, A, D, E, PredR);
503
504 // Otherwise, LHS and RHS contradict and the whole expression becomes false
505 // (or true if negated.) For example,
506 // (icmp ne (A & 7), 0) & (icmp eq (A & 15), 8) -> false.
507 // (icmp ne (A & 6), 0) & (icmp eq (A & 15), 8) -> false.
508 return ConstantInt::get(LHS->getType(), !IsAnd);
509}
510
511/// Try to fold (icmp(A & B) ==/!= 0) &/| (icmp(A & D) ==/!= E) into a single
512/// (icmp(A & X) ==/!= Y), where the left-hand side and the right hand side
513/// aren't of the common mask pattern type.
514/// Also used for logical and/or, must be poison safe.
516 Value *LHS, Value *RHS, bool IsAnd, bool IsLogical, Value *A, Value *B,
518 ICmpInst::Predicate PredR, unsigned LHSMask, unsigned RHSMask,
519 InstCombiner::BuilderTy &Builder) {
521 "Expected equality predicates for masked type of icmps.");
522 // Handle Mask_NotAllZeros-BMask_Mixed cases.
523 // (icmp ne/eq (A & B), C) &/| (icmp eq/ne (A & D), E), or
524 // (icmp eq/ne (A & B), C) &/| (icmp ne/eq (A & D), E)
525 // which gets swapped to
526 // (icmp ne/eq (A & D), E) &/| (icmp eq/ne (A & B), C).
527 if (!IsAnd) {
528 LHSMask = conjugateICmpMask(LHSMask);
529 RHSMask = conjugateICmpMask(RHSMask);
530 }
531 if ((LHSMask & Mask_NotAllZeros) && (RHSMask & BMask_Mixed)) {
533 LHS, RHS, IsAnd, IsLogical, A, B, D, E, PredL, PredR, Builder)) {
534 return V;
535 }
536 } else if ((LHSMask & BMask_Mixed) && (RHSMask & Mask_NotAllZeros)) {
538 RHS, LHS, IsAnd, /*IsLogical=*/false, A, D, B, C, PredR, PredL,
539 Builder)) {
540 return V;
541 }
542 }
543 return nullptr;
544}
545
546/// Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E)
547/// into a single (icmp(A & X) ==/!= Y).
549 bool IsLogical,
551 const SimplifyQuery &Q) {
552 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr, *E = nullptr;
553 ICmpInst::Predicate PredL, PredR;
554 std::optional<std::pair<unsigned, unsigned>> MaskPair =
555 getMaskedTypeForICmpPair(A, B, C, D, E, LHS, RHS, PredL, PredR);
556 if (!MaskPair)
557 return nullptr;
559 "Expected equality predicates for masked type of icmps.");
560 unsigned LHSMask = MaskPair->first;
561 unsigned RHSMask = MaskPair->second;
562 unsigned Mask = LHSMask & RHSMask;
563 if (Mask == 0) {
564 // Even if the two sides don't share a common pattern, check if folding can
565 // still happen.
566 if (Value *V = foldLogOpOfMaskedICmpsAsymmetric(LHS, RHS, IsAnd, IsLogical,
567 A, B, C, D, E, PredL, PredR,
568 LHSMask, RHSMask, Builder))
569 return V;
570 return nullptr;
571 }
572
573 // In full generality:
574 // (icmp (A & B) Op C) | (icmp (A & D) Op E)
575 // == ![ (icmp (A & B) !Op C) & (icmp (A & D) !Op E) ]
576 //
577 // If the latter can be converted into (icmp (A & X) Op Y) then the former is
578 // equivalent to (icmp (A & X) !Op Y).
579 //
580 // Therefore, we can pretend for the rest of this function that we're dealing
581 // with the conjunction, provided we flip the sense of any comparisons (both
582 // input and output).
583
584 // In most cases we're going to produce an EQ for the "&&" case.
586 if (!IsAnd) {
587 // Convert the masking analysis into its equivalent with negated
588 // comparisons.
589 Mask = conjugateICmpMask(Mask);
590 }
591
592 if (Mask & Mask_AllZeros) {
593 // (icmp eq (A & B), 0) & (icmp eq (A & D), 0)
594 // -> (icmp eq (A & (B|D)), 0)
595 if (IsLogical && !isGuaranteedNotToBeUndefOrPoison(D))
596 return nullptr; // TODO: Use freeze?
597 Value *NewOr = Builder.CreateOr(B, D);
598 Value *NewAnd = Builder.CreateAnd(A, NewOr);
599 // We can't use C as zero because we might actually handle
600 // (icmp ne (A & B), B) & (icmp ne (A & D), D)
601 // with B and D, having a single bit set.
602 Value *Zero = Constant::getNullValue(A->getType());
603 return Builder.CreateICmp(NewCC, NewAnd, Zero);
604 }
605 if (Mask & BMask_AllOnes) {
606 // (icmp eq (A & B), B) & (icmp eq (A & D), D)
607 // -> (icmp eq (A & (B|D)), (B|D))
608 if (IsLogical && !isGuaranteedNotToBeUndefOrPoison(D))
609 return nullptr; // TODO: Use freeze?
610 Value *NewOr = Builder.CreateOr(B, D);
611 Value *NewAnd = Builder.CreateAnd(A, NewOr);
612 return Builder.CreateICmp(NewCC, NewAnd, NewOr);
613 }
614 if (Mask & AMask_AllOnes) {
615 // (icmp eq (A & B), A) & (icmp eq (A & D), A)
616 // -> (icmp eq (A & (B&D)), A)
617 if (IsLogical && !isGuaranteedNotToBeUndefOrPoison(D))
618 return nullptr; // TODO: Use freeze?
619 Value *NewAnd1 = Builder.CreateAnd(B, D);
620 Value *NewAnd2 = Builder.CreateAnd(A, NewAnd1);
621 return Builder.CreateICmp(NewCC, NewAnd2, A);
622 }
623
624 const APInt *ConstB, *ConstD;
625 if (match(B, m_APInt(ConstB)) && match(D, m_APInt(ConstD))) {
626 if (Mask & (Mask_NotAllZeros | BMask_NotAllOnes)) {
627 // (icmp ne (A & B), 0) & (icmp ne (A & D), 0) and
628 // (icmp ne (A & B), B) & (icmp ne (A & D), D)
629 // -> (icmp ne (A & B), 0) or (icmp ne (A & D), 0)
630 // Only valid if one of the masks is a superset of the other (check "B&D"
631 // is the same as either B or D).
632 APInt NewMask = *ConstB & *ConstD;
633 if (NewMask == *ConstB)
634 return LHS;
635 if (NewMask == *ConstD)
636 return getReconstructedMaskedICmp(Builder, IsLogical, RHS, A, D, E,
637 PredR);
638 }
639
640 if (Mask & AMask_NotAllOnes) {
641 // (icmp ne (A & B), B) & (icmp ne (A & D), D)
642 // -> (icmp ne (A & B), A) or (icmp ne (A & D), A)
643 // Only valid if one of the masks is a superset of the other (check "B|D"
644 // is the same as either B or D).
645 APInt NewMask = *ConstB | *ConstD;
646 if (NewMask == *ConstB)
647 return LHS;
648 if (NewMask == *ConstD)
649 return RHS;
650 }
651
652 if (Mask & (BMask_Mixed | BMask_NotMixed)) {
653 // Mixed:
654 // (icmp eq (A & B), C) & (icmp eq (A & D), E)
655 // We already know that B & C == C && D & E == E.
656 // If we can prove that (B & D) & (C ^ E) == 0, that is, the bits of
657 // C and E, which are shared by both the mask B and the mask D, don't
658 // contradict, then we can transform to
659 // -> (icmp eq (A & (B|D)), (C|E))
660 // Currently, we only handle the case of B, C, D, and E being constant.
661 // We can't simply use C and E because we might actually handle
662 // (icmp ne (A & B), B) & (icmp eq (A & D), D)
663 // with B and D, having a single bit set.
664
665 // NotMixed:
666 // (icmp ne (A & B), C) & (icmp ne (A & D), E)
667 // -> (icmp ne (A & (B & D)), (C & E))
668 // Check the intersection (B & D) for inequality.
669 // Assume that (B & D) == B || (B & D) == D, i.e B/D is a subset of D/B
670 // and (B & D) & (C ^ E) == 0, bits of C and E, which are shared by both
671 // the B and the D, don't contradict. Note that we can assume (~B & C) ==
672 // 0 && (~D & E) == 0, previous operation should delete these icmps if it
673 // hadn't been met.
674
675 const APInt *OldConstC, *OldConstE;
676 if (!match(C, m_APInt(OldConstC)) || !match(E, m_APInt(OldConstE)))
677 return nullptr;
678
679 auto FoldBMixed = [&](ICmpInst::Predicate CC, bool IsNot) -> Value * {
681 const APInt ConstC = PredL != CC ? *ConstB ^ *OldConstC : *OldConstC;
682 const APInt ConstE = PredR != CC ? *ConstD ^ *OldConstE : *OldConstE;
683
684 if (((*ConstB & *ConstD) & (ConstC ^ ConstE)).getBoolValue())
685 return IsNot ? nullptr : ConstantInt::get(LHS->getType(), !IsAnd);
686
687 if (IsNot && !ConstB->isSubsetOf(*ConstD) &&
688 !ConstD->isSubsetOf(*ConstB))
689 return nullptr;
690
691 APInt BD, CE;
692 if (IsNot) {
693 BD = *ConstB & *ConstD;
694 CE = ConstC & ConstE;
695 } else {
696 BD = *ConstB | *ConstD;
697 CE = ConstC | ConstE;
698 }
699 Value *NewAnd = Builder.CreateAnd(A, BD);
700 Value *CEVal = ConstantInt::get(A->getType(), CE);
701 return Builder.CreateICmp(CC, NewAnd, CEVal);
702 };
703
704 if (Mask & BMask_Mixed)
705 return FoldBMixed(NewCC, false);
706 if (Mask & BMask_NotMixed) // can be else also
707 return FoldBMixed(NewCC, true);
708 }
709 }
710
711 // (icmp eq (A & B), 0) | (icmp eq (A & D), 0)
712 // -> (icmp ne (A & (B|D)), (B|D))
713 // (icmp ne (A & B), 0) & (icmp ne (A & D), 0)
714 // -> (icmp eq (A & (B|D)), (B|D))
715 // iff B and D is known to be a power of two
716 if (Mask & Mask_NotAllZeros &&
717 isKnownToBeAPowerOfTwo(B, /*OrZero=*/false, Q) &&
718 isKnownToBeAPowerOfTwo(D, /*OrZero=*/false, Q)) {
719 // If this is a logical and/or, then we must prevent propagation of a
720 // poison value from the RHS by inserting freeze.
721 if (IsLogical)
722 D = Builder.CreateFreeze(D);
723 Value *Mask = Builder.CreateOr(B, D);
724 Value *Masked = Builder.CreateAnd(A, Mask);
725 return Builder.CreateICmp(NewCC, Masked, Mask);
726 }
727 return nullptr;
728}
729
730/// Try to fold a signed range checked with lower bound 0 to an unsigned icmp.
731/// Example: (icmp sge x, 0) & (icmp slt x, n) --> icmp ult x, n
732/// If \p Inverted is true then the check is for the inverted range, e.g.
733/// (icmp slt x, 0) | (icmp sgt x, n) --> icmp ugt x, n
735 Value *LHS1, CmpPredicate PredR,
736 Value *RHS0, Value *RHS1,
737 Instruction *CtxI, bool Inverted) {
738 // Check the lower range comparison, e.g. x >= 0
739 // InstCombine already ensured that if there is a constant it's on the RHS.
740 ConstantInt *RangeStart = dyn_cast<ConstantInt>(LHS1);
741 if (!RangeStart)
742 return nullptr;
743
744 if (Inverted) {
745 PredL = CmpPredicate::getInverse(PredL);
746 PredR = CmpPredicate::getInverse(PredR);
747 }
748
749 // Accept x > -1 or x >= 0 (after potentially inverting the predicate).
750 if (!((PredL == ICmpInst::ICMP_SGT && RangeStart->isMinusOne()) ||
751 (PredL == ICmpInst::ICMP_SGE && RangeStart->isZero())))
752 return nullptr;
753
754 Value *Input = LHS0;
755 Value *RangeEnd;
756 if (match(RHS0, m_SExtOrSelf(m_Specific(Input)))) {
757 // For the upper range compare we have: icmp x, n
758 Input = RHS0;
759 RangeEnd = RHS1;
760 } else if (match(RHS1, m_SExtOrSelf(m_Specific(Input)))) {
761 // For the upper range compare we have: icmp n, x
762 Input = RHS1;
763 RangeEnd = RHS0;
764 PredR = CmpPredicate::getSwapped(PredR);
765 } else {
766 return nullptr;
767 }
768
769 // Check the upper range comparison, e.g. x < n
770 ICmpInst::Predicate NewPred;
771 switch (PredR) {
773 NewPred = ICmpInst::ICMP_ULT;
774 break;
776 NewPred = ICmpInst::ICMP_ULE;
777 break;
778 default:
779 return nullptr;
780 }
781
782 // This simplification is only valid if the upper range is not negative.
783 KnownBits Known = computeKnownBits(RangeEnd, CtxI);
784 if (!Known.isNonNegative())
785 return nullptr;
786
787 if (Inverted)
788 NewPred = ICmpInst::getInversePredicate(NewPred);
789
790 return Builder.CreateICmp(NewPred, Input, RangeEnd);
791}
792
793// (or (icmp eq X, 0), (icmp eq X, Pow2OrZero))
794// -> (icmp eq (and X, Pow2OrZero), X)
795// (and (icmp ne X, 0), (icmp ne X, Pow2OrZero))
796// -> (icmp ne (and X, Pow2OrZero), X)
798 InstCombiner::BuilderTy &Builder, CmpPredicate PredL, Value *LHS0,
799 Value *LHS1, bool LHSOneUse, CmpPredicate PredR, Value *RHS0, Value *RHS1,
800 bool RHSOneUse, bool IsAnd, const SimplifyQuery &Q) {
802 // Make sure we have right compares for our op.
803 if (PredL != Pred || PredR != Pred)
804 return nullptr;
805
806 // Make it so we can match LHS against the (icmp eq/ne X, 0) just for
807 // simplicity.
808 if (match(RHS1, m_Zero())) {
809 std::swap(PredL, PredR);
810 std::swap(LHS0, RHS0);
811 std::swap(LHS1, RHS1);
812 }
813
814 if (RHS1 == LHS0)
815 std::swap(RHS0, RHS1);
816
817 // Match the desired pattern:
818 // LHS: (icmp eq/ne X, 0)
819 // RHS: (icmp eq/ne X, Pow2OrZero)
820 // Skip if Pow2OrZero is 1. Either way it gets folded to (icmp ugt X, 1) but
821 // this form ends up slightly less canonical.
822 // We could potentially be more sophisticated than requiring LHS/RHS
823 // be one-use. We don't create additional instructions if only one
824 // of them is one-use. So cases where one is one-use and the other
825 // is two-use might be profitable.
826 if (!LHSOneUse || !RHSOneUse || !match(LHS1, m_Zero()) || RHS0 != LHS0 ||
827 match(RHS1, m_One()) || !isKnownToBeAPowerOfTwo(RHS1, /*OrZero=*/true, Q))
828 return nullptr;
829
830 Value *And = Builder.CreateAnd(LHS0, RHS1);
831 return Builder.CreateICmp(Pred, And, LHS0);
832}
833
834/// General pattern:
835/// X & Y
836///
837/// Where Y is checking that all the high bits (covered by a mask 4294967168)
838/// are uniform, i.e. %arg & 4294967168 can be either 4294967168 or 0
839/// Pattern can be one of:
840/// %t = add i32 %arg, 128
841/// %r = icmp ult i32 %t, 256
842/// Or
843/// %t0 = shl i32 %arg, 24
844/// %t1 = ashr i32 %t0, 24
845/// %r = icmp eq i32 %t1, %arg
846/// Or
847/// %t0 = trunc i32 %arg to i8
848/// %t1 = sext i8 %t0 to i32
849/// %r = icmp eq i32 %t1, %arg
850/// This pattern is a signed truncation check.
851///
852/// And X is checking that some bit in that same mask is zero.
853/// I.e. can be one of:
854/// %r = icmp sgt i32 %arg, -1
855/// Or
856/// %t = and i32 %arg, 2147483648
857/// %r = icmp eq i32 %t, 0
858///
859/// Since we are checking that all the bits in that mask are the same,
860/// and a particular bit is zero, what we are really checking is that all the
861/// masked bits are zero.
862/// So this should be transformed to:
863/// %r = icmp ult i32 %arg, 128
865 Value *LHS1, CmpPredicate PredR,
866 Value *RHS0, Value *RHS1,
867 Instruction &CtxI,
868 InstCombiner::BuilderTy &Builder) {
869 assert(CtxI.getOpcode() == Instruction::And);
870
871 // Match icmp ult (add %arg, C01), C1 (C1 == C01 << 1; powers of two)
872 auto tryToMatchSignedTruncationCheck = [](CmpPredicate Pred, Value *LHS,
873 Value *RHS, Value *&X,
874 APInt &SignBitMask) -> bool {
875 const APInt *I01, *I1; // powers of two; I1 == I01 << 1
876 if (Pred != ICmpInst::ICMP_ULT ||
877 !match(LHS, m_Add(m_Value(X), m_Power2(I01))) ||
878 !match(RHS, m_Power2(I1)) || I1->ule(*I01) || I01->shl(1) != *I1)
879 return false;
880 // Which bit is the new sign bit as per the 'signed truncation' pattern?
881 SignBitMask = *I01;
882 return true;
883 };
884
885 // One icmp needs to be 'signed truncation check'.
886 // We need to match this first, else we will mismatch commutative cases.
887 Value *X1;
888 APInt HighestBit;
889 if (tryToMatchSignedTruncationCheck(PredR, RHS0, RHS1, X1, HighestBit)) {
890 std::swap(PredL, PredR);
891 std::swap(LHS0, RHS0);
892 std::swap(LHS1, RHS1);
893 } else if (!tryToMatchSignedTruncationCheck(PredL, LHS0, LHS1, X1,
894 HighestBit))
895 return nullptr;
896
897 assert(HighestBit.isPowerOf2() && "expected to be power of two (non-zero)");
898
899 // Try to match/decompose into: icmp eq (X & Mask), 0
900 auto tryToDecompose = [](CmpPredicate Pred, Value *LHS, Value *RHS, Value *&X,
901 APInt &UnsetBitsMask) -> bool {
902 // Can it be decomposed into icmp eq (X & Mask), 0 ?
904 /*LookThroughTrunc=*/false,
905 /*AllowNonZeroC=*/false,
906 /*DecomposeAnd=*/true);
907 if (Res && Res->Pred == ICmpInst::ICMP_EQ) {
908 X = Res->X;
909 UnsetBitsMask = Res->Mask;
910 return true;
911 }
912
913 return false;
914 };
915
916 // And the other icmp needs to be decomposable into a bit test.
917 Value *X0;
918 APInt UnsetBitsMask;
919 if (!tryToDecompose(PredR, RHS0, RHS1, X0, UnsetBitsMask))
920 return nullptr;
921
922 assert(!UnsetBitsMask.isZero() && "empty mask makes no sense.");
923
924 // Are they working on the same value?
925 Value *X;
926 if (X1 == X0) {
927 // Ok as is.
928 X = X1;
929 } else if (match(X0, m_Trunc(m_Specific(X1)))) {
930 UnsetBitsMask = UnsetBitsMask.zext(X1->getType()->getScalarSizeInBits());
931 X = X1;
932 } else
933 return nullptr;
934
935 // So which bits should be uniform as per the 'signed truncation check'?
936 // (all the bits starting with (i.e. including) HighestBit)
937 APInt SignBitsMask = ~(HighestBit - 1U);
938
939 // UnsetBitsMask must have some common bits with SignBitsMask,
940 if (!UnsetBitsMask.intersects(SignBitsMask))
941 return nullptr;
942
943 // Does UnsetBitsMask contain any bits outside of SignBitsMask?
944 if (!UnsetBitsMask.isSubsetOf(SignBitsMask)) {
945 APInt OtherHighestBit = (~UnsetBitsMask) + 1U;
946 if (!OtherHighestBit.isPowerOf2())
947 return nullptr;
948 HighestBit = APIntOps::umin(HighestBit, OtherHighestBit);
949 }
950 // Else, if it does not, then all is ok as-is.
951
952 // %r = icmp ult %X, SignBit
953 return Builder.CreateICmpULT(X, ConstantInt::get(X->getType(), HighestBit),
954 CtxI.getName() + ".simplified");
955}
956
957/// Fold (icmp eq ctpop(X) 1) | (icmp eq X 0) into (icmp ult ctpop(X) 2) and
958/// fold (icmp ne ctpop(X) 1) & (icmp ne X 0) into (icmp ugt ctpop(X) 1).
959/// Also used for logical and/or, must be poison safe if range attributes are
960/// dropped.
962 CmpPredicate PredR, Value *RHS0, Value *RHS1,
963 bool IsAnd, InstCombiner::BuilderTy &Builder,
964 InstCombinerImpl &IC) {
965
966 Value *X;
967 if (!match(LHS0, m_Ctpop(m_Value(X))) || !match(LHS1, m_SpecificInt(1)) ||
968 RHS0 != X || !match(RHS1, m_ZeroInt()))
969 return nullptr;
970
971 auto *CtPop = cast<Instruction>(LHS0);
972 if (IsAnd && PredL == ICmpInst::ICMP_NE && PredR == ICmpInst::ICMP_NE) {
973 // Drop range attributes and re-infer them in the next iteration.
974 CtPop->dropPoisonGeneratingAnnotations();
975 IC.addToWorklist(CtPop);
976 return Builder.CreateICmpUGT(CtPop, ConstantInt::get(CtPop->getType(), 1));
977 }
978 if (!IsAnd && PredL == ICmpInst::ICMP_EQ && PredR == ICmpInst::ICMP_EQ) {
979 // Drop range attributes and re-infer them in the next iteration.
980 CtPop->dropPoisonGeneratingAnnotations();
981 IC.addToWorklist(CtPop);
982 return Builder.CreateICmpULT(CtPop, ConstantInt::get(CtPop->getType(), 2));
983 }
984
985 return nullptr;
986}
987
988/// Reduce a pair of compares that check if a value has exactly 1 bit set.
989/// Also used for logical and/or, must be poison safe if range attributes are
990/// dropped.
991static Value *foldIsPowerOf2(CmpPredicate PredL, Value *LHS0, Value *LHS1,
992 CmpPredicate PredR, Value *RHS0, Value *RHS1,
993 bool JoinedByAnd, InstCombiner::BuilderTy &Builder,
994 InstCombinerImpl &IC) {
995 // Handle 'and' / 'or' commutation: make the equality check the first operand.
996 if (PredR == (JoinedByAnd ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ)) {
997 std::swap(PredL, PredR);
998 std::swap(LHS0, RHS0);
999 std::swap(LHS1, RHS1);
1000 }
1001
1002 // (X != 0) && (ctpop(X) u< 2) --> ctpop(X) == 1
1003 if (JoinedByAnd && PredL == ICmpInst::ICMP_NE && match(LHS1, m_ZeroInt()) &&
1004 PredR == ICmpInst::ICMP_ULT && match(RHS0, m_Ctpop(m_Specific(LHS0))) &&
1005 match(RHS1, m_SpecificInt(2))) {
1006 auto *CtPop = cast<Instruction>(RHS0);
1007 // Drop range attributes and re-infer them in the next iteration.
1008 CtPop->dropPoisonGeneratingAnnotations();
1009 IC.addToWorklist(CtPop);
1010 return Builder.CreateICmpEQ(CtPop, ConstantInt::get(CtPop->getType(), 1));
1011 }
1012 // (X == 0) || (ctpop(X) u> 1) --> ctpop(X) != 1
1013 if (!JoinedByAnd && PredL == ICmpInst::ICMP_EQ && match(LHS1, m_ZeroInt()) &&
1014 PredR == ICmpInst::ICMP_UGT && match(RHS0, m_Ctpop(m_Specific(LHS0))) &&
1015 match(RHS1, m_SpecificInt(1))) {
1016 auto *CtPop = cast<Instruction>(RHS0);
1017 // Drop range attributes and re-infer them in the next iteration.
1018 CtPop->dropPoisonGeneratingAnnotations();
1019 IC.addToWorklist(CtPop);
1020 return Builder.CreateICmpNE(CtPop, ConstantInt::get(CtPop->getType(), 1));
1021 }
1022 return nullptr;
1023}
1024
1025/// Try to fold (icmp(A & B) == 0) & (icmp(A & D) != E) into (icmp A u< D) iff
1026/// B is a contiguous set of ones starting from the most significant bit
1027/// (negative power of 2), D and E are equal, and D is a contiguous set of ones
1028/// starting at the most significant zero bit in B. Parameter B supports masking
1029/// using undef/poison in either scalar or vector values.
1031 Value *A, Value *B, Value *D, Value *E, ICmpInst::Predicate PredL,
1034 "Expected equality predicates for masked type of icmps.");
1035 if (PredL != ICmpInst::ICMP_EQ || PredR != ICmpInst::ICMP_NE)
1036 return nullptr;
1037
1038 if (!match(B, m_NegatedPower2()) || !match(D, m_ShiftedMask()) ||
1039 !match(E, m_ShiftedMask()))
1040 return nullptr;
1041
1042 // Test scalar arguments for conversion. B has been validated earlier to be a
1043 // negative power of two and thus is guaranteed to have one or more contiguous
1044 // ones starting from the MSB followed by zero or more contiguous zeros. D has
1045 // been validated earlier to be a shifted set of one or more contiguous ones.
1046 // In order to match, B leading ones and D leading zeros should be equal. The
1047 // predicate that B be a negative power of 2 prevents the condition of there
1048 // ever being zero leading ones. Thus 0 == 0 cannot occur. The predicate that
1049 // D always be a shifted mask prevents the condition of D equaling 0. This
1050 // prevents matching the condition where B contains the maximum number of
1051 // leading one bits (-1) and D contains the maximum number of leading zero
1052 // bits (0).
1053 auto isReducible = [](const Value *B, const Value *D, const Value *E) {
1054 const APInt *BCst, *DCst, *ECst;
1055 return match(B, m_APIntAllowPoison(BCst)) && match(D, m_APInt(DCst)) &&
1056 match(E, m_APInt(ECst)) && *DCst == *ECst &&
1057 (isa<PoisonValue>(B) ||
1058 (BCst->countLeadingOnes() == DCst->countLeadingZeros()));
1059 };
1060
1061 // Test vector type arguments for conversion.
1062 if (const auto *BVTy = dyn_cast<VectorType>(B->getType())) {
1063 const auto *BFVTy = dyn_cast<FixedVectorType>(BVTy);
1064 const auto *BConst = dyn_cast<Constant>(B);
1065 const auto *DConst = dyn_cast<Constant>(D);
1066 const auto *EConst = dyn_cast<Constant>(E);
1067
1068 if (!BFVTy || !BConst || !DConst || !EConst)
1069 return nullptr;
1070
1071 for (unsigned I = 0; I != BFVTy->getNumElements(); ++I) {
1072 const auto *BElt = BConst->getAggregateElement(I);
1073 const auto *DElt = DConst->getAggregateElement(I);
1074 const auto *EElt = EConst->getAggregateElement(I);
1075
1076 if (!BElt || !DElt || !EElt)
1077 return nullptr;
1078 if (!isReducible(BElt, DElt, EElt))
1079 return nullptr;
1080 }
1081 } else {
1082 // Test scalar type arguments for conversion.
1083 if (!isReducible(B, D, E))
1084 return nullptr;
1085 }
1086 return Builder.CreateICmp(ICmpInst::ICMP_ULT, A, D);
1087}
1088
1089/// Try to fold ((icmp X u< P) & (icmp(X & M) != M)) or ((icmp X s> -1) &
1090/// (icmp(X & M) != M)) into (icmp X u< M). Where P is a power of 2, M < P, and
1091/// M is a contiguous shifted mask starting at the right most significant zero
1092/// bit in P. SGT is supported as when P is the largest representable power of
1093/// 2, an earlier optimization converts the expression into (icmp X s> -1).
1094/// Parameter P supports masking using undef/poison in either scalar or vector
1095/// values.
1097 bool JoinedByAnd,
1098 InstCombiner::BuilderTy &Builder) {
1099 if (!JoinedByAnd)
1100 return nullptr;
1101 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr, *E = nullptr;
1102 ICmpInst::Predicate CmpPred0, CmpPred1;
1103 // Assuming P is a 2^n, getMaskedTypeForICmpPair will normalize (icmp X u<
1104 // 2^n) into (icmp (X & ~(2^n-1)) == 0) and (icmp X s> -1) into (icmp (X &
1105 // SignMask) == 0).
1106 std::optional<std::pair<unsigned, unsigned>> MaskPair =
1107 getMaskedTypeForICmpPair(A, B, C, D, E, Cmp0, Cmp1, CmpPred0, CmpPred1);
1108 if (!MaskPair)
1109 return nullptr;
1110
1111 const auto compareBMask = BMask_NotMixed | BMask_NotAllOnes;
1112 unsigned CmpMask0 = MaskPair->first;
1113 unsigned CmpMask1 = MaskPair->second;
1114 if ((CmpMask0 & Mask_AllZeros) && (CmpMask1 == compareBMask)) {
1115 if (Value *V = foldNegativePower2AndShiftedMask(A, B, D, E, CmpPred0,
1116 CmpPred1, Builder))
1117 return V;
1118 } else if ((CmpMask0 == compareBMask) && (CmpMask1 & Mask_AllZeros)) {
1119 if (Value *V = foldNegativePower2AndShiftedMask(A, D, B, C, CmpPred1,
1120 CmpPred0, Builder))
1121 return V;
1122 }
1123 return nullptr;
1124}
1125
1126/// Commuted variants are assumed to be handled by calling this function again
1127/// with the parameters swapped.
1129 Value *LHS1, bool LHSOneUse,
1130 CmpPredicate PredR, Value *RHS0,
1131 Value *RHS1, bool RHSOneUse,
1132 bool IsAnd, const SimplifyQuery &Q,
1133 InstCombiner::BuilderTy &Builder) {
1134 if (!match(LHS1, m_Zero()) || !ICmpInst::isEquality(PredL))
1135 return nullptr;
1136
1137 Value *A, *B;
1138 if (RHS0 == LHS0)
1139 A = RHS1;
1140 else if (RHS1 == LHS0) {
1141 A = RHS0;
1142 PredR = CmpPredicate::getSwapped(PredR);
1143 } else
1144 return nullptr;
1145
1146 if (!match(LHS0, m_c_Add(m_Specific(A), m_Value(B))) ||
1147 !(LHSOneUse || RHSOneUse))
1148 return nullptr;
1149
1150 auto GetKnownNonZeroAndOther = [&](Value *&NonZero, Value *&Other) {
1151 if (!isKnownNonZero(NonZero, Q))
1152 std::swap(NonZero, Other);
1153 return isKnownNonZero(NonZero, Q);
1154 };
1155
1156 // Given ZeroCmpOp = (A + B)
1157 // ZeroCmpOp < A && ZeroCmpOp != 0 --> (0-X) < Y iff
1158 // ZeroCmpOp >= A || ZeroCmpOp == 0 --> (0-X) >= Y iff
1159 // with X being the value (A/B) that is known to be non-zero,
1160 // and Y being remaining value.
1161 if (PredR == ICmpInst::ICMP_ULT && PredL == ICmpInst::ICMP_NE && IsAnd &&
1162 GetKnownNonZeroAndOther(B, A))
1163 return Builder.CreateICmpULT(Builder.CreateNeg(B), A);
1164 if (PredR == ICmpInst::ICMP_UGE && PredL == ICmpInst::ICMP_EQ && !IsAnd &&
1165 GetKnownNonZeroAndOther(B, A))
1166 return Builder.CreateICmpUGE(Builder.CreateNeg(B), A);
1167
1168 return nullptr;
1169}
1170
1171struct IntPart {
1173 unsigned StartBit;
1174 unsigned NumBits;
1175};
1176
1177/// Match an extraction of bits from an integer.
1178static std::optional<IntPart> matchIntPart(Value *V) {
1179 Value *X;
1180 if (!match(V, m_OneUse(m_Trunc(m_Value(X)))))
1181 return std::nullopt;
1182
1183 unsigned NumOriginalBits = X->getType()->getScalarSizeInBits();
1184 unsigned NumExtractedBits = V->getType()->getScalarSizeInBits();
1185 Value *Y;
1186 const APInt *Shift;
1187 // For a trunc(lshr Y, Shift) pattern, make sure we're only extracting bits
1188 // from Y, not any shifted-in zeroes.
1189 if (match(X, m_OneUse(m_LShr(m_Value(Y), m_APInt(Shift)))) &&
1190 Shift->ule(NumOriginalBits - NumExtractedBits))
1191 return {{Y, (unsigned)Shift->getZExtValue(), NumExtractedBits}};
1192 return {{X, 0, NumExtractedBits}};
1193}
1194
1195/// Materialize an extraction of bits from an integer in IR.
1196static Value *extractIntPart(const IntPart &P, IRBuilderBase &Builder) {
1197 Value *V = P.From;
1198 if (P.StartBit)
1199 V = Builder.CreateLShr(V, P.StartBit);
1200 Type *TruncTy = V->getType()->getWithNewBitWidth(P.NumBits);
1201 if (TruncTy != V->getType())
1202 V = Builder.CreateTrunc(V, TruncTy);
1203 return V;
1204}
1205
1206/// (icmp eq X0, Y0) & (icmp eq X1, Y1) -> icmp eq X01, Y01
1207/// (icmp ne X0, Y0) | (icmp ne X1, Y1) -> icmp ne X01, Y01
1208/// where X0, X1 and Y0, Y1 are adjacent parts extracted from an integer.
1209Value *InstCombinerImpl::foldEqOfParts(Value *Cmp0, Value *Cmp1, bool IsAnd) {
1210 if (!Cmp0->hasOneUse() || !Cmp1->hasOneUse())
1211 return nullptr;
1212
1214 auto GetMatchPart = [&](Value *CmpV,
1215 unsigned OpNo) -> std::optional<IntPart> {
1216 assert(CmpV->getType()->isIntOrIntVectorTy(1) && "Must be bool");
1217
1218 Value *X, *Y;
1219 // icmp ne (and x, 1), (and y, 1) <=> trunc (xor x, y) to i1
1220 // icmp eq (and x, 1), (and y, 1) <=> not (trunc (xor x, y) to i1)
1221 if (Pred == CmpInst::ICMP_NE
1222 ? match(CmpV, m_Trunc(m_Xor(m_Value(X), m_Value(Y))))
1223 : match(CmpV, m_Not(m_Trunc(m_Xor(m_Value(X), m_Value(Y))))))
1224 return {{OpNo == 0 ? X : Y, 0, 1}};
1225
1226 auto *Cmp = dyn_cast<ICmpInst>(CmpV);
1227 if (!Cmp)
1228 return std::nullopt;
1229
1230 if (Pred == Cmp->getPredicate())
1231 return matchIntPart(Cmp->getOperand(OpNo));
1232
1233 const APInt *C;
1234 // (icmp eq (lshr x, C), (lshr y, C)) gets optimized to:
1235 // (icmp ult (xor x, y), 1 << C) so also look for that.
1236 if (Pred == CmpInst::ICMP_EQ && Cmp->getPredicate() == CmpInst::ICMP_ULT) {
1237 if (!match(Cmp->getOperand(1), m_Power2(C)) ||
1238 !match(Cmp->getOperand(0), m_Xor(m_Value(), m_Value())))
1239 return std::nullopt;
1240 }
1241
1242 // (icmp ne (lshr x, C), (lshr y, C)) gets optimized to:
1243 // (icmp ugt (xor x, y), (1 << C) - 1) so also look for that.
1244 else if (Pred == CmpInst::ICMP_NE &&
1245 Cmp->getPredicate() == CmpInst::ICMP_UGT) {
1246 if (!match(Cmp->getOperand(1), m_LowBitMask(C)) ||
1247 !match(Cmp->getOperand(0), m_Xor(m_Value(), m_Value())))
1248 return std::nullopt;
1249 } else {
1250 return std::nullopt;
1251 }
1252
1253 unsigned From = Pred == CmpInst::ICMP_NE ? C->popcount() : C->countr_zero();
1254 Instruction *I = cast<Instruction>(Cmp->getOperand(0));
1255 return {{I->getOperand(OpNo), From, C->getBitWidth() - From}};
1256 };
1257
1258 std::optional<IntPart> L0 = GetMatchPart(Cmp0, 0);
1259 std::optional<IntPart> R0 = GetMatchPart(Cmp0, 1);
1260 std::optional<IntPart> L1 = GetMatchPart(Cmp1, 0);
1261 std::optional<IntPart> R1 = GetMatchPart(Cmp1, 1);
1262 if (!L0 || !R0 || !L1 || !R1)
1263 return nullptr;
1264
1265 // Make sure the LHS/RHS compare a part of the same value, possibly after
1266 // an operand swap.
1267 if (L0->From != L1->From || R0->From != R1->From) {
1268 if (L0->From != R1->From || R0->From != L1->From)
1269 return nullptr;
1270 std::swap(L1, R1);
1271 }
1272
1273 // Make sure the extracted parts are adjacent, canonicalizing to L0/R0 being
1274 // the low part and L1/R1 being the high part.
1275 if (L0->StartBit + L0->NumBits != L1->StartBit ||
1276 R0->StartBit + R0->NumBits != R1->StartBit) {
1277 if (L1->StartBit + L1->NumBits != L0->StartBit ||
1278 R1->StartBit + R1->NumBits != R0->StartBit)
1279 return nullptr;
1280 std::swap(L0, L1);
1281 std::swap(R0, R1);
1282 }
1283
1284 // We can simplify to a comparison of these larger parts of the integers.
1285 IntPart L = {L0->From, L0->StartBit, L0->NumBits + L1->NumBits};
1286 IntPart R = {R0->From, R0->StartBit, R0->NumBits + R1->NumBits};
1289 return Builder.CreateICmp(Pred, LValue, RValue);
1290}
1291
1292/// Reduce logic-of-compares with equality to a constant by substituting a
1293/// common operand with the constant. Callers are expected to call this with
1294/// Cmp0/Cmp1 switched to handle logic op commutativity.
1295static Value *
1297 Value *LHS, CmpPredicate PredR, Value *RHS0,
1298 Value *RHS1, bool RHSOneUse, bool IsAnd,
1299 bool IsLogical, InstCombiner::BuilderTy &Builder,
1300 const SimplifyQuery &Q, Instruction &I) {
1301 // Match an equality compare with a non-poison constant as Cmp0.
1302 // Also, give up if the compare can be constant-folded to avoid looping.
1303 if (!isa<Constant>(LHS1) || !isGuaranteedNotToBeUndefOrPoison(LHS1) ||
1304 isa<Constant>(LHS0))
1305 return nullptr;
1306 if ((IsAnd && PredL != ICmpInst::ICMP_EQ) ||
1307 (!IsAnd && PredL != ICmpInst::ICMP_NE))
1308 return nullptr;
1309
1310 // The other compare must include a common operand (X). Canonicalize the
1311 // common operand as operand 1 (Pred1 is swapped if the common operand was
1312 // operand 0).
1313 Value *Y;
1314
1315 if (LHS0 == RHS0) {
1316 Y = RHS1;
1317 PredR = CmpPredicate::getSwapped(PredR);
1318 } else if (LHS0 == RHS1)
1319 Y = RHS0;
1320 else
1321 return nullptr;
1322
1323 // Replace variable with constant value equivalence to remove a variable use:
1324 // (X == C) && (Y Pred1 X) --> (X == C) && (Y Pred1 C)
1325 // (X != C) || (Y Pred1 X) --> (X != C) || (Y Pred1 C)
1326 // Can think of the 'or' substitution with the 'and' bool equivalent:
1327 // A || B --> A || (!A && B)
1328 Value *SubstituteCmp = simplifyICmpInst(PredR, Y, LHS1, Q);
1329 if (!SubstituteCmp) {
1330 // If we need to create a new instruction, require that the old compare can
1331 // be removed.
1332 if (!RHSOneUse)
1333 return nullptr;
1334 SubstituteCmp = Builder.CreateICmp(PredR, Y, LHS1);
1335 }
1336 if (IsLogical) {
1337 Instruction *MDFrom = isa<SelectInst>(I) ? &I : nullptr;
1338 return IsAnd ? Builder.CreateLogicalAnd(LHS, SubstituteCmp, "", MDFrom)
1339 : Builder.CreateLogicalOr(LHS, SubstituteCmp, "", MDFrom);
1340 }
1341 return Builder.CreateBinOp(IsAnd ? Instruction::And : Instruction::Or, LHS,
1342 SubstituteCmp);
1343}
1344
1345/// Fold (icmp Pred1 V1, C1) & (icmp Pred2 V2, C2)
1346/// or (icmp Pred1 V1, C1) | (icmp Pred2 V2, C2)
1347/// into a single comparison using range-based reasoning.
1348/// NOTE: This is also used for logical and/or, must be poison-safe!
1349Value *InstCombinerImpl::foldAndOrOfICmpsUsingRanges(
1350 CmpPredicate PredL, Value *LHS0, Value *LHS1, bool LHSOneUse,
1351 CmpPredicate PredR, Value *RHS0, Value *RHS1, bool RHSOneUse, bool IsAnd) {
1352 // Return (V, CR) for a range check idiom V in CR.
1353 auto MatchExactRangeCheck =
1354 [](CmpPredicate Pred, Value *LHS,
1355 Value *RHS) -> std::optional<std::pair<Value *, ConstantRange>> {
1356 const APInt *C;
1357 if (!match(RHS, m_APInt(C)))
1358 return std::nullopt;
1359
1360 Value *X;
1361 // Match (x & NegPow2) ==/!= C
1362 const APInt *Mask;
1365 C->countr_zero() >= Mask->countr_zero()) {
1366 ConstantRange CR(*C, *C - *Mask);
1367 if (Pred == ICmpInst::ICMP_NE)
1368 CR = CR.inverse();
1369 return std::make_pair(X, CR);
1370 }
1371 ConstantRange CR = ConstantRange::makeExactICmpRegion(Pred, *C);
1372 // Match (add X, C1) pred C
1373 // TODO: investigate whether we should apply the one-use check on m_AddLike.
1374 const APInt *C1;
1375 if (match(LHS, m_AddLike(m_Value(X), m_APInt(C1))))
1376 return std::make_pair(X, CR.subtract(*C1));
1377 return std::make_pair(LHS, CR);
1378 };
1379
1380 auto RC1 = MatchExactRangeCheck(PredL, LHS0, LHS1);
1381 if (!RC1)
1382 return nullptr;
1383
1384 auto RC2 = MatchExactRangeCheck(PredR, RHS0, RHS1);
1385 if (!RC2)
1386 return nullptr;
1387
1388 auto &[V1, CR1] = *RC1;
1389 auto &[V2, CR2] = *RC2;
1390 if (V1 != V2)
1391 return nullptr;
1392
1393 // For 'and', we use the De Morgan's Laws to simplify the implementation.
1394 if (IsAnd) {
1395 CR1 = CR1.inverse();
1396 CR2 = CR2.inverse();
1397 }
1398
1399 Type *Ty = V1->getType();
1400 Value *NewV = V1;
1401 std::optional<ConstantRange> CR = CR1.exactUnionWith(CR2);
1402 if (!CR) {
1403 if (!LHSOneUse || !RHSOneUse || CR1.isWrappedSet() || CR2.isWrappedSet())
1404 return nullptr;
1405
1406 // Check whether we have equal-size ranges that only differ by one bit.
1407 // In that case we can apply a mask to map one range onto the other.
1408 APInt LowerDiff = CR1.getLower() ^ CR2.getLower();
1409 APInt UpperDiff = (CR1.getUpper() - 1) ^ (CR2.getUpper() - 1);
1410 APInt CR1Size = CR1.getUpper() - CR1.getLower();
1411 if (!LowerDiff.isPowerOf2() || LowerDiff != UpperDiff ||
1412 CR1Size != CR2.getUpper() - CR2.getLower())
1413 return nullptr;
1414
1415 CR = CR1.getLower().ult(CR2.getLower()) ? CR1 : CR2;
1416 NewV = Builder.CreateAnd(NewV, ConstantInt::get(Ty, ~LowerDiff));
1417 }
1418
1419 if (IsAnd)
1420 CR = CR->inverse();
1421
1422 CmpInst::Predicate NewPred;
1423 APInt NewC, Offset;
1424 CR->getEquivalentICmp(NewPred, NewC, Offset);
1425
1426 if (Offset != 0)
1427 NewV = Builder.CreateAdd(NewV, ConstantInt::get(Ty, Offset));
1428 return Builder.CreateICmp(NewPred, NewV, ConstantInt::get(Ty, NewC));
1429}
1430
1431/// Matches canonical form of isnan, fcmp ord x, 0
1435
1436/// Matches fcmp u__ x, +/-inf
1441
1442/// and (fcmp ord x, 0), (fcmp u* x, inf) -> fcmp o* x, inf
1443///
1444/// Clang emits this pattern for doing an isfinite check in __builtin_isnormal.
1446 FCmpInst *RHS) {
1447 Value *LHS0 = LHS->getOperand(0), *LHS1 = LHS->getOperand(1);
1448 Value *RHS0 = RHS->getOperand(0), *RHS1 = RHS->getOperand(1);
1449 FCmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
1450
1451 if (!matchIsNotNaN(PredL, LHS0, LHS1) ||
1452 !matchUnorderedInfCompare(PredR, RHS0, RHS1))
1453 return nullptr;
1454
1455 return Builder.CreateFCmpFMF(FCmpInst::getOrderedPredicate(PredR), RHS0, RHS1,
1457}
1458
1459Value *InstCombinerImpl::foldLogicOfFCmps(FCmpInst *LHS, FCmpInst *RHS,
1460 bool IsAnd, bool IsLogicalSelect) {
1461 Value *LHS0 = LHS->getOperand(0), *LHS1 = LHS->getOperand(1);
1462 Value *RHS0 = RHS->getOperand(0), *RHS1 = RHS->getOperand(1);
1463 FCmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
1464
1465 if (LHS0 == RHS1 && RHS0 == LHS1) {
1466 // Swap RHS operands to match LHS.
1467 PredR = FCmpInst::getSwappedPredicate(PredR);
1468 std::swap(RHS0, RHS1);
1469 }
1470
1471 // Simplify (fcmp cc0 x, y) & (fcmp cc1 x, y).
1472 // Suppose the relation between x and y is R, where R is one of
1473 // U(1000), L(0100), G(0010) or E(0001), and CC0 and CC1 are the bitmasks for
1474 // testing the desired relations.
1475 //
1476 // Since (R & CC0) and (R & CC1) are either R or 0, we actually have this:
1477 // bool(R & CC0) && bool(R & CC1)
1478 // = bool((R & CC0) & (R & CC1))
1479 // = bool(R & (CC0 & CC1)) <= by re-association, commutation, and idempotency
1480 //
1481 // Since (R & CC0) and (R & CC1) are either R or 0, we actually have this:
1482 // bool(R & CC0) || bool(R & CC1)
1483 // = bool((R & CC0) | (R & CC1))
1484 // = bool(R & (CC0 | CC1)) <= by reversed distribution (contribution? ;)
1485 if (LHS0 == RHS0 && LHS1 == RHS1) {
1486 unsigned FCmpCodeL = getFCmpCode(PredL);
1487 unsigned FCmpCodeR = getFCmpCode(PredR);
1488 unsigned NewPred = IsAnd ? FCmpCodeL & FCmpCodeR : FCmpCodeL | FCmpCodeR;
1489
1490 // Intersect the fast math flags.
1491 // TODO: We can union the fast math flags unless this is a logical select.
1492 return getFCmpValue(NewPred, LHS0, LHS1, Builder,
1494 }
1495
1496 if ((PredL == FCmpInst::FCMP_ORD && PredR == FCmpInst::FCMP_ORD && IsAnd) ||
1497 (PredL == FCmpInst::FCMP_UNO && PredR == FCmpInst::FCMP_UNO && !IsAnd)) {
1498 if (LHS0->getType() != RHS0->getType())
1499 return nullptr;
1500
1501 // FCmp canonicalization ensures that (fcmp ord/uno X, X) and
1502 // (fcmp ord/uno X, C) will be transformed to (fcmp X, +0.0).
1503 if (match(LHS1, m_PosZeroFP()) && match(RHS1, m_PosZeroFP())) {
1504 // Ignore the constants because they are obviously not NANs:
1505 // (fcmp ord x, 0.0) & (fcmp ord y, 0.0) -> (fcmp ord x, y)
1506 // (fcmp uno x, 0.0) | (fcmp uno y, 0.0) -> (fcmp uno x, y)
1507 Value *Y = RHS0;
1508 FastMathFlags FMF = LHS->getFastMathFlags() & RHS->getFastMathFlags();
1509 if (IsLogicalSelect) {
1510 Y = Builder.CreateFreeze(Y, Y->getName() + ".fr");
1511 FMF.setNoNaNs(false);
1512 FMF.setNoInfs(false);
1513 }
1514 return Builder.CreateFCmpFMF(PredL, LHS0, Y, FMF);
1515 }
1516 }
1517
1518 // This transform is not valid for a logical select.
1519 if (!IsLogicalSelect && IsAnd &&
1520 stripSignOnlyFPOps(LHS0) == stripSignOnlyFPOps(RHS0)) {
1521 // and (fcmp ord x, 0), (fcmp u* x, inf) -> fcmp o* x, inf
1522 // and (fcmp ord x, 0), (fcmp u* fabs(x), inf) -> fcmp o* x, inf
1524 return Left;
1526 return Right;
1527 }
1528
1529 // Turn at least two fcmps with constants into llvm.is.fpclass.
1530 //
1531 // If we can represent a combined value test with one class call, we can
1532 // potentially eliminate 4-6 instructions. If we can represent a test with a
1533 // single fcmp with fneg and fabs, that's likely a better canonical form.
1534 if (LHS->hasOneUse() && RHS->hasOneUse()) {
1535 auto [ClassValRHS, ClassMaskRHS] =
1536 fcmpToClassTest(PredR, *RHS->getFunction(), RHS0, RHS1);
1537 if (ClassValRHS) {
1538 auto [ClassValLHS, ClassMaskLHS] =
1539 fcmpToClassTest(PredL, *LHS->getFunction(), LHS0, LHS1);
1540 if (ClassValLHS == ClassValRHS) {
1541 unsigned CombinedMask = IsAnd ? (ClassMaskLHS & ClassMaskRHS)
1542 : (ClassMaskLHS | ClassMaskRHS);
1543 return Builder.CreateIntrinsic(
1544 Intrinsic::is_fpclass, {ClassValLHS->getType()},
1545 {ClassValLHS, Builder.getInt32(CombinedMask)});
1546 }
1547 }
1548 }
1549
1550 // Canonicalize the range check idiom:
1551 // and (fcmp olt/ole/ult/ule x, C), (fcmp ogt/oge/ugt/uge x, -C)
1552 // --> fabs(x) olt/ole/ult/ule C
1553 // or (fcmp ogt/oge/ugt/uge x, C), (fcmp olt/ole/ult/ule x, -C)
1554 // --> fabs(x) ogt/oge/ugt/uge C
1555 // TODO: Generalize to handle a negated variable operand?
1556 const APFloat *LHSC, *RHSC;
1557 if (LHS0 == RHS0 && LHS->hasOneUse() && RHS->hasOneUse() &&
1558 FCmpInst::getSwappedPredicate(PredL) == PredR &&
1559 match(LHS1, m_APFloatAllowPoison(LHSC)) &&
1560 match(RHS1, m_APFloatAllowPoison(RHSC)) &&
1561 LHSC->bitwiseIsEqual(neg(*RHSC))) {
1562 auto IsLessThanOrLessEqual = [](FCmpInst::Predicate Pred) {
1563 switch (Pred) {
1564 case FCmpInst::FCMP_OLT:
1565 case FCmpInst::FCMP_OLE:
1566 case FCmpInst::FCMP_ULT:
1567 case FCmpInst::FCMP_ULE:
1568 return true;
1569 default:
1570 return false;
1571 }
1572 };
1573 if (IsLessThanOrLessEqual(IsAnd ? PredR : PredL)) {
1574 std::swap(LHSC, RHSC);
1575 std::swap(PredL, PredR);
1576 }
1577 if (IsLessThanOrLessEqual(IsAnd ? PredL : PredR)) {
1578 FastMathFlags NewFlag = LHS->getFastMathFlags();
1579 if (!IsLogicalSelect)
1580 NewFlag |= RHS->getFastMathFlags();
1581
1582 Value *FAbs = Builder.CreateFAbs(LHS0, NewFlag);
1583 return Builder.CreateFCmpFMF(
1584 PredL, FAbs, ConstantFP::get(LHS0->getType(), *LHSC), NewFlag);
1585 }
1586 }
1587
1588 return nullptr;
1589}
1590
1591/// Match an fcmp against a special value that performs a test possible by
1592/// llvm.is.fpclass.
1593static bool matchIsFPClassLikeFCmp(Value *Op, Value *&ClassVal,
1594 uint64_t &ClassMask) {
1595 auto *FCmp = dyn_cast<FCmpInst>(Op);
1596 if (!FCmp || !FCmp->hasOneUse())
1597 return false;
1598
1599 std::tie(ClassVal, ClassMask) =
1600 fcmpToClassTest(FCmp->getPredicate(), *FCmp->getParent()->getParent(),
1601 FCmp->getOperand(0), FCmp->getOperand(1));
1602 return ClassVal != nullptr;
1603}
1604
1605/// or (is_fpclass x, mask0), (is_fpclass x, mask1)
1606/// -> is_fpclass x, (mask0 | mask1)
1607/// and (is_fpclass x, mask0), (is_fpclass x, mask1)
1608/// -> is_fpclass x, (mask0 & mask1)
1609/// xor (is_fpclass x, mask0), (is_fpclass x, mask1)
1610/// -> is_fpclass x, (mask0 ^ mask1)
1611Instruction *InstCombinerImpl::foldLogicOfIsFPClass(BinaryOperator &BO,
1612 Value *Op0, Value *Op1) {
1613 Value *ClassVal0 = nullptr;
1614 Value *ClassVal1 = nullptr;
1615 uint64_t ClassMask0, ClassMask1;
1616
1617 // Restrict to folding one fcmp into one is.fpclass for now, don't introduce a
1618 // new class.
1619 //
1620 // TODO: Support forming is.fpclass out of 2 separate fcmps when codegen is
1621 // better.
1622
1623 bool IsLHSClass =
1625 m_Value(ClassVal0), m_ConstantInt(ClassMask0))));
1626 bool IsRHSClass =
1628 m_Value(ClassVal1), m_ConstantInt(ClassMask1))));
1629 if ((((IsLHSClass || matchIsFPClassLikeFCmp(Op0, ClassVal0, ClassMask0)) &&
1630 (IsRHSClass || matchIsFPClassLikeFCmp(Op1, ClassVal1, ClassMask1)))) &&
1631 ClassVal0 == ClassVal1) {
1632 unsigned NewClassMask;
1633 switch (BO.getOpcode()) {
1634 case Instruction::And:
1635 NewClassMask = ClassMask0 & ClassMask1;
1636 break;
1637 case Instruction::Or:
1638 NewClassMask = ClassMask0 | ClassMask1;
1639 break;
1640 case Instruction::Xor:
1641 NewClassMask = ClassMask0 ^ ClassMask1;
1642 break;
1643 default:
1644 llvm_unreachable("not a binary logic operator");
1645 }
1646
1647 if (IsLHSClass) {
1648 auto *II = cast<IntrinsicInst>(Op0);
1649 II->setArgOperand(
1650 1, ConstantInt::get(II->getArgOperand(1)->getType(), NewClassMask));
1651 return replaceInstUsesWith(BO, II);
1652 }
1653
1654 if (IsRHSClass) {
1655 auto *II = cast<IntrinsicInst>(Op1);
1656 II->setArgOperand(
1657 1, ConstantInt::get(II->getArgOperand(1)->getType(), NewClassMask));
1658 return replaceInstUsesWith(BO, II);
1659 }
1660
1661 Value *NewClass =
1662 Builder.CreateIntrinsic(Intrinsic::is_fpclass, {ClassVal0->getType()},
1663 {ClassVal0, Builder.getInt32(NewClassMask)});
1664 return replaceInstUsesWith(BO, NewClass);
1665 }
1666
1667 return nullptr;
1668}
1669
1670/// Look for the pattern that conditionally negates a value via math operations:
1671/// cond.splat = sext i1 cond
1672/// sub = add cond.splat, x
1673/// xor = xor sub, cond.splat
1674/// and rewrite it to do the same, but via logical operations:
1675/// value.neg = sub 0, value
1676/// cond = select i1 neg, value.neg, value
1677Instruction *InstCombinerImpl::canonicalizeConditionalNegationViaMathToSelect(
1678 BinaryOperator &I) {
1679 assert(I.getOpcode() == BinaryOperator::Xor && "Only for xor!");
1680 Value *Cond, *X;
1681 // As per complexity ordering, `xor` is not commutative here.
1682 if (!match(&I, m_c_BinOp(m_OneUse(m_Value()), m_Value())) ||
1683 !match(I.getOperand(1), m_SExt(m_Value(Cond))) ||
1684 !Cond->getType()->isIntOrIntVectorTy(1) ||
1685 !match(I.getOperand(0), m_c_Add(m_SExt(m_Specific(Cond)), m_Value(X))))
1686 return nullptr;
1687 return createSelectInstWithUnknownProfile(
1688 Cond, Builder.CreateNeg(X, X->getName() + ".neg"), X);
1689}
1690
1691/// This a limited reassociation for a special case (see above) where we are
1692/// checking if two values are either both NAN (unordered) or not-NAN (ordered).
1693/// This could be handled more generally in '-reassociation', but it seems like
1694/// an unlikely pattern for a large number of logic ops and fcmps.
1696 InstCombiner::BuilderTy &Builder) {
1697 Instruction::BinaryOps Opcode = BO.getOpcode();
1698 assert((Opcode == Instruction::And || Opcode == Instruction::Or) &&
1699 "Expecting and/or op for fcmp transform");
1700
1701 // There are 4 commuted variants of the pattern. Canonicalize operands of this
1702 // logic op so an fcmp is operand 0 and a matching logic op is operand 1.
1703 Value *Op0 = BO.getOperand(0), *Op1 = BO.getOperand(1), *X;
1704 if (match(Op1, m_FCmp(m_Value(), m_AnyZeroFP())))
1705 std::swap(Op0, Op1);
1706
1707 // Match inner binop and the predicate for combining 2 NAN checks into 1.
1708 Value *BO10, *BO11;
1709 FCmpInst::Predicate NanPred = Opcode == Instruction::And ? FCmpInst::FCMP_ORD
1711 if (!match(Op0, m_SpecificFCmp(NanPred, m_Value(X), m_AnyZeroFP())) ||
1712 !match(Op1, m_BinOp(Opcode, m_Value(BO10), m_Value(BO11))))
1713 return nullptr;
1714
1715 // The inner logic op must have a matching fcmp operand.
1716 Value *Y;
1717 if (!match(BO10, m_SpecificFCmp(NanPred, m_Value(Y), m_AnyZeroFP())) ||
1718 X->getType() != Y->getType())
1719 std::swap(BO10, BO11);
1720
1721 if (!match(BO10, m_SpecificFCmp(NanPred, m_Value(Y), m_AnyZeroFP())) ||
1722 X->getType() != Y->getType())
1723 return nullptr;
1724
1725 // and (fcmp ord X, 0), (and (fcmp ord Y, 0), Z) --> and (fcmp ord X, Y), Z
1726 // or (fcmp uno X, 0), (or (fcmp uno Y, 0), Z) --> or (fcmp uno X, Y), Z
1727 // Intersect FMF from the 2 source fcmps.
1728 Value *NewFCmp =
1729 Builder.CreateFCmpFMF(NanPred, X, Y, FMFSource::intersect(Op0, BO10));
1730 return BinaryOperator::Create(Opcode, NewFCmp, BO11);
1731}
1732
1733/// Match variations of De Morgan's Laws:
1734/// (~A & ~B) == (~(A | B))
1735/// (~A | ~B) == (~(A & B))
1737 InstCombiner &IC) {
1738 const Instruction::BinaryOps Opcode = I.getOpcode();
1739 assert((Opcode == Instruction::And || Opcode == Instruction::Or) &&
1740 "Trying to match De Morgan's Laws with something other than and/or");
1741
1742 // Flip the logic operation.
1743 const Instruction::BinaryOps FlippedOpcode =
1744 (Opcode == Instruction::And) ? Instruction::Or : Instruction::And;
1745
1746 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
1747 Value *A, *B;
1748 if (match(Op0, m_OneUse(m_Not(m_Value(A)))) &&
1749 match(Op1, m_OneUse(m_Not(m_Value(B)))) &&
1750 !IC.isFreeToInvert(A, A->hasOneUse()) &&
1751 !IC.isFreeToInvert(B, B->hasOneUse())) {
1752 Value *AndOr =
1753 IC.Builder.CreateBinOp(FlippedOpcode, A, B, I.getName() + ".demorgan");
1754 return BinaryOperator::CreateNot(AndOr);
1755 }
1756
1757 // The 'not' ops may require reassociation.
1758 // (A & ~B) & ~C --> A & ~(B | C)
1759 // (~B & A) & ~C --> A & ~(B | C)
1760 // (A | ~B) | ~C --> A | ~(B & C)
1761 // (~B | A) | ~C --> A | ~(B & C)
1762 Value *C;
1763 if (match(Op0, m_OneUse(m_c_BinOp(Opcode, m_Value(A), m_Not(m_Value(B))))) &&
1764 match(Op1, m_Not(m_Value(C)))) {
1765 Value *FlippedBO = IC.Builder.CreateBinOp(FlippedOpcode, B, C);
1766 return BinaryOperator::Create(Opcode, A, IC.Builder.CreateNot(FlippedBO));
1767 }
1768
1769 return nullptr;
1770}
1771
1772bool InstCombinerImpl::shouldOptimizeCast(CastInst *CI) {
1773 Value *CastSrc = CI->getOperand(0);
1774
1775 // Noop casts and casts of constants should be eliminated trivially.
1776 if (CI->getSrcTy() == CI->getDestTy() || isa<Constant>(CastSrc))
1777 return false;
1778
1779 // If this cast is paired with another cast that can be eliminated, we prefer
1780 // to have it eliminated.
1781 if (const auto *PrecedingCI = dyn_cast<CastInst>(CastSrc))
1782 if (isEliminableCastPair(PrecedingCI, CI))
1783 return false;
1784
1785 return true;
1786}
1787
1788/// Fold {and,or,xor} (cast X), C.
1790 InstCombinerImpl &IC) {
1792 if (!C)
1793 return nullptr;
1794
1795 auto LogicOpc = Logic.getOpcode();
1796 Type *DestTy = Logic.getType();
1797 Type *SrcTy = Cast->getSrcTy();
1798
1799 // Move the logic operation ahead of a zext or sext if the constant is
1800 // unchanged in the smaller source type. Performing the logic in a smaller
1801 // type may provide more information to later folds, and the smaller logic
1802 // instruction may be cheaper (particularly in the case of vectors).
1803 Value *X;
1804 auto &DL = IC.getDataLayout();
1805 if (match(Cast, m_OneUse(m_ZExt(m_Value(X))))) {
1806 PreservedCastFlags Flags;
1807 if (Constant *TruncC = getLosslessUnsignedTrunc(C, SrcTy, DL, &Flags)) {
1808 // LogicOpc (zext X), C --> zext (LogicOpc X, C)
1809 Value *NewOp = IC.Builder.CreateBinOp(LogicOpc, X, TruncC);
1810 auto *ZExt = new ZExtInst(NewOp, DestTy);
1811 ZExt->setNonNeg(Flags.NNeg);
1812 ZExt->andIRFlags(Cast);
1813 return ZExt;
1814 }
1815 }
1816
1817 if (match(Cast, m_OneUse(m_SExtLike(m_Value(X))))) {
1818 if (Constant *TruncC = getLosslessSignedTrunc(C, SrcTy, DL)) {
1819 // LogicOpc (sext X), C --> sext (LogicOpc X, C)
1820 Value *NewOp = IC.Builder.CreateBinOp(LogicOpc, X, TruncC);
1821 return new SExtInst(NewOp, DestTy);
1822 }
1823 }
1824
1825 return nullptr;
1826}
1827
1828/// Fold {and,or,xor} (cast X), Y.
1829Instruction *InstCombinerImpl::foldCastedBitwiseLogic(BinaryOperator &I) {
1830 auto LogicOpc = I.getOpcode();
1831 assert(I.isBitwiseLogicOp() && "Unexpected opcode for bitwise logic folding");
1832
1833 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
1834
1835 // fold bitwise(A >> BW - 1, zext(icmp)) (BW is the scalar bits of the
1836 // type of A)
1837 // -> bitwise(zext(A < 0), zext(icmp))
1838 // -> zext(bitwise(A < 0, icmp))
1839 auto FoldBitwiseICmpZeroWithICmp = [&](Value *Op0,
1840 Value *Op1) -> Instruction * {
1841 Value *A;
1842 bool IsMatched =
1843 match(Op0,
1845 m_Value(A),
1846 m_SpecificInt(Op0->getType()->getScalarSizeInBits() - 1)))) &&
1847 match(Op1, m_OneUse(m_ZExt(m_ICmp(m_Value(), m_Value()))));
1848
1849 if (!IsMatched)
1850 return nullptr;
1851
1852 auto *ICmpL =
1853 Builder.CreateICmpSLT(A, Constant::getNullValue(A->getType()));
1854 auto *ICmpR = cast<ZExtInst>(Op1)->getOperand(0);
1855 auto *BitwiseOp = Builder.CreateBinOp(LogicOpc, ICmpL, ICmpR);
1856
1857 return new ZExtInst(BitwiseOp, Op0->getType());
1858 };
1859
1860 if (auto *Ret = FoldBitwiseICmpZeroWithICmp(Op0, Op1))
1861 return Ret;
1862
1863 if (auto *Ret = FoldBitwiseICmpZeroWithICmp(Op1, Op0))
1864 return Ret;
1865
1866 CastInst *Cast0 = dyn_cast<CastInst>(Op0);
1867 if (!Cast0)
1868 return nullptr;
1869
1870 // This must be a cast from an integer or integer vector source type to allow
1871 // transformation of the logic operation to the source type.
1872 Type *DestTy = I.getType();
1873 Type *SrcTy = Cast0->getSrcTy();
1874 if (!SrcTy->isIntOrIntVectorTy())
1875 return nullptr;
1876
1877 if (Instruction *Ret = foldLogicCastConstant(I, Cast0, *this))
1878 return Ret;
1879
1880 CastInst *Cast1 = dyn_cast<CastInst>(Op1);
1881 if (!Cast1)
1882 return nullptr;
1883
1884 // Both operands of the logic operation are casts. The casts must be the
1885 // same kind for reduction.
1886 Instruction::CastOps CastOpcode = Cast0->getOpcode();
1887 if (CastOpcode != Cast1->getOpcode())
1888 return nullptr;
1889
1890 // Can't fold it profitably if no one of casts has one use.
1891 if (!Cast0->hasOneUse() && !Cast1->hasOneUse())
1892 return nullptr;
1893
1894 Value *X, *Y;
1895 if (match(Cast0, m_ZExtOrSExt(m_Value(X))) &&
1896 match(Cast1, m_ZExtOrSExt(m_Value(Y)))) {
1897 // Cast the narrower source to the wider source type.
1898 unsigned XNumBits = X->getType()->getScalarSizeInBits();
1899 unsigned YNumBits = Y->getType()->getScalarSizeInBits();
1900 if (XNumBits != YNumBits) {
1901 // Cast the narrower source to the wider source type only if both of casts
1902 // have one use to avoid creating an extra instruction.
1903 if (!Cast0->hasOneUse() || !Cast1->hasOneUse())
1904 return nullptr;
1905
1906 // If the source types do not match, but the casts are matching extends,
1907 // we can still narrow the logic op.
1908 if (XNumBits < YNumBits) {
1909 X = Builder.CreateCast(CastOpcode, X, Y->getType());
1910 } else if (YNumBits < XNumBits) {
1911 Y = Builder.CreateCast(CastOpcode, Y, X->getType());
1912 }
1913 }
1914
1915 // Do the logic op in the intermediate width, then widen more.
1916 Value *NarrowLogic = Builder.CreateBinOp(LogicOpc, X, Y, I.getName());
1917 auto *Disjoint = dyn_cast<PossiblyDisjointInst>(&I);
1918 auto *NewDisjoint = dyn_cast<PossiblyDisjointInst>(NarrowLogic);
1919 if (Disjoint && NewDisjoint)
1920 NewDisjoint->setIsDisjoint(Disjoint->isDisjoint());
1921 return CastInst::Create(CastOpcode, NarrowLogic, DestTy);
1922 }
1923
1924 // If the src type of casts are different, give up for other cast opcodes.
1925 if (SrcTy != Cast1->getSrcTy())
1926 return nullptr;
1927
1928 Value *Cast0Src = Cast0->getOperand(0);
1929 Value *Cast1Src = Cast1->getOperand(0);
1930
1931 // fold logic(cast(A), cast(B)) -> cast(logic(A, B))
1932 if (shouldOptimizeCast(Cast0) && shouldOptimizeCast(Cast1)) {
1933 Value *NewOp = Builder.CreateBinOp(LogicOpc, Cast0Src, Cast1Src,
1934 I.getName());
1935 auto *NewCast = CastInst::Create(CastOpcode, NewOp, DestTy);
1936 if (auto *NewTrunc = dyn_cast<TruncInst>(NewCast)) {
1937 auto *Trunc0 = cast<TruncInst>(Cast0);
1938 auto *Trunc1 = cast<TruncInst>(Cast1);
1939 NewTrunc->setHasNoUnsignedWrap(
1940 LogicOpc == Instruction::And
1941 ? Trunc0->hasNoUnsignedWrap() || Trunc1->hasNoUnsignedWrap()
1942 : Trunc0->hasNoUnsignedWrap() && Trunc1->hasNoUnsignedWrap());
1943 NewTrunc->setHasNoSignedWrap(Trunc0->hasNoSignedWrap() &&
1944 Trunc1->hasNoSignedWrap());
1945 }
1946 return NewCast;
1947 }
1948
1949 return nullptr;
1950}
1951
1953 InstCombiner::BuilderTy &Builder) {
1954 assert(I.getOpcode() == Instruction::And);
1955 Value *Op0 = I.getOperand(0);
1956 Value *Op1 = I.getOperand(1);
1957 Value *A, *B;
1958
1959 // Operand complexity canonicalization guarantees that the 'or' is Op0.
1960 // (A | B) & ~(A & B) --> A ^ B
1961 // (A | B) & ~(B & A) --> A ^ B
1962 if (match(&I, m_BinOp(m_Or(m_Value(A), m_Value(B)),
1964 return BinaryOperator::CreateXor(A, B);
1965
1966 // (A | ~B) & (~A | B) --> ~(A ^ B)
1967 // (A | ~B) & (B | ~A) --> ~(A ^ B)
1968 // (~B | A) & (~A | B) --> ~(A ^ B)
1969 // (~B | A) & (B | ~A) --> ~(A ^ B)
1970 if (Op0->hasOneUse() || Op1->hasOneUse())
1973 return BinaryOperator::CreateNot(Builder.CreateXor(A, B));
1974
1975 return nullptr;
1976}
1977
1979 InstCombiner::BuilderTy &Builder) {
1980 assert(I.getOpcode() == Instruction::Or);
1981 Value *Op0 = I.getOperand(0);
1982 Value *Op1 = I.getOperand(1);
1983 Value *A, *B;
1984
1985 // Operand complexity canonicalization guarantees that the 'and' is Op0.
1986 // (A & B) | ~(A | B) --> ~(A ^ B)
1987 // (A & B) | ~(B | A) --> ~(A ^ B)
1988 if (Op0->hasOneUse() || Op1->hasOneUse())
1989 if (match(Op0, m_And(m_Value(A), m_Value(B))) &&
1991 return BinaryOperator::CreateNot(Builder.CreateXor(A, B));
1992
1993 // Operand complexity canonicalization guarantees that the 'xor' is Op0.
1994 // (A ^ B) | ~(A | B) --> ~(A & B)
1995 // (A ^ B) | ~(B | A) --> ~(A & B)
1996 if (Op0->hasOneUse() || Op1->hasOneUse())
1997 if (match(Op0, m_Xor(m_Value(A), m_Value(B))) &&
1999 return BinaryOperator::CreateNot(Builder.CreateAnd(A, B));
2000
2001 // (A & ~B) | (~A & B) --> A ^ B
2002 // (A & ~B) | (B & ~A) --> A ^ B
2003 // (~B & A) | (~A & B) --> A ^ B
2004 // (~B & A) | (B & ~A) --> A ^ B
2005 if (match(Op0, m_c_And(m_Value(A), m_Not(m_Value(B)))) &&
2007 return BinaryOperator::CreateXor(A, B);
2008
2009 return nullptr;
2010}
2011
2012/// Return true if a constant shift amount is always less than the specified
2013/// bit-width. If not, the shift could create poison in the narrower type.
2014static bool canNarrowShiftAmt(Constant *C, unsigned BitWidth) {
2015 APInt Threshold(C->getType()->getScalarSizeInBits(), BitWidth);
2016 return match(C, m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, Threshold));
2017}
2018
2019/// Try to use narrower ops (sink zext ops) for an 'and' with binop operand and
2020/// a common zext operand: and (binop (zext X), C), (zext X).
2021Instruction *InstCombinerImpl::narrowMaskedBinOp(BinaryOperator &And) {
2022 // This transform could also apply to {or, and, xor}, but there are better
2023 // folds for those cases, so we don't expect those patterns here. AShr is not
2024 // handled because it should always be transformed to LShr in this sequence.
2025 // The subtract transform is different because it has a constant on the left.
2026 // Add/mul commute the constant to RHS; sub with constant RHS becomes add.
2027 Value *Op0 = And.getOperand(0), *Op1 = And.getOperand(1);
2028 Constant *C;
2029 if (!match(Op0, m_OneUse(m_Add(m_Specific(Op1), m_Constant(C)))) &&
2030 !match(Op0, m_OneUse(m_Mul(m_Specific(Op1), m_Constant(C)))) &&
2031 !match(Op0, m_OneUse(m_LShr(m_Specific(Op1), m_Constant(C)))) &&
2032 !match(Op0, m_OneUse(m_Shl(m_Specific(Op1), m_Constant(C)))) &&
2033 !match(Op0, m_OneUse(m_Sub(m_Constant(C), m_Specific(Op1)))))
2034 return nullptr;
2035
2036 Value *X;
2037 if (!match(Op1, m_ZExt(m_Value(X))) || Op1->hasNUsesOrMore(3))
2038 return nullptr;
2039
2040 Type *Ty = And.getType();
2041 if (!isa<VectorType>(Ty) && !shouldChangeType(Ty, X->getType()))
2042 return nullptr;
2043
2044 // If we're narrowing a shift, the shift amount must be safe (less than the
2045 // width) in the narrower type. If the shift amount is greater, instsimplify
2046 // usually handles that case, but we can't guarantee/assert it.
2048 if (Opc == Instruction::LShr || Opc == Instruction::Shl)
2049 if (!canNarrowShiftAmt(C, X->getType()->getScalarSizeInBits()))
2050 return nullptr;
2051
2052 // and (sub C, (zext X)), (zext X) --> zext (and (sub C', X), X)
2053 // and (binop (zext X), C), (zext X) --> zext (and (binop X, C'), X)
2054 Value *NewC = ConstantExpr::getTrunc(C, X->getType());
2055 Value *NewBO = Opc == Instruction::Sub ? Builder.CreateBinOp(Opc, NewC, X)
2056 : Builder.CreateBinOp(Opc, X, NewC);
2057 return new ZExtInst(Builder.CreateAnd(NewBO, X), Ty);
2058}
2059
2060/// Try folding relatively complex patterns for both And and Or operations
2061/// with all And and Or swapped.
2063 InstCombiner::BuilderTy &Builder) {
2064 const Instruction::BinaryOps Opcode = I.getOpcode();
2065 assert(Opcode == Instruction::And || Opcode == Instruction::Or);
2066
2067 // Flip the logic operation.
2068 const Instruction::BinaryOps FlippedOpcode =
2069 (Opcode == Instruction::And) ? Instruction::Or : Instruction::And;
2070
2071 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
2072 Value *A, *B, *C, *X, *Y, *Dummy;
2073
2074 // Match following expressions:
2075 // (~(A | B) & C)
2076 // (~(A & B) | C)
2077 // Captures X = ~(A | B) or ~(A & B)
2078 const auto matchNotOrAnd =
2079 [Opcode, FlippedOpcode](Value *Op, auto m_A, auto m_B, auto m_C,
2080 Value *&X, bool CountUses = false) -> bool {
2081 if (CountUses && !Op->hasOneUse())
2082 return false;
2083
2084 if (match(Op,
2085 m_c_BinOp(FlippedOpcode,
2086 m_Value(X, m_Not(m_c_BinOp(Opcode, m_A, m_B))), m_C)))
2087 return !CountUses || X->hasOneUse();
2088
2089 return false;
2090 };
2091
2092 // (~(A | B) & C) | ... --> ...
2093 // (~(A & B) | C) & ... --> ...
2094 // TODO: One use checks are conservative. We just need to check that a total
2095 // number of multiple used values does not exceed reduction
2096 // in operations.
2097 if (matchNotOrAnd(Op0, m_Value(A), m_Value(B), m_Value(C), X)) {
2098 // (~(A | B) & C) | (~(A | C) & B) --> (B ^ C) & ~A
2099 // (~(A & B) | C) & (~(A & C) | B) --> ~((B ^ C) & A)
2100 if (matchNotOrAnd(Op1, m_Specific(A), m_Specific(C), m_Specific(B), Dummy,
2101 true)) {
2102 Value *Xor = Builder.CreateXor(B, C);
2103 return (Opcode == Instruction::Or)
2104 ? BinaryOperator::CreateAnd(Xor, Builder.CreateNot(A))
2105 : BinaryOperator::CreateNot(Builder.CreateAnd(Xor, A));
2106 }
2107
2108 // (~(A | B) & C) | (~(B | C) & A) --> (A ^ C) & ~B
2109 // (~(A & B) | C) & (~(B & C) | A) --> ~((A ^ C) & B)
2110 if (matchNotOrAnd(Op1, m_Specific(B), m_Specific(C), m_Specific(A), Dummy,
2111 true)) {
2112 Value *Xor = Builder.CreateXor(A, C);
2113 return (Opcode == Instruction::Or)
2114 ? BinaryOperator::CreateAnd(Xor, Builder.CreateNot(B))
2115 : BinaryOperator::CreateNot(Builder.CreateAnd(Xor, B));
2116 }
2117
2118 // (~(A | B) & C) | ~(A | C) --> ~((B & C) | A)
2119 // (~(A & B) | C) & ~(A & C) --> ~((B | C) & A)
2120 if (match(Op1, m_OneUse(m_Not(m_OneUse(
2121 m_c_BinOp(Opcode, m_Specific(A), m_Specific(C)))))))
2122 return BinaryOperator::CreateNot(Builder.CreateBinOp(
2123 Opcode, Builder.CreateBinOp(FlippedOpcode, B, C), A));
2124
2125 // (~(A | B) & C) | ~(B | C) --> ~((A & C) | B)
2126 // (~(A & B) | C) & ~(B & C) --> ~((A | C) & B)
2127 if (match(Op1, m_OneUse(m_Not(m_OneUse(
2128 m_c_BinOp(Opcode, m_Specific(B), m_Specific(C)))))))
2129 return BinaryOperator::CreateNot(Builder.CreateBinOp(
2130 Opcode, Builder.CreateBinOp(FlippedOpcode, A, C), B));
2131
2132 // (~(A | B) & C) | ~(C | (A ^ B)) --> ~((A | B) & (C | (A ^ B)))
2133 // Note, the pattern with swapped and/or is not handled because the
2134 // result is more undefined than a source:
2135 // (~(A & B) | C) & ~(C & (A ^ B)) --> (A ^ B ^ C) | ~(A | C) is invalid.
2136 if (Opcode == Instruction::Or && Op0->hasOneUse() &&
2137 match(Op1,
2139 Y, m_c_BinOp(Opcode, m_Specific(C),
2140 m_c_Xor(m_Specific(A), m_Specific(B)))))))) {
2141 // X = ~(A | B)
2142 // Y = (C | (A ^ B)
2143 Value *Or = cast<BinaryOperator>(X)->getOperand(0);
2144 return BinaryOperator::CreateNot(Builder.CreateAnd(Or, Y));
2145 }
2146 }
2147
2148 // (~A & B & C) | ... --> ...
2149 // (~A | B | C) | ... --> ...
2150 // TODO: One use checks are conservative. We just need to check that a total
2151 // number of multiple used values does not exceed reduction
2152 // in operations.
2153 if (match(Op0,
2154 m_OneUse(m_c_BinOp(FlippedOpcode,
2155 m_BinOp(FlippedOpcode, m_Value(B), m_Value(C)),
2156 m_Value(X, m_Not(m_Value(A)))))) ||
2157 match(Op0, m_OneUse(m_c_BinOp(FlippedOpcode,
2158 m_c_BinOp(FlippedOpcode, m_Value(C),
2159 m_Value(X, m_Not(m_Value(A)))),
2160 m_Value(B))))) {
2161 // X = ~A
2162 // (~A & B & C) | ~(A | B | C) --> ~(A | (B ^ C))
2163 // (~A | B | C) & ~(A & B & C) --> (~A | (B ^ C))
2164 if (match(Op1, m_OneUse(m_Not(m_c_BinOp(
2165 Opcode, m_c_BinOp(Opcode, m_Specific(A), m_Specific(B)),
2166 m_Specific(C))))) ||
2168 Opcode, m_c_BinOp(Opcode, m_Specific(B), m_Specific(C)),
2169 m_Specific(A))))) ||
2171 Opcode, m_c_BinOp(Opcode, m_Specific(A), m_Specific(C)),
2172 m_Specific(B)))))) {
2173 Value *Xor = Builder.CreateXor(B, C);
2174 return (Opcode == Instruction::Or)
2175 ? BinaryOperator::CreateNot(Builder.CreateOr(Xor, A))
2176 : BinaryOperator::CreateOr(Xor, X);
2177 }
2178
2179 // (~A & B & C) | ~(A | B) --> (C | ~B) & ~A
2180 // (~A | B | C) & ~(A & B) --> (C & ~B) | ~A
2181 if (match(Op1, m_OneUse(m_Not(m_OneUse(
2182 m_c_BinOp(Opcode, m_Specific(A), m_Specific(B)))))))
2184 FlippedOpcode, Builder.CreateBinOp(Opcode, C, Builder.CreateNot(B)),
2185 X);
2186
2187 // (~A & B & C) | ~(A | C) --> (B | ~C) & ~A
2188 // (~A | B | C) & ~(A & C) --> (B & ~C) | ~A
2189 if (match(Op1, m_OneUse(m_Not(m_OneUse(
2190 m_c_BinOp(Opcode, m_Specific(A), m_Specific(C)))))))
2192 FlippedOpcode, Builder.CreateBinOp(Opcode, B, Builder.CreateNot(C)),
2193 X);
2194 }
2195
2196 return nullptr;
2197}
2198
2199/// Try to reassociate a pair of binops so that values with one use only are
2200/// part of the same instruction. This may enable folds that are limited with
2201/// multi-use restrictions and makes it more likely to match other patterns that
2202/// are looking for a common operand.
2204 InstCombinerImpl::BuilderTy &Builder) {
2205 Instruction::BinaryOps Opcode = BO.getOpcode();
2206 Value *X, *Y, *Z;
2207 if (match(&BO,
2208 m_c_BinOp(Opcode, m_OneUse(m_BinOp(Opcode, m_Value(X), m_Value(Y))),
2209 m_OneUse(m_Value(Z))))) {
2210 if (!isa<Constant>(X) && !isa<Constant>(Y) && !isa<Constant>(Z)) {
2211 // (X op Y) op Z --> (Y op Z) op X
2212 if (!X->hasOneUse()) {
2213 Value *YZ = Builder.CreateBinOp(Opcode, Y, Z);
2214 return BinaryOperator::Create(Opcode, YZ, X);
2215 }
2216 // (X op Y) op Z --> (X op Z) op Y
2217 if (!Y->hasOneUse()) {
2218 Value *XZ = Builder.CreateBinOp(Opcode, X, Z);
2219 return BinaryOperator::Create(Opcode, XZ, Y);
2220 }
2221 }
2222 }
2223
2224 return nullptr;
2225}
2226
2227// Match
2228// (X + C2) | C
2229// (X + C2) ^ C
2230// (X + C2) & C
2231// and convert to do the bitwise logic first:
2232// (X | C) + C2
2233// (X ^ C) + C2
2234// (X & C) + C2
2235// iff bits affected by logic op are lower than last bit affected by math op
2237 InstCombiner::BuilderTy &Builder) {
2238 Type *Ty = I.getType();
2239 Instruction::BinaryOps OpC = I.getOpcode();
2240 Value *Op0 = I.getOperand(0);
2241 Value *Op1 = I.getOperand(1);
2242 Value *X;
2243 const APInt *C, *C2;
2244
2245 if (!(match(Op0, m_OneUse(m_Add(m_Value(X), m_APInt(C2)))) &&
2246 match(Op1, m_APInt(C))))
2247 return nullptr;
2248
2249 unsigned Width = Ty->getScalarSizeInBits();
2250 unsigned LastOneMath = Width - C2->countr_zero();
2251
2252 switch (OpC) {
2253 case Instruction::And:
2254 if (C->countl_one() < LastOneMath)
2255 return nullptr;
2256 break;
2257 case Instruction::Xor:
2258 case Instruction::Or:
2259 if (C->countl_zero() < LastOneMath)
2260 return nullptr;
2261 break;
2262 default:
2263 llvm_unreachable("Unexpected BinaryOp!");
2264 }
2265
2266 Value *NewBinOp = Builder.CreateBinOp(OpC, X, ConstantInt::get(Ty, *C));
2267 return BinaryOperator::CreateWithCopiedFlags(Instruction::Add, NewBinOp,
2268 ConstantInt::get(Ty, *C2), Op0);
2269}
2270
2271// binop(shift(ShiftedC1, ShAmt), shift(ShiftedC2, add(ShAmt, AddC))) ->
2272// shift(binop(ShiftedC1, shift(ShiftedC2, AddC)), ShAmt)
2273// where both shifts are the same and AddC is a valid shift amount.
2274Instruction *InstCombinerImpl::foldBinOpOfDisplacedShifts(BinaryOperator &I) {
2275 assert((I.isBitwiseLogicOp() || I.getOpcode() == Instruction::Add) &&
2276 "Unexpected opcode");
2277
2278 Value *ShAmt;
2279 Constant *ShiftedC1, *ShiftedC2, *AddC;
2280 Type *Ty = I.getType();
2281 unsigned BitWidth = Ty->getScalarSizeInBits();
2282 if (!match(&I, m_c_BinOp(m_Shift(m_ImmConstant(ShiftedC1), m_Value(ShAmt)),
2283 m_Shift(m_ImmConstant(ShiftedC2),
2284 m_AddLike(m_Deferred(ShAmt),
2285 m_ImmConstant(AddC))))))
2286 return nullptr;
2287
2288 // Make sure the add constant is a valid shift amount.
2289 if (!match(AddC,
2291 return nullptr;
2292
2293 // Avoid constant expressions.
2294 auto *Op0Inst = dyn_cast<Instruction>(I.getOperand(0));
2295 auto *Op1Inst = dyn_cast<Instruction>(I.getOperand(1));
2296 if (!Op0Inst || !Op1Inst)
2297 return nullptr;
2298
2299 // Both shifts must be the same.
2300 Instruction::BinaryOps ShiftOp =
2301 static_cast<Instruction::BinaryOps>(Op0Inst->getOpcode());
2302 if (ShiftOp != Op1Inst->getOpcode())
2303 return nullptr;
2304
2305 // For adds, only left shifts are supported.
2306 if (I.getOpcode() == Instruction::Add && ShiftOp != Instruction::Shl)
2307 return nullptr;
2308
2309 Value *NewC = Builder.CreateBinOp(
2310 I.getOpcode(), ShiftedC1, Builder.CreateBinOp(ShiftOp, ShiftedC2, AddC));
2311 return BinaryOperator::Create(ShiftOp, NewC, ShAmt);
2312}
2313
2314// Fold and/or/xor with two equal intrinsic IDs:
2315// bitwise(fshl (A, B, ShAmt), fshl(C, D, ShAmt))
2316// -> fshl(bitwise(A, C), bitwise(B, D), ShAmt)
2317// bitwise(fshr (A, B, ShAmt), fshr(C, D, ShAmt))
2318// -> fshr(bitwise(A, C), bitwise(B, D), ShAmt)
2319// bitwise(bswap(A), bswap(B)) -> bswap(bitwise(A, B))
2320// bitwise(bswap(A), C) -> bswap(bitwise(A, bswap(C)))
2321// bitwise(bitreverse(A), bitreverse(B)) -> bitreverse(bitwise(A, B))
2322// bitwise(bitreverse(A), C) -> bitreverse(bitwise(A, bitreverse(C)))
2323static Instruction *
2325 InstCombiner::BuilderTy &Builder) {
2326 assert(I.isBitwiseLogicOp() && "Should and/or/xor");
2327 if (!I.getOperand(0)->hasOneUse())
2328 return nullptr;
2329 IntrinsicInst *X = dyn_cast<IntrinsicInst>(I.getOperand(0));
2330 if (!X)
2331 return nullptr;
2332
2333 IntrinsicInst *Y = dyn_cast<IntrinsicInst>(I.getOperand(1));
2334 if (Y && (!Y->hasOneUse() || X->getIntrinsicID() != Y->getIntrinsicID()))
2335 return nullptr;
2336
2337 Intrinsic::ID IID = X->getIntrinsicID();
2338 const APInt *RHSC;
2339 // Try to match constant RHS.
2340 if (!Y && (!(IID == Intrinsic::bswap || IID == Intrinsic::bitreverse) ||
2341 !match(I.getOperand(1), m_APInt(RHSC))))
2342 return nullptr;
2343
2344 switch (IID) {
2345 case Intrinsic::fshl:
2346 case Intrinsic::fshr: {
2347 if (X->getOperand(2) != Y->getOperand(2))
2348 return nullptr;
2349 Value *NewOp0 =
2350 Builder.CreateBinOp(I.getOpcode(), X->getOperand(0), Y->getOperand(0));
2351 Value *NewOp1 =
2352 Builder.CreateBinOp(I.getOpcode(), X->getOperand(1), Y->getOperand(1));
2353 Function *F =
2354 Intrinsic::getOrInsertDeclaration(I.getModule(), IID, I.getType());
2355 return CallInst::Create(F, {NewOp0, NewOp1, X->getOperand(2)});
2356 }
2357 case Intrinsic::bswap:
2358 case Intrinsic::bitreverse: {
2359 Value *NewOp0 = Builder.CreateBinOp(
2360 I.getOpcode(), X->getOperand(0),
2361 Y ? Y->getOperand(0)
2362 : ConstantInt::get(I.getType(), IID == Intrinsic::bswap
2363 ? RHSC->byteSwap()
2364 : RHSC->reverseBits()));
2365 Function *F =
2366 Intrinsic::getOrInsertDeclaration(I.getModule(), IID, I.getType());
2367 return CallInst::Create(F, {NewOp0});
2368 }
2369 default:
2370 return nullptr;
2371 }
2372}
2373
2374// Try to simplify V by replacing occurrences of Op with RepOp, but only look
2375// through bitwise operations. In particular, for X | Y we try to replace Y with
2376// 0 inside X and for X & Y we try to replace Y with -1 inside X.
2377// Return the simplified result of X if successful, and nullptr otherwise.
2378// If SimplifyOnly is true, no new instructions will be created.
2380 bool SimplifyOnly,
2381 InstCombinerImpl &IC,
2382 unsigned Depth = 0) {
2383 if (Op == RepOp)
2384 return nullptr;
2385
2386 if (V == Op)
2387 return RepOp;
2388
2389 auto *I = dyn_cast<BinaryOperator>(V);
2390 if (!I || !I->isBitwiseLogicOp() || Depth >= 3)
2391 return nullptr;
2392
2393 if (!I->hasOneUse())
2394 SimplifyOnly = true;
2395
2396 Value *NewOp0 = simplifyAndOrWithOpReplaced(I->getOperand(0), Op, RepOp,
2397 SimplifyOnly, IC, Depth + 1);
2398 Value *NewOp1 = simplifyAndOrWithOpReplaced(I->getOperand(1), Op, RepOp,
2399 SimplifyOnly, IC, Depth + 1);
2400 if (!NewOp0 && !NewOp1)
2401 return nullptr;
2402
2403 if (!NewOp0)
2404 NewOp0 = I->getOperand(0);
2405 if (!NewOp1)
2406 NewOp1 = I->getOperand(1);
2407
2408 if (Value *Res = simplifyBinOp(I->getOpcode(), NewOp0, NewOp1,
2410 return Res;
2411
2412 if (SimplifyOnly)
2413 return nullptr;
2414 return IC.Builder.CreateBinOp(I->getOpcode(), NewOp0, NewOp1);
2415}
2416
2417/// The pattern div_ceil(X, P) * P, where P is a power of 2, lowers to the
2418/// following conditional round-up: (X + select(C, 0, Pow2)) & -Pow2, where
2419/// C is X % Pow2 == 0. This may be simplified to (X + (Pow2-1)) & -Pow2.
2420static Instruction *
2422 InstCombiner::BuilderTy &Builder) {
2423 const APInt *NegP;
2424 Value *Add;
2425 if (!match(&I, m_And(m_Value(Add), m_NegatedPower2(NegP))))
2426 return nullptr;
2427
2428 Value *X, *Cond;
2429 APInt Mask = ~*NegP;
2430
2431 // Match the pattern. Ensure the true arm of the select is zero, and the false
2432 // one is the Pow2.
2433 if (!match(Add,
2435 m_SpecificInt(-*NegP))))))
2436 return nullptr;
2437
2438 // icmp ne should have already been canonicalized to the eq form for this
2439 // pattern.
2442 m_Zero())))
2443 return nullptr;
2444
2445 Type *Ty = I.getType();
2446 Value *NewAdd = Builder.CreateAdd(X, ConstantInt::get(Ty, Mask));
2447 return BinaryOperator::CreateAnd(NewAdd, ConstantInt::get(Ty, *NegP));
2448}
2449
2450/// Reassociate and/or expressions to see if we can fold the inner and/or ops.
2451/// TODO: Make this recursive; it's a little tricky because an arbitrary
2452/// number of and/or instructions might have to be created.
2453Value *InstCombinerImpl::reassociateBooleanAndOr(Value *LHS, Value *X, Value *Y,
2454 Instruction &I, bool IsAnd,
2455 bool RHSIsLogical) {
2456 Instruction::BinaryOps Opcode = IsAnd ? Instruction::And : Instruction::Or;
2457 Value *Folded = nullptr;
2458 // LHS bop (X lop Y) --> (LHS bop X) lop Y
2459 // LHS bop (X bop Y) --> (LHS bop X) bop Y
2460 if (Value *Res = foldBooleanAndOr(LHS, X, I, IsAnd, /*IsLogical=*/false))
2461 Folded = RHSIsLogical ? Builder.CreateLogicalOp(Opcode, Res, Y)
2462 : Builder.CreateBinOp(Opcode, Res, Y);
2463 // LHS bop (X bop Y) --> X bop (LHS bop Y)
2464 // LHS bop (X lop Y) --> X lop (LHS bop Y)
2465 else if (Value *Res = foldBooleanAndOr(LHS, Y, I, IsAnd, /*IsLogical=*/false))
2466 Folded = RHSIsLogical ? Builder.CreateLogicalOp(Opcode, X, Res)
2467 : Builder.CreateBinOp(Opcode, X, Res);
2468 if (SelectInst *SI = dyn_cast_or_null<SelectInst>(Folded); SI != nullptr)
2469 // If the bop I was originally a lop, we could recover branch weight
2470 // information using that lop's weights. However, InstCombine usually
2471 // replaces the lop with a bop by the time we get here, deleting the branch
2472 // weight information. Therefore, we can only assume unknown branch weights.
2473 // TODO: see if it's possible to recover branch weight information from the
2474 // original lop (https://github.com/llvm/llvm-project/issues/183864).
2476 I.getFunction());
2477 return Folded;
2478}
2479
2480// FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
2481// here. We should standardize that construct where it is needed or choose some
2482// other way to ensure that commutated variants of patterns are not missed.
2484 Type *Ty = I.getType();
2485
2486 if (Value *V = simplifyAndInst(I.getOperand(0), I.getOperand(1),
2487 SQ.getWithInstruction(&I)))
2488 return replaceInstUsesWith(I, V);
2489
2491 return &I;
2492
2494 return X;
2495
2497 return Phi;
2498
2499 // See if we can simplify any instructions used by the instruction whose sole
2500 // purpose is to compute bits we don't care about.
2502 return &I;
2503
2504 // Do this before using distributive laws to catch simple and/or/not patterns.
2506 return Xor;
2507
2509 return X;
2510
2511 // (A|B)&(A|C) -> A|(B&C) etc
2513 return replaceInstUsesWith(I, V);
2514
2516 return R;
2517
2518 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
2519
2520 Value *X, *Y;
2521 const APInt *C;
2522 if ((match(Op0, m_OneUse(m_LogicalShift(m_One(), m_Value(X)))) ||
2523 (match(Op0, m_OneUse(m_Shl(m_APInt(C), m_Value(X)))) && (*C)[0])) &&
2524 match(Op1, m_One())) {
2525 // (1 >> X) & 1 --> zext(X == 0)
2526 // (C << X) & 1 --> zext(X == 0), when C is odd
2527 Value *IsZero = Builder.CreateICmpEQ(X, ConstantInt::get(Ty, 0));
2528 return new ZExtInst(IsZero, Ty);
2529 }
2530
2531 // (-(X & 1)) & Y --> (X & 1) == 0 ? 0 : Y
2532 Value *Neg;
2533 if (match(&I,
2535 m_Value(Y)))) {
2536 Value *Cmp = Builder.CreateIsNull(Neg);
2537 return createSelectInstWithUnknownProfile(Cmp,
2539 }
2540
2541 // Canonicalize:
2542 // (X +/- Y) & Y --> ~X & Y when Y is a power of 2.
2545 m_Sub(m_Value(X), m_Deferred(Y)))))) &&
2546 isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, &I))
2547 return BinaryOperator::CreateAnd(Builder.CreateNot(X), Y);
2548
2549 if (match(Op1, m_APInt(C))) {
2550 const APInt *XorC;
2551 if (match(Op0, m_OneUse(m_Xor(m_Value(X), m_APInt(XorC))))) {
2552 // (X ^ C1) & C2 --> (X & C2) ^ (C1&C2)
2553 Constant *NewC = ConstantInt::get(Ty, *C & *XorC);
2554 Value *And = Builder.CreateAnd(X, Op1);
2555 And->takeName(Op0);
2556 return BinaryOperator::CreateXor(And, NewC);
2557 }
2558
2559 const APInt *OrC;
2560 if (match(Op0, m_OneUse(m_Or(m_Value(X), m_APInt(OrC))))) {
2561 // (X | C1) & C2 --> (X & C2^(C1&C2)) | (C1&C2)
2562 // NOTE: This reduces the number of bits set in the & mask, which
2563 // can expose opportunities for store narrowing for scalars.
2564 // NOTE: SimplifyDemandedBits should have already removed bits from C1
2565 // that aren't set in C2. Meaning we can replace (C1&C2) with C1 in
2566 // above, but this feels safer.
2567 APInt Together = *C & *OrC;
2568 Value *And = Builder.CreateAnd(X, ConstantInt::get(Ty, Together ^ *C));
2569 And->takeName(Op0);
2570 return BinaryOperator::CreateOr(And, ConstantInt::get(Ty, Together));
2571 }
2572
2573 unsigned Width = Ty->getScalarSizeInBits();
2574 const APInt *ShiftC;
2575 if (match(Op0, m_OneUse(m_SExt(m_AShr(m_Value(X), m_APInt(ShiftC))))) &&
2576 ShiftC->ult(Width)) {
2577 if (*C == APInt::getLowBitsSet(Width, Width - ShiftC->getZExtValue())) {
2578 // We are clearing high bits that were potentially set by sext+ashr:
2579 // and (sext (ashr X, ShiftC)), C --> lshr (sext X), ShiftC
2580 Value *Sext = Builder.CreateSExt(X, Ty);
2581 Constant *ShAmtC = ConstantInt::get(Ty, ShiftC->zext(Width));
2582 return BinaryOperator::CreateLShr(Sext, ShAmtC);
2583 }
2584 }
2585
2586 // If this 'and' clears the sign-bits added by ashr, replace with lshr:
2587 // and (ashr X, ShiftC), C --> lshr X, ShiftC
2588 if (match(Op0, m_AShr(m_Value(X), m_APInt(ShiftC))) && ShiftC->ult(Width) &&
2589 C->isMask(Width - ShiftC->getZExtValue()))
2590 return BinaryOperator::CreateLShr(X, ConstantInt::get(Ty, *ShiftC));
2591
2592 const APInt *AddC;
2593 if (match(Op0, m_Add(m_Value(X), m_APInt(AddC)))) {
2594 // If we are masking the result of the add down to exactly one bit and
2595 // the constant we are adding has no bits set below that bit, then the
2596 // add is flipping a single bit. Example:
2597 // (X + 4) & 4 --> (X & 4) ^ 4
2598 if (Op0->hasOneUse() && C->isPowerOf2() && (*AddC & (*C - 1)) == 0) {
2599 assert((*C & *AddC) != 0 && "Expected common bit");
2600 Value *NewAnd = Builder.CreateAnd(X, Op1);
2601 return BinaryOperator::CreateXor(NewAnd, Op1);
2602 }
2603 }
2604
2605 // ((C1 OP zext(X)) & C2) -> zext((C1 OP X) & C2) if C2 fits in the
2606 // bitwidth of X and OP behaves well when given trunc(C1) and X.
2607 auto isNarrowableBinOpcode = [](BinaryOperator *B) {
2608 switch (B->getOpcode()) {
2609 case Instruction::Xor:
2610 case Instruction::Or:
2611 case Instruction::Mul:
2612 case Instruction::Add:
2613 case Instruction::Sub:
2614 return true;
2615 default:
2616 return false;
2617 }
2618 };
2619 BinaryOperator *BO;
2620 if (match(Op0, m_OneUse(m_BinOp(BO))) && isNarrowableBinOpcode(BO)) {
2621 Instruction::BinaryOps BOpcode = BO->getOpcode();
2622 Value *X;
2623 const APInt *C1;
2624 // TODO: The one-use restrictions could be relaxed a little if the AND
2625 // is going to be removed.
2626 // Try to narrow the 'and' and a binop with constant operand:
2627 // and (bo (zext X), C1), C --> zext (and (bo X, TruncC1), TruncC)
2628 if (match(BO, m_c_BinOp(m_OneUse(m_ZExt(m_Value(X))), m_APInt(C1))) &&
2629 C->isIntN(X->getType()->getScalarSizeInBits())) {
2630 unsigned XWidth = X->getType()->getScalarSizeInBits();
2631 Constant *TruncC1 = ConstantInt::get(X->getType(), C1->trunc(XWidth));
2632 Value *BinOp = isa<ZExtInst>(BO->getOperand(0))
2633 ? Builder.CreateBinOp(BOpcode, X, TruncC1)
2634 : Builder.CreateBinOp(BOpcode, TruncC1, X);
2635 Constant *TruncC = ConstantInt::get(X->getType(), C->trunc(XWidth));
2636 Value *And = Builder.CreateAnd(BinOp, TruncC);
2637 return new ZExtInst(And, Ty);
2638 }
2639
2640 // Similar to above: if the mask matches the zext input width, then the
2641 // 'and' can be eliminated, so we can truncate the other variable op:
2642 // and (bo (zext X), Y), C --> zext (bo X, (trunc Y))
2643 if (isa<Instruction>(BO->getOperand(0)) &&
2644 match(BO->getOperand(0), m_OneUse(m_ZExt(m_Value(X)))) &&
2645 C->isMask(X->getType()->getScalarSizeInBits())) {
2646 Y = BO->getOperand(1);
2647 Value *TrY = Builder.CreateTrunc(Y, X->getType(), Y->getName() + ".tr");
2648 Value *NewBO =
2649 Builder.CreateBinOp(BOpcode, X, TrY, BO->getName() + ".narrow");
2650 return new ZExtInst(NewBO, Ty);
2651 }
2652 // and (bo Y, (zext X)), C --> zext (bo (trunc Y), X)
2653 if (isa<Instruction>(BO->getOperand(1)) &&
2654 match(BO->getOperand(1), m_OneUse(m_ZExt(m_Value(X)))) &&
2655 C->isMask(X->getType()->getScalarSizeInBits())) {
2656 Y = BO->getOperand(0);
2657 Value *TrY = Builder.CreateTrunc(Y, X->getType(), Y->getName() + ".tr");
2658 Value *NewBO =
2659 Builder.CreateBinOp(BOpcode, TrY, X, BO->getName() + ".narrow");
2660 return new ZExtInst(NewBO, Ty);
2661 }
2662 }
2663
2664 // This is intentionally placed after the narrowing transforms for
2665 // efficiency (transform directly to the narrow logic op if possible).
2666 // If the mask is only needed on one incoming arm, push the 'and' op up.
2667 if (match(Op0, m_OneUse(m_Xor(m_Value(X), m_Value(Y)))) ||
2668 match(Op0, m_OneUse(m_Or(m_Value(X), m_Value(Y))))) {
2669 APInt NotAndMask(~(*C));
2670 BinaryOperator::BinaryOps BinOp = cast<BinaryOperator>(Op0)->getOpcode();
2671 if (MaskedValueIsZero(X, NotAndMask, &I)) {
2672 // Not masking anything out for the LHS, move mask to RHS.
2673 // and ({x}or X, Y), C --> {x}or X, (and Y, C)
2674 Value *NewRHS = Builder.CreateAnd(Y, Op1, Y->getName() + ".masked");
2675 return BinaryOperator::Create(BinOp, X, NewRHS);
2676 }
2677 if (!isa<Constant>(Y) && MaskedValueIsZero(Y, NotAndMask, &I)) {
2678 // Not masking anything out for the RHS, move mask to LHS.
2679 // and ({x}or X, Y), C --> {x}or (and X, C), Y
2680 Value *NewLHS = Builder.CreateAnd(X, Op1, X->getName() + ".masked");
2681 return BinaryOperator::Create(BinOp, NewLHS, Y);
2682 }
2683 }
2684
2685 // When the mask is a power-of-2 constant and op0 is a shifted-power-of-2
2686 // constant, test if the shift amount equals the offset bit index:
2687 // (ShiftC << X) & C --> X == (log2(C) - log2(ShiftC)) ? C : 0
2688 // (ShiftC >> X) & C --> X == (log2(ShiftC) - log2(C)) ? C : 0
2689 if (C->isPowerOf2() &&
2690 match(Op0, m_OneUse(m_LogicalShift(m_Power2(ShiftC), m_Value(X))))) {
2691 int Log2ShiftC = ShiftC->exactLogBase2();
2692 int Log2C = C->exactLogBase2();
2693 bool IsShiftLeft =
2694 cast<BinaryOperator>(Op0)->getOpcode() == Instruction::Shl;
2695 int BitNum = IsShiftLeft ? Log2C - Log2ShiftC : Log2ShiftC - Log2C;
2696 assert(BitNum >= 0 && "Expected demanded bits to handle impossible mask");
2697 Value *Cmp = Builder.CreateICmpEQ(X, ConstantInt::get(Ty, BitNum));
2698 return createSelectInstWithUnknownProfile(Cmp, ConstantInt::get(Ty, *C),
2700 }
2701
2702 Constant *C1, *C2;
2703 const APInt *C3 = C;
2704 Value *X;
2705 if (C3->isPowerOf2()) {
2706 Constant *Log2C3 = ConstantInt::get(Ty, C3->countr_zero());
2708 m_ImmConstant(C2)))) &&
2709 match(C1, m_Power2())) {
2711 Constant *LshrC = ConstantExpr::getAdd(C2, Log2C3);
2712 KnownBits KnownLShrc = computeKnownBits(LshrC, nullptr);
2713 if (KnownLShrc.getMaxValue().ult(Width)) {
2714 // iff C1,C3 is pow2 and C2 + cttz(C3) < BitWidth:
2715 // ((C1 << X) >> C2) & C3 -> X == (cttz(C3)+C2-cttz(C1)) ? C3 : 0
2716 Constant *CmpC = ConstantExpr::getSub(LshrC, Log2C1);
2717 Value *Cmp = Builder.CreateICmpEQ(X, CmpC);
2718 return createSelectInstWithUnknownProfile(
2719 Cmp, ConstantInt::get(Ty, *C3), ConstantInt::getNullValue(Ty));
2720 }
2721 }
2722
2724 m_ImmConstant(C2)))) &&
2725 match(C1, m_Power2())) {
2727 Constant *Cmp =
2729 if (Cmp && Cmp->isNullValue()) {
2730 // iff C1,C3 is pow2 and Log2(C3) >= C2:
2731 // ((C1 >> X) << C2) & C3 -> X == (cttz(C1)+C2-cttz(C3)) ? C3 : 0
2732 Constant *ShlC = ConstantExpr::getAdd(C2, Log2C1);
2733 Constant *CmpC = ConstantExpr::getSub(ShlC, Log2C3);
2734 Value *Cmp = Builder.CreateICmpEQ(X, CmpC);
2735 return createSelectInstWithUnknownProfile(
2736 Cmp, ConstantInt::get(Ty, *C3), ConstantInt::getNullValue(Ty));
2737 }
2738 }
2739 }
2740
2741 // and (or (shr (sub C, X), ShAmt), X), C --> umin(X, C)
2742 // The fold holds when all of:
2743 // - C is a low-bit mask: C = (1<<S)-1
2744 // - S u<= ShAmt -- When X u<= C, sub(C, X) u< 1<<ShAmt, so shr is 0
2745 // - ShAmt + S u<= BW -- the borrow's all-ones run reaches up to bit S-1
2746 // - X u< 1<<ShAmt -- bounds X - C u< 1<<ShAmt in the X u> C case
2747 // Then shr(sub(C, X), ShAmt) is 0 when X u<= C and has all of C's bits set
2748 // when X u> C, so (shr | X) & C clamps X to C. For ashr, the bound on X
2749 // also means sub(C, X) is negative when X u> C, so shr is all-ones and
2750 // ShAmt + S u<= BW is not needed.
2751 const APInt *ShAmtC;
2752 Instruction *Shr;
2753 if (C->isMask() &&
2754 match(Op0, m_OneUse(m_c_Or(
2757 m_APInt(ShAmtC))),
2758 m_Deferred(X)))) &&
2759 ShAmtC->ult(Width)) {
2760 unsigned S = C->popcount();
2761 unsigned ShAmt = ShAmtC->getZExtValue();
2762 bool IsAShr = Shr->getOpcode() == Instruction::AShr;
2763 if (S <= ShAmt && (IsAShr || ShAmt + S <= Width) &&
2764 MaskedValueIsZero(X, APInt::getBitsSetFrom(Width, ShAmt), &I))
2765 return replaceInstUsesWith(
2766 I, Builder.CreateBinaryIntrinsic(Intrinsic::umin, X, Op1));
2767 }
2768 }
2769
2770 // If we are clearing the sign bit of a floating-point value, convert this to
2771 // fabs, then cast back to integer.
2772 //
2773 // This is a generous interpretation for noimplicitfloat, this is not a true
2774 // floating-point operation.
2775 //
2776 // Assumes any IEEE-represented type has the sign bit in the high bit.
2777 // TODO: Unify with APInt matcher. This version allows undef unlike m_APInt
2778 Value *CastOp;
2779 if (match(Op0, m_ElementWiseBitCast(m_Value(CastOp))) &&
2780 match(Op1, m_MaxSignedValue()) &&
2781 !Builder.GetInsertBlock()->getParent()->hasFnAttribute(
2782 Attribute::NoImplicitFloat)) {
2783 Type *EltTy = CastOp->getType()->getScalarType();
2784 if (EltTy->isFloatingPointTy() &&
2786 Value *FAbs = Builder.CreateFAbs(CastOp);
2787 return new BitCastInst(FAbs, I.getType());
2788 }
2789 }
2790
2791 // and(shl(zext(X), Y), SignMask) -> and(sext(X), SignMask)
2792 // where Y is a valid shift amount.
2794 m_SignMask())) &&
2797 APInt(Ty->getScalarSizeInBits(),
2798 Ty->getScalarSizeInBits() -
2799 X->getType()->getScalarSizeInBits())))) {
2800 auto *SExt = Builder.CreateSExt(X, Ty, X->getName() + ".signext");
2801 return BinaryOperator::CreateAnd(SExt, Op1);
2802 }
2803
2804 if (Instruction *Z = narrowMaskedBinOp(I))
2805 return Z;
2806
2807 if (I.getType()->isIntOrIntVectorTy(1)) {
2808 if (auto *SI0 = dyn_cast<SelectInst>(Op0)) {
2809 if (auto *R =
2810 foldAndOrOfSelectUsingImpliedCond(Op1, *SI0, /* IsAnd */ true))
2811 return R;
2812 }
2813 if (auto *SI1 = dyn_cast<SelectInst>(Op1)) {
2814 if (auto *R =
2815 foldAndOrOfSelectUsingImpliedCond(Op0, *SI1, /* IsAnd */ true))
2816 return R;
2817 }
2818 }
2819
2820 if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I))
2821 return FoldedLogic;
2822
2823 if (Instruction *DeMorgan = matchDeMorgansLaws(I, *this))
2824 return DeMorgan;
2825
2826 {
2827 Value *A, *B, *C;
2828 // A & ~(A ^ B) --> A & B
2829 if (match(Op1, m_Not(m_c_Xor(m_Specific(Op0), m_Value(B)))))
2830 return BinaryOperator::CreateAnd(Op0, B);
2831 // ~(A ^ B) & A --> A & B
2832 if (match(Op0, m_Not(m_c_Xor(m_Specific(Op1), m_Value(B)))))
2833 return BinaryOperator::CreateAnd(Op1, B);
2834
2835 // (A ^ B) & ((B ^ C) ^ A) -> (A ^ B) & ~C
2836 if (match(Op0, m_Xor(m_Value(A), m_Value(B))) &&
2837 match(Op1, m_Xor(m_Xor(m_Specific(B), m_Value(C)), m_Specific(A)))) {
2838 Value *NotC = Op1->hasOneUse()
2839 ? Builder.CreateNot(C)
2840 : getFreelyInverted(C, C->hasOneUse(), &Builder);
2841 if (NotC != nullptr)
2842 return BinaryOperator::CreateAnd(Op0, NotC);
2843 }
2844
2845 // ((A ^ C) ^ B) & (B ^ A) -> (B ^ A) & ~C
2846 if (match(Op0, m_Xor(m_Xor(m_Value(A), m_Value(C)), m_Value(B))) &&
2847 match(Op1, m_Xor(m_Specific(B), m_Specific(A)))) {
2848 Value *NotC = Op0->hasOneUse()
2849 ? Builder.CreateNot(C)
2850 : getFreelyInverted(C, C->hasOneUse(), &Builder);
2851 if (NotC != nullptr)
2852 return BinaryOperator::CreateAnd(Op1, NotC);
2853 }
2854
2855 // (A | B) & (~A ^ B) -> A & B
2856 // (A | B) & (B ^ ~A) -> A & B
2857 // (B | A) & (~A ^ B) -> A & B
2858 // (B | A) & (B ^ ~A) -> A & B
2859 if (match(Op1, m_c_Xor(m_Not(m_Value(A)), m_Value(B))) &&
2860 match(Op0, m_c_Or(m_Specific(A), m_Specific(B))))
2861 return BinaryOperator::CreateAnd(A, B);
2862
2863 // (~A ^ B) & (A | B) -> A & B
2864 // (~A ^ B) & (B | A) -> A & B
2865 // (B ^ ~A) & (A | B) -> A & B
2866 // (B ^ ~A) & (B | A) -> A & B
2867 if (match(Op0, m_c_Xor(m_Not(m_Value(A)), m_Value(B))) &&
2868 match(Op1, m_c_Or(m_Specific(A), m_Specific(B))))
2869 return BinaryOperator::CreateAnd(A, B);
2870
2871 // (~A | B) & (A ^ B) -> ~A & B
2872 // (~A | B) & (B ^ A) -> ~A & B
2873 // (B | ~A) & (A ^ B) -> ~A & B
2874 // (B | ~A) & (B ^ A) -> ~A & B
2875 if (match(Op0, m_c_Or(m_Not(m_Value(A)), m_Value(B))) &&
2877 return BinaryOperator::CreateAnd(Builder.CreateNot(A), B);
2878
2879 // (A ^ B) & (~A | B) -> ~A & B
2880 // (B ^ A) & (~A | B) -> ~A & B
2881 // (A ^ B) & (B | ~A) -> ~A & B
2882 // (B ^ A) & (B | ~A) -> ~A & B
2883 if (match(Op1, m_c_Or(m_Not(m_Value(A)), m_Value(B))) &&
2885 return BinaryOperator::CreateAnd(Builder.CreateNot(A), B);
2886 }
2887
2888 if (Value *Res =
2889 foldBooleanAndOr(Op0, Op1, I, /*IsAnd=*/true, /*IsLogical=*/false))
2890 return replaceInstUsesWith(I, Res);
2891
2892 if (match(Op1, m_OneUse(m_LogicalAnd(m_Value(X), m_Value(Y))))) {
2893 bool IsLogical = isa<SelectInst>(Op1);
2894 if (auto *V = reassociateBooleanAndOr(Op0, X, Y, I, /*IsAnd=*/true,
2895 /*RHSIsLogical=*/IsLogical))
2896 return replaceInstUsesWith(I, V);
2897 }
2898 if (match(Op0, m_OneUse(m_LogicalAnd(m_Value(X), m_Value(Y))))) {
2899 bool IsLogical = isa<SelectInst>(Op0);
2900 if (auto *V = reassociateBooleanAndOr(Op1, X, Y, I, /*IsAnd=*/true,
2901 /*RHSIsLogical=*/IsLogical))
2902 return replaceInstUsesWith(I, V);
2903 }
2904
2905 if (Instruction *FoldedFCmps = reassociateFCmps(I, Builder))
2906 return FoldedFCmps;
2907
2908 if (Instruction *CastedAnd = foldCastedBitwiseLogic(I))
2909 return CastedAnd;
2910
2911 if (Instruction *Sel = foldBinopOfSextBoolToSelect(I))
2912 return Sel;
2913
2914 // and(sext(A), B) / and(B, sext(A)) --> A ? B : 0, where A is i1 or <N x i1>.
2915 // TODO: Move this into foldBinopOfSextBoolToSelect as a more generalized fold
2916 // with binop identity constant. But creating a select with non-constant
2917 // arm may not be reversible due to poison semantics. Is that a good
2918 // canonicalization?
2919 Value *A, *B;
2920 if (match(&I, m_c_And(m_SExt(m_Value(A)), m_Value(B))) &&
2921 A->getType()->isIntOrIntVectorTy(1))
2922 return createSelectInstWithUnknownProfile(A, B, Constant::getNullValue(Ty));
2923
2924 // Similarly, a 'not' of the bool translates to a swap of the select arms:
2925 // ~sext(A) & B / B & ~sext(A) --> A ? 0 : B
2926 if (match(&I, m_c_And(m_Not(m_SExt(m_Value(A))), m_Value(B))) &&
2927 A->getType()->isIntOrIntVectorTy(1))
2928 return createSelectInstWithUnknownProfile(A, Constant::getNullValue(Ty), B);
2929
2930 // and(zext(A), B) -> A ? (B & 1) : 0
2931 if (match(&I, m_c_And(m_OneUse(m_ZExt(m_Value(A))), m_Value(B))) &&
2932 A->getType()->isIntOrIntVectorTy(1))
2933 return createSelectInstWithUnknownProfile(
2934 A, Builder.CreateAnd(B, ConstantInt::get(Ty, 1)),
2936
2937 // (-1 + A) & B --> A ? 0 : B where A is 0/1.
2939 m_Value(B)))) {
2940 if (A->getType()->isIntOrIntVectorTy(1))
2941 return createSelectInstWithUnknownProfile(A, Constant::getNullValue(Ty),
2942 B);
2943 if (computeKnownBits(A, &I).countMaxActiveBits() <= 1) {
2944 return createSelectInstWithUnknownProfile(
2945 Builder.CreateICmpEQ(A, Constant::getNullValue(A->getType())), B,
2947 }
2948 }
2949
2950 // (iN X s>> (N-1)) & Y --> (X s< 0) ? Y : 0 -- with optional sext
2953 m_Value(Y))) &&
2954 *C == X->getType()->getScalarSizeInBits() - 1) {
2955 Value *IsNeg = Builder.CreateIsNeg(X, "isneg");
2956 return createSelectInstWithUnknownProfile(IsNeg, Y,
2958 }
2959 // If there's a 'not' of the shifted value, swap the select operands:
2960 // ~(iN X s>> (N-1)) & Y --> (X s< 0) ? 0 : Y -- with optional sext
2963 m_Value(Y))) &&
2964 *C == X->getType()->getScalarSizeInBits() - 1) {
2965 Value *IsNeg = Builder.CreateIsNeg(X, "isneg");
2966 return createSelectInstWithUnknownProfile(IsNeg,
2968 }
2969
2970 // (~x) & y --> ~(x | (~y)) iff that gets rid of inversions
2972 return &I;
2973
2974 // An and recurrence w/loop invariant step is equivelent to (and start, step)
2975 PHINode *PN = nullptr;
2976 Value *Start = nullptr, *Step = nullptr;
2977 if (matchSimpleRecurrence(&I, PN, Start, Step) && DT.dominates(Step, PN))
2978 return replaceInstUsesWith(I, Builder.CreateAnd(Start, Step));
2979
2981 return R;
2982
2983 if (Instruction *Canonicalized = canonicalizeLogicFirst(I, Builder))
2984 return Canonicalized;
2985
2986 if (Instruction *Folded = foldLogicOfIsFPClass(I, Op0, Op1))
2987 return Folded;
2988
2989 if (Instruction *Res = foldBinOpOfDisplacedShifts(I))
2990 return Res;
2991
2993 return Res;
2994
2995 if (Value *V =
2997 /*SimplifyOnly*/ false, *this))
2998 return BinaryOperator::CreateAnd(V, Op1);
2999 if (Value *V =
3001 /*SimplifyOnly*/ false, *this))
3002 return BinaryOperator::CreateAnd(Op0, V);
3003
3005 return Res;
3006
3007 return nullptr;
3008}
3009
3011 bool MatchBSwaps,
3012 bool MatchBitReversals) {
3014 if (!recognizeBSwapOrBitReverseIdiom(&I, MatchBSwaps, MatchBitReversals,
3015 Insts))
3016 return nullptr;
3017 Instruction *LastInst = Insts.pop_back_val();
3018 LastInst->removeFromParent();
3019
3020 for (auto *Inst : Insts) {
3021 Inst->setDebugLoc(I.getDebugLoc());
3022 Worklist.push(Inst);
3023 }
3024 return LastInst;
3025}
3026
3027std::optional<std::pair<Intrinsic::ID, SmallVector<Value *, 3>>>
3029 // TODO: Can we reduce the code duplication between this and the related
3030 // rotate matching code under visitSelect and visitTrunc?
3031 assert(Or.getOpcode() == BinaryOperator::Or && "Expecting or instruction");
3032
3033 unsigned Width = Or.getType()->getScalarSizeInBits();
3034
3035 Instruction *Or0, *Or1;
3036 if (!match(Or.getOperand(0), m_Instruction(Or0)) ||
3037 !match(Or.getOperand(1), m_Instruction(Or1)))
3038 return std::nullopt;
3039
3040 bool IsFshl = true; // Sub on LSHR.
3041 SmallVector<Value *, 3> FShiftArgs;
3042
3043 // First, find an or'd pair of opposite shifts:
3044 // or (lshr ShVal0, ShAmt0), (shl ShVal1, ShAmt1)
3045 if (isa<BinaryOperator>(Or0) && isa<BinaryOperator>(Or1)) {
3046 Value *ShVal0, *ShVal1, *ShAmt0, *ShAmt1;
3047 if (!match(Or0,
3048 m_OneUse(m_LogicalShift(m_Value(ShVal0), m_Value(ShAmt0)))) ||
3049 !match(Or1,
3050 m_OneUse(m_LogicalShift(m_Value(ShVal1), m_Value(ShAmt1)))) ||
3051 Or0->getOpcode() == Or1->getOpcode())
3052 return std::nullopt;
3053
3054 // Canonicalize to or(shl(ShVal0, ShAmt0), lshr(ShVal1, ShAmt1)).
3055 if (Or0->getOpcode() == BinaryOperator::LShr) {
3056 std::swap(Or0, Or1);
3057 std::swap(ShVal0, ShVal1);
3058 std::swap(ShAmt0, ShAmt1);
3059 }
3060 assert(Or0->getOpcode() == BinaryOperator::Shl &&
3061 Or1->getOpcode() == BinaryOperator::LShr &&
3062 "Illegal or(shift,shift) pair");
3063
3064 // Match the shift amount operands for a funnel shift pattern. This always
3065 // matches a subtraction on the R operand.
3066 auto matchShiftAmount = [&](Value *L, Value *R, unsigned Width) -> Value * {
3067 // Check for constant shift amounts that sum to the bitwidth.
3068 const APInt *LI, *RI;
3069 if (match(L, m_APIntAllowPoison(LI)) && match(R, m_APIntAllowPoison(RI)))
3070 if (LI->ult(Width) && RI->ult(Width) && (*LI + *RI) == Width)
3071 return ConstantInt::get(L->getType(), *LI);
3072
3073 Constant *LC, *RC;
3074 if (match(L, m_Constant(LC)) && match(R, m_Constant(RC)) &&
3075 match(L,
3076 m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, APInt(Width, Width))) &&
3077 match(R,
3078 m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, APInt(Width, Width))) &&
3080 return ConstantExpr::mergeUndefsWith(LC, RC);
3081
3082 // (shl ShVal, X) | (lshr ShVal, (Width - x)) iff X < Width.
3083 // We limit this to X < Width in case the backend re-expands the
3084 // intrinsic, and has to reintroduce a shift modulo operation (InstCombine
3085 // might remove it after this fold). This still doesn't guarantee that the
3086 // final codegen will match this original pattern.
3087 if (match(R, m_OneUse(m_Sub(m_SpecificInt(Width), m_Specific(L))))) {
3088 KnownBits KnownL = computeKnownBits(L, &Or);
3089 return KnownL.getMaxValue().ult(Width) ? L : nullptr;
3090 }
3091
3092 // For non-constant cases, the following patterns currently only work for
3093 // rotation patterns.
3094 // TODO: Add general funnel-shift compatible patterns.
3095 if (ShVal0 != ShVal1)
3096 return nullptr;
3097
3098 // For non-constant cases we don't support non-pow2 shift masks.
3099 // TODO: Is it worth matching urem as well?
3100 if (!isPowerOf2_32(Width))
3101 return nullptr;
3102
3103 // The shift amount may be masked with negation:
3104 // (shl ShVal, (X & (Width - 1))) | (lshr ShVal, ((-X) & (Width - 1)))
3105 Value *X;
3106 unsigned Mask = Width - 1;
3107 if (match(L, m_And(m_Value(X), m_SpecificInt(Mask))) &&
3108 match(R, m_And(m_Neg(m_Specific(X)), m_SpecificInt(Mask))))
3109 return X;
3110
3111 // (shl ShVal,(X+1) & (Width-1)) | (lshr ShVal,((X & (Width-1)) ^
3112 // (Width-1)))
3113 {
3114 Value *XPlusOne = nullptr;
3115 if (match(L, m_And(m_Value(XPlusOne, m_Add(m_Value(X), m_One())),
3116 m_SpecificInt(Mask))) &&
3118 m_SpecificInt(Mask))))
3119 return XPlusOne;
3120 }
3121
3122 // (shl ShVal, X) | (lshr ShVal, ((-X) & (Width - 1)))
3123 if (match(R, m_And(m_Neg(m_Specific(L)), m_SpecificInt(Mask))))
3124 return L;
3125
3126 // Similar to above, but the shift amount may be extended after masking,
3127 // so return the extended value as the parameter for the intrinsic.
3128 if (match(L, m_ZExt(m_And(m_Value(X), m_SpecificInt(Mask)))) &&
3129 match(R,
3131 m_SpecificInt(Mask))))
3132 return L;
3133
3134 if (match(L, m_ZExt(m_And(m_Value(X), m_SpecificInt(Mask)))) &&
3136 return L;
3137
3138 return nullptr;
3139 };
3140
3141 Value *ShAmt = matchShiftAmount(ShAmt0, ShAmt1, Width);
3142 if (!ShAmt) {
3143 ShAmt = matchShiftAmount(ShAmt1, ShAmt0, Width);
3144 IsFshl = false; // Sub on SHL.
3145 }
3146 if (!ShAmt)
3147 return std::nullopt;
3148
3149 FShiftArgs = {ShVal0, ShVal1, ShAmt};
3150 } else if (isa<ZExtInst>(Or0) || isa<ZExtInst>(Or1)) {
3151 // If there are two 'or' instructions concat variables in opposite order:
3152 //
3153 // Slot1 and Slot2 are all zero bits.
3154 // | Slot1 | Low | Slot2 | High |
3155 // LowHigh = or (shl (zext Low), ZextLowShlAmt), (zext High)
3156 // | Slot2 | High | Slot1 | Low |
3157 // HighLow = or (shl (zext High), ZextHighShlAmt), (zext Low)
3158 //
3159 // the latter 'or' can be safely convert to
3160 // -> HighLow = fshl LowHigh, LowHigh, ZextHighShlAmt
3161 // if ZextLowShlAmt + ZextHighShlAmt == Width.
3162 if (!isa<ZExtInst>(Or1))
3163 std::swap(Or0, Or1);
3164
3165 Value *High, *ZextHigh, *Low;
3166 const APInt *ZextHighShlAmt;
3167 if (!match(Or0,
3168 m_OneUse(m_Shl(m_Value(ZextHigh), m_APInt(ZextHighShlAmt)))))
3169 return std::nullopt;
3170
3171 if (!match(Or1, m_ZExt(m_Value(Low))) ||
3172 !match(ZextHigh, m_ZExt(m_Value(High))))
3173 return std::nullopt;
3174
3175 unsigned HighSize = High->getType()->getScalarSizeInBits();
3176 unsigned LowSize = Low->getType()->getScalarSizeInBits();
3177 // Make sure High does not overlap with Low and most significant bits of
3178 // High aren't shifted out.
3179 if (ZextHighShlAmt->ult(LowSize) || ZextHighShlAmt->ugt(Width - HighSize))
3180 return std::nullopt;
3181
3182 for (User *U : ZextHigh->users()) {
3183 Value *X, *Y;
3184 if (!match(U, m_Or(m_Value(X), m_Value(Y))))
3185 continue;
3186
3187 if (!isa<ZExtInst>(Y))
3188 std::swap(X, Y);
3189
3190 const APInt *ZextLowShlAmt;
3191 if (!match(X, m_Shl(m_Specific(Or1), m_APInt(ZextLowShlAmt))) ||
3192 !match(Y, m_Specific(ZextHigh)) || !DT.dominates(U, &Or))
3193 continue;
3194
3195 // HighLow is good concat. If sum of two shifts amount equals to Width,
3196 // LowHigh must also be a good concat.
3197 if (*ZextLowShlAmt + *ZextHighShlAmt != Width)
3198 continue;
3199
3200 // Low must not overlap with High and most significant bits of Low must
3201 // not be shifted out.
3202 assert(ZextLowShlAmt->uge(HighSize) &&
3203 ZextLowShlAmt->ule(Width - LowSize) && "Invalid concat");
3204
3205 // We cannot reuse the result if it may produce poison.
3206 // Drop poison generating flags in the expression tree.
3207 // Or
3208 cast<Instruction>(U)->dropPoisonGeneratingFlags();
3209 // Shl
3210 cast<Instruction>(X)->dropPoisonGeneratingFlags();
3211
3212 FShiftArgs = {U, U, ConstantInt::get(Or0->getType(), *ZextHighShlAmt)};
3213 break;
3214 }
3215 }
3216
3217 if (FShiftArgs.empty())
3218 return std::nullopt;
3219
3220 Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
3221 return std::make_pair(IID, FShiftArgs);
3222}
3223
3224/// Match UB-safe variants of the funnel shift intrinsic.
3226 if (auto Opt = IC.convertOrOfShiftsToFunnelShift(Or)) {
3227 auto [IID, FShiftArgs] = *Opt;
3228 Function *F =
3229 Intrinsic::getOrInsertDeclaration(Or.getModule(), IID, Or.getType());
3230 return CallInst::Create(F, FShiftArgs);
3231 }
3232
3233 return nullptr;
3234}
3235
3236/// Attempt to combine or(zext(x),shl(zext(y),bw/2) concat packing patterns.
3238 assert(Or.getOpcode() == Instruction::Or && "bswap requires an 'or'");
3239 Value *Op0 = Or.getOperand(0), *Op1 = Or.getOperand(1);
3240 Type *Ty = Or.getType();
3241
3242 unsigned Width = Ty->getScalarSizeInBits();
3243 if ((Width & 1) != 0)
3244 return nullptr;
3245 unsigned HalfWidth = Width / 2;
3246
3247 // Canonicalize zext (lower half) to LHS.
3248 if (!isa<ZExtInst>(Op0))
3249 std::swap(Op0, Op1);
3250
3251 // Find lower/upper half.
3252 Value *LowerSrc, *ShlVal, *UpperSrc;
3253 const APInt *C;
3254 if (!match(Op0, m_OneUse(m_ZExt(m_Value(LowerSrc)))) ||
3255 !match(Op1, m_OneUse(m_Shl(m_Value(ShlVal), m_APInt(C)))) ||
3256 !match(ShlVal, m_OneUse(m_ZExt(m_Value(UpperSrc)))))
3257 return nullptr;
3258 if (*C != HalfWidth || LowerSrc->getType() != UpperSrc->getType() ||
3259 LowerSrc->getType()->getScalarSizeInBits() != HalfWidth)
3260 return nullptr;
3261
3262 auto ConcatIntrinsicCalls = [&](Intrinsic::ID id, Value *Lo, Value *Hi) {
3263 Value *NewLower = Builder.CreateZExt(Lo, Ty);
3264 Value *NewUpper = Builder.CreateZExt(Hi, Ty);
3265 NewUpper = Builder.CreateShl(NewUpper, HalfWidth);
3266 Value *BinOp = Builder.CreateDisjointOr(NewLower, NewUpper);
3267 return Builder.CreateIntrinsic(id, Ty, BinOp);
3268 };
3269
3270 // BSWAP: Push the concat down, swapping the lower/upper sources.
3271 // concat(bswap(x),bswap(y)) -> bswap(concat(x,y))
3272 Value *LowerBSwap, *UpperBSwap;
3273 if (match(LowerSrc, m_BSwap(m_Value(LowerBSwap))) &&
3274 match(UpperSrc, m_BSwap(m_Value(UpperBSwap))))
3275 return ConcatIntrinsicCalls(Intrinsic::bswap, UpperBSwap, LowerBSwap);
3276
3277 // BITREVERSE: Push the concat down, swapping the lower/upper sources.
3278 // concat(bitreverse(x),bitreverse(y)) -> bitreverse(concat(x,y))
3279 Value *LowerBRev, *UpperBRev;
3280 if (match(LowerSrc, m_BitReverse(m_Value(LowerBRev))) &&
3281 match(UpperSrc, m_BitReverse(m_Value(UpperBRev))))
3282 return ConcatIntrinsicCalls(Intrinsic::bitreverse, UpperBRev, LowerBRev);
3283
3284 // iX ext split: extending or(zext(x),shl(zext(y),bw/2) pattern
3285 // to consume sext/ashr:
3286 // or(zext(sext(x)),shl(zext(sext(ashr(x,xbw-1))),bw/2)
3287 // or(zext(x),shl(zext(ashr(x,xbw-1)),bw/2)
3288 Value *X;
3289 if (match(LowerSrc, m_SExtOrSelf(m_Value(X))) &&
3290 match(UpperSrc,
3292 m_Specific(X),
3293 m_SpecificInt(X->getType()->getScalarSizeInBits() - 1)))))
3294 return Builder.CreateSExt(X, Ty);
3295
3296 return nullptr;
3297}
3298
3299/// If all elements of two constant vectors are 0/-1 and inverses, return true.
3301 unsigned NumElts = cast<FixedVectorType>(C1->getType())->getNumElements();
3302 for (unsigned i = 0; i != NumElts; ++i) {
3303 Constant *EltC1 = C1->getAggregateElement(i);
3304 Constant *EltC2 = C2->getAggregateElement(i);
3305 if (!EltC1 || !EltC2)
3306 return false;
3307
3308 // One element must be all ones, and the other must be all zeros.
3309 if (!((match(EltC1, m_Zero()) && match(EltC2, m_AllOnes())) ||
3310 (match(EltC2, m_Zero()) && match(EltC1, m_AllOnes()))))
3311 return false;
3312 }
3313 return true;
3314}
3315
3316/// We have an expression of the form (A & C) | (B & D). If A is a scalar or
3317/// vector composed of all-zeros or all-ones values and is the bitwise 'not' of
3318/// B, it can be used as the condition operand of a select instruction.
3319/// We will detect (A & C) | ~(B | D) when the flag ABIsTheSame enabled.
3320Value *InstCombinerImpl::getSelectCondition(Value *A, Value *B,
3321 bool ABIsTheSame) {
3322 // We may have peeked through bitcasts in the caller.
3323 // Exit immediately if we don't have (vector) integer types.
3324 Type *Ty = A->getType();
3325 if (!Ty->isIntOrIntVectorTy() || !B->getType()->isIntOrIntVectorTy())
3326 return nullptr;
3327
3328 // If A is the 'not' operand of B and has enough signbits, we have our answer.
3329 if (ABIsTheSame ? (A == B) : match(B, m_Not(m_Specific(A)))) {
3330 // If these are scalars or vectors of i1, A can be used directly.
3331 if (Ty->isIntOrIntVectorTy(1))
3332 return A;
3333
3334 // If we look through a vector bitcast, the caller will bitcast the operands
3335 // to match the condition's number of bits (N x i1).
3336 // To make this poison-safe, disallow bitcast from wide element to narrow
3337 // element. That could allow poison in lanes where it was not present in the
3338 // original code.
3340 if (A->getType()->isIntOrIntVectorTy()) {
3341 unsigned NumSignBits = ComputeNumSignBits(A);
3342 if (NumSignBits == A->getType()->getScalarSizeInBits() &&
3343 NumSignBits <= Ty->getScalarSizeInBits())
3344 return Builder.CreateTrunc(A, CmpInst::makeCmpResultType(A->getType()));
3345 }
3346 return nullptr;
3347 }
3348
3349 // TODO: add support for sext and constant case
3350 if (ABIsTheSame)
3351 return nullptr;
3352
3353 // If both operands are constants, see if the constants are inverse bitmasks.
3354 Constant *AConst, *BConst;
3355 if (match(A, m_Constant(AConst)) && match(B, m_Constant(BConst)))
3356 if (AConst == ConstantExpr::getNot(BConst) &&
3358 return Builder.CreateZExtOrTrunc(A, CmpInst::makeCmpResultType(Ty));
3359
3360 // Look for more complex patterns. The 'not' op may be hidden behind various
3361 // casts. Look through sexts and bitcasts to find the booleans.
3362 Value *Cond;
3363 Value *NotB;
3364 if (match(A, m_SExt(m_Value(Cond))) &&
3365 Cond->getType()->isIntOrIntVectorTy(1)) {
3366 // A = sext i1 Cond; B = sext (not (i1 Cond))
3367 if (match(B, m_SExt(m_Not(m_Specific(Cond)))))
3368 return Cond;
3369
3370 // A = sext i1 Cond; B = not ({bitcast} (sext (i1 Cond)))
3371 // TODO: The one-use checks are unnecessary or misplaced. If the caller
3372 // checked for uses on logic ops/casts, that should be enough to
3373 // make this transform worthwhile.
3374 if (match(B, m_OneUse(m_Not(m_Value(NotB))))) {
3375 NotB = peekThroughBitcast(NotB, true);
3376 if (match(NotB, m_SExt(m_Specific(Cond))))
3377 return Cond;
3378 }
3379 }
3380
3381 // All scalar (and most vector) possibilities should be handled now.
3382 // Try more matches that only apply to non-splat constant vectors.
3383 if (!Ty->isVectorTy())
3384 return nullptr;
3385
3386 // If both operands are xor'd with constants using the same sexted boolean
3387 // operand, see if the constants are inverse bitmasks.
3388 // TODO: Use ConstantExpr::getNot()?
3389 if (match(A, (m_Xor(m_SExt(m_Value(Cond)), m_Constant(AConst)))) &&
3390 match(B, (m_Xor(m_SExt(m_Specific(Cond)), m_Constant(BConst)))) &&
3391 Cond->getType()->isIntOrIntVectorTy(1) &&
3392 areInverseVectorBitmasks(AConst, BConst)) {
3394 return Builder.CreateXor(Cond, AConst);
3395 }
3396 return nullptr;
3397}
3398
3399/// We have an expression of the form (A & B) | (C & D). Try to simplify this
3400/// to "A' ? B : D", where A' is a boolean or vector of booleans.
3401/// When InvertFalseVal is set to true, we try to match the pattern
3402/// where we have peeked through a 'not' op and A and C are the same:
3403/// (A & B) | ~(A | D) --> (A & B) | (~A & ~D) --> A' ? B : ~D
3404Value *InstCombinerImpl::matchSelectFromAndOr(Value *A, Value *B, Value *C,
3405 Value *D, bool InvertFalseVal) {
3406 // The potential condition of the select may be bitcasted. In that case, look
3407 // through its bitcast and the corresponding bitcast of the 'not' condition.
3408 Type *OrigType = A->getType();
3409 A = peekThroughBitcast(A, true);
3410 C = peekThroughBitcast(C, true);
3411 if (Value *Cond = getSelectCondition(A, C, InvertFalseVal)) {
3412 // ((bc Cond) & B) | ((bc ~Cond) & D) --> bc (select Cond, (bc B), (bc D))
3413 // If this is a vector, we may need to cast to match the condition's length.
3414 // The bitcasts will either all exist or all not exist. The builder will
3415 // not create unnecessary casts if the types already match.
3416 Type *SelTy = A->getType();
3417 if (auto *VecTy = dyn_cast<VectorType>(Cond->getType())) {
3418 // For a fixed or scalable vector get N from <{vscale x} N x iM>
3419 unsigned Elts = VecTy->getElementCount().getKnownMinValue();
3420 // For a fixed or scalable vector, get the size in bits of N x iM; for a
3421 // scalar this is just M.
3422 unsigned SelEltSize = SelTy->getPrimitiveSizeInBits().getKnownMinValue();
3423 Type *EltTy = Builder.getIntNTy(SelEltSize / Elts);
3424 SelTy = VectorType::get(EltTy, VecTy->getElementCount());
3425 }
3426 Value *BitcastB = Builder.CreateBitCast(B, SelTy);
3427 if (InvertFalseVal)
3428 D = Builder.CreateNot(D);
3429 Value *BitcastD = Builder.CreateBitCast(D, SelTy);
3430 // The condition here is synthesized and thus we have no way of knowing the
3431 // distribution in general. Thus, mark the branch weights of the created
3432 // select unknown.
3433 Value *Select = Builder.CreateSelectWithUnknownProfile(
3434 Cond, BitcastB, BitcastD, DEBUG_TYPE);
3435 return Builder.CreateBitCast(Select, OrigType);
3436 }
3437
3438 return nullptr;
3439}
3440
3441// (icmp eq X, C) | (icmp ult Other, (X - C)) -> (icmp ule Other, (X - (C + 1)))
3442// (icmp ne X, C) & (icmp uge Other, (X - C)) -> (icmp ugt Other, (X - (C + 1)))
3444 Value *LHS1, bool LHSOneUse,
3445 CmpPredicate PredR, Value *RHS0,
3446 Value *RHS1, bool RHSOneUse,
3447 bool IsAnd, bool IsLogical,
3448 IRBuilderBase &Builder) {
3449 if (IsAnd) {
3450 PredL = CmpPredicate::getInverse(PredL);
3451 PredR = CmpPredicate::getInverse(PredR);
3452 }
3453
3454 const APInt *CInt;
3455 if (PredL != ICmpInst::ICMP_EQ || !match(LHS1, m_APIntAllowPoison(CInt)) ||
3456 !LHS0->getType()->isIntOrIntVectorTy() || !(LHSOneUse || RHSOneUse))
3457 return nullptr;
3458
3459 auto MatchRHSOp = [LHS0, CInt](const Value *RHSOp) {
3460 return match(RHSOp,
3461 m_Add(m_Specific(LHS0), m_SpecificIntAllowPoison(-*CInt))) ||
3462 (CInt->isZero() && RHSOp == LHS0);
3463 };
3464
3465 Value *Other;
3466 if (PredR == ICmpInst::ICMP_ULT && MatchRHSOp(RHS1))
3467 Other = RHS0;
3468 else if (PredR == ICmpInst::ICMP_UGT && MatchRHSOp(RHS0))
3469 Other = RHS1;
3470 else
3471 return nullptr;
3472
3473 if (IsLogical)
3474 Other = Builder.CreateFreeze(Other);
3475
3476 return Builder.CreateICmp(
3478 Builder.CreateSub(LHS0, ConstantInt::get(LHS0->getType(), *CInt + 1)),
3479 Other);
3480}
3481
3482/// Fold (icmp)&(icmp) or (icmp)|(icmp) if possible.
3483/// If IsLogical is true, then the and/or is in select form and the transform
3484/// must be poison-safe.
3485Value *InstCombinerImpl::foldAndOrOfICmps(Value *LHS, Value *RHS,
3486 Instruction &I, bool IsAnd,
3487 bool IsLogical) {
3488 CmpPredicate PredL, PredR;
3489 Value *LHS0, *LHS1, *RHS0, *RHS1;
3490 if (!match(LHS, m_ICmpLike(PredL, m_Value(LHS0), m_Value(LHS1))) ||
3491 !match(RHS, m_ICmpLike(PredR, m_Value(RHS0), m_Value(RHS1))))
3492 return nullptr;
3493
3494 bool LHSOneUse = LHS->hasOneUse();
3495 bool RHSOneUse = RHS->hasOneUse();
3496
3497 const SimplifyQuery Q = SQ.getWithInstruction(&I);
3498
3499 const APInt *LHSC = nullptr, *RHSC = nullptr;
3500 match(LHS1, m_APInt(LHSC));
3501 match(RHS1, m_APInt(RHSC));
3502
3503 // (icmp1 A, B) | (icmp2 A, B) --> (icmp3 A, B)
3504 // (icmp1 A, B) & (icmp2 A, B) --> (icmp3 A, B)
3505 if (predicatesFoldable(PredL, PredR)) {
3506 if (LHS0 == RHS1 && LHS1 == RHS0) {
3507 PredL = ICmpInst::getSwappedPredicate(PredL);
3508 std::swap(LHS0, LHS1);
3509 }
3510 if (LHS0 == RHS0 && LHS1 == RHS1) {
3511 unsigned Code = IsAnd ? getICmpCode(PredL) & getICmpCode(PredR)
3512 : getICmpCode(PredL) | getICmpCode(PredR);
3513 bool IsSigned = ICmpInst::isSigned(PredL) || ICmpInst::isSigned(PredR);
3514 return getNewICmpValue(Code, IsSigned, LHS0, LHS1, Builder);
3515 }
3516 }
3517
3518 if (Value *V = foldAndOrOfICmpEqConstantAndICmp(PredL, LHS0, LHS1, LHSOneUse,
3519 PredR, RHS0, RHS1, RHSOneUse,
3520 IsAnd, IsLogical, Builder))
3521 return V;
3522 // We can treat logical like bitwise here, because both operands are used on
3523 // the LHS, and as such poison from both will propagate.
3525 PredR, RHS0, RHS1, RHSOneUse, PredL, LHS0, LHS1, LHSOneUse, IsAnd,
3526 /*IsLogical*/ false, Builder))
3527 return V;
3528
3529 if (Value *V = foldAndOrOfICmpsWithConstEq(PredL, LHS0, LHS1, LHS, PredR,
3530 RHS0, RHS1, RHSOneUse, IsAnd,
3531 IsLogical, Builder, Q, I))
3532 return V;
3533 // We can convert this case to bitwise and, because both operands are used
3534 // on the LHS, and as such poison from both will propagate.
3535 // Can not handle RHS = trunc nuw as it is not same as icmp ne 0 for all
3536 // values
3537 if (isa<ICmpInst>(RHS))
3539 PredR, RHS0, RHS1, RHS, PredL, LHS0, LHS1, LHSOneUse, IsAnd,
3540 /*IsLogical=*/false, Builder, Q, I)) {
3541 // If RHS is still used, we should drop samesign flag.
3542 if (IsLogical && PredR.hasSameSign() && !RHS->use_empty()) {
3543 auto *CmpR = cast<ICmpInst>(RHS);
3544 CmpR->setSameSign(false);
3545 addToWorklist(CmpR);
3546 }
3547 return V;
3548 }
3549
3550 if (Value *V = foldIsPowerOf2OrZero(PredL, LHS0, LHS1, PredR, RHS0, RHS1,
3551 IsAnd, Builder, *this))
3552 return V;
3553 if (Value *V = foldIsPowerOf2OrZero(PredR, RHS0, RHS1, PredL, LHS0, LHS1,
3554 IsAnd, Builder, *this))
3555 return V;
3556
3557 // TODO: One of these directions is fine with logical and/or, the other could
3558 // be supported by inserting freeze.
3559 if (!IsLogical) {
3560 // E.g. (icmp slt x, 0) | (icmp sgt x, n) --> icmp ugt x, n
3561 // E.g. (icmp sge x, 0) & (icmp slt x, n) --> icmp ult x, n
3562 if (Value *V = simplifyRangeCheck(PredL, LHS0, LHS1, PredR, RHS0, RHS1, &I,
3563 /*Inverted=*/!IsAnd))
3564 return V;
3565
3566 // E.g. (icmp sgt x, n) | (icmp slt x, 0) --> icmp ugt x, n
3567 // E.g. (icmp slt x, n) & (icmp sge x, 0) --> icmp ult x, n
3568 if (Value *V = simplifyRangeCheck(PredR, RHS0, RHS1, PredL, LHS0, LHS1, &I,
3569 /*Inverted=*/!IsAnd))
3570 return V;
3571 }
3572
3573 // TODO: Add conjugated or fold, check whether it is safe for logical and/or.
3574 if (IsAnd && !IsLogical)
3575 if (Value *V = foldSignedTruncationCheck(PredL, LHS0, LHS1, PredR, RHS0,
3576 RHS1, I, Builder))
3577 return V;
3578
3579 if (Value *V = foldIsPowerOf2(PredL, LHS0, LHS1, PredR, RHS0, RHS1, IsAnd,
3580 Builder, *this))
3581 return V;
3582
3584 return V;
3585
3586 // TODO: Verify whether this is safe for logical and/or.
3587 if (!IsLogical) {
3588 if (Value *X = foldUnsignedUnderflowCheck(PredL, LHS0, LHS1, LHSOneUse,
3589 PredR, RHS0, RHS1, RHSOneUse,
3590 IsAnd, Q, Builder))
3591 return X;
3592 if (Value *X = foldUnsignedUnderflowCheck(PredR, RHS0, RHS1, RHSOneUse,
3593 PredL, LHS0, LHS1, LHSOneUse,
3594 IsAnd, Q, Builder))
3595 return X;
3596 }
3597
3598 // (icmp ne A, 0) | (icmp ne B, 0) --> (icmp ne (A|B), 0)
3599 // (icmp eq A, 0) & (icmp eq B, 0) --> (icmp eq (A|B), 0)
3600 // TODO: Remove this and below when foldLogOpOfMaskedICmps can handle undefs.
3601 if (PredL == (IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE) &&
3602 PredL.dropSameSign() == PredR.dropSameSign() &&
3603 match(LHS1, m_ZeroInt()) && match(RHS1, m_ZeroInt()) &&
3604 LHS0->getType() == RHS0->getType() &&
3605 (!IsLogical || isGuaranteedNotToBePoison(RHS0))) {
3606 Value *NewOr = Builder.CreateOr(LHS0, RHS0);
3607 return Builder.CreateICmp(PredL, NewOr,
3609 }
3610
3611 // (icmp ne A, -1) | (icmp ne B, -1) --> (icmp ne (A&B), -1)
3612 // (icmp eq A, -1) & (icmp eq B, -1) --> (icmp eq (A&B), -1)
3613 if (PredL == (IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE) &&
3614 PredL.dropSameSign() == PredR.dropSameSign() &&
3615 match(LHS1, m_AllOnes()) && match(RHS1, m_AllOnes()) &&
3616 LHS0->getType() == RHS0->getType() &&
3617 (!IsLogical || isGuaranteedNotToBePoison(RHS0))) {
3618 Value *NewAnd = Builder.CreateAnd(LHS0, RHS0);
3619 return Builder.CreateICmp(PredL, NewAnd,
3621 }
3622
3623 if (!IsLogical)
3625 Builder, PredL, LHS0, LHS1, LHSOneUse, PredR, RHS0, RHS1, RHSOneUse,
3626 IsAnd, Q))
3627 return V;
3628
3629 // This only handles icmp of constants: (icmp1 A, C1) | (icmp2 B, C2).
3630 if (!LHSC || !RHSC)
3631 return nullptr;
3632
3633 // Match naive pattern (and its inverted form) for checking if two values
3634 // share same sign. An example of the pattern:
3635 // (icmp slt (X & Y), 0) | (icmp sgt (X | Y), -1) -> (icmp sgt (X ^ Y), -1)
3636 // Inverted form (example):
3637 // (icmp slt (X | Y), 0) & (icmp sgt (X & Y), -1) -> (icmp slt (X ^ Y), 0)
3638 bool TrueIfSignedL, TrueIfSignedR;
3639 if (isSignBitCheck(PredL, *LHSC, TrueIfSignedL) &&
3640 isSignBitCheck(PredR, *RHSC, TrueIfSignedR) &&
3641 (RHS->hasOneUse() || LHS->hasOneUse())) {
3642 Value *X, *Y;
3643 if (IsAnd) {
3644 if ((TrueIfSignedL && !TrueIfSignedR &&
3645 match(LHS0, m_Or(m_Value(X), m_Value(Y))) &&
3646 match(RHS0, m_c_And(m_Specific(X), m_Specific(Y)))) ||
3647 (!TrueIfSignedL && TrueIfSignedR &&
3648 match(LHS0, m_And(m_Value(X), m_Value(Y))) &&
3649 match(RHS0, m_c_Or(m_Specific(X), m_Specific(Y))))) {
3650 Value *NewXor = Builder.CreateXor(X, Y);
3651 return Builder.CreateIsNeg(NewXor);
3652 }
3653 } else {
3654 if ((TrueIfSignedL && !TrueIfSignedR &&
3655 match(LHS0, m_And(m_Value(X), m_Value(Y))) &&
3656 match(RHS0, m_c_Or(m_Specific(X), m_Specific(Y)))) ||
3657 (!TrueIfSignedL && TrueIfSignedR &&
3658 match(LHS0, m_Or(m_Value(X), m_Value(Y))) &&
3659 match(RHS0, m_c_And(m_Specific(X), m_Specific(Y))))) {
3660 Value *NewXor = Builder.CreateXor(X, Y);
3661 return Builder.CreateIsNotNeg(NewXor);
3662 }
3663 }
3664 }
3665
3666 // (X & ExpMask) != 0 && (X & ExpMask) != ExpMask -> isnormal(X)
3667 // (X & ExpMask) == 0 || (X & ExpMask) == ExpMask -> !isnormal(X)
3668 Value *X;
3669 const APInt *MaskC;
3670 if (LHS0 == RHS0 && PredL.dropSameSign() == PredR.dropSameSign() &&
3671 PredL == (IsAnd ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ) &&
3672 !I.getFunction()->hasFnAttribute(Attribute::NoImplicitFloat) &&
3673 LHSOneUse && RHSOneUse &&
3674 match(LHS0, m_And(m_ElementWiseBitCast(m_Value(X)), m_APInt(MaskC))) &&
3675 X->getType()->getScalarType()->isIEEELikeFPTy() &&
3676 APFloat(X->getType()->getScalarType()->getFltSemantics(), *MaskC)
3677 .isPosInfinity() &&
3678 ((LHSC->isZero() && *RHSC == *MaskC) ||
3679 (RHSC->isZero() && *LHSC == *MaskC)))
3680 return Builder.createIsFPClass(X, IsAnd ? FPClassTest::fcNormal
3682
3683 return foldAndOrOfICmpsUsingRanges(PredL, LHS0, LHS1, LHSOneUse, PredR, RHS0,
3684 RHS1, RHSOneUse, IsAnd);
3685}
3686
3687/// If IsLogical is true, then the and/or is in select form and the transform
3688/// must be poison-safe.
3689Value *InstCombinerImpl::foldBooleanAndOr(Value *LHS, Value *RHS,
3690 Instruction &I, bool IsAnd,
3691 bool IsLogical) {
3692 if (!LHS->getType()->isIntOrIntVectorTy(1))
3693 return nullptr;
3694
3695 // handle (roughly):
3696 // (icmp ne (A & B), C) | (icmp ne (A & D), E)
3697 // (icmp eq (A & B), C) & (icmp eq (A & D), E)
3698 if (Value *V = foldLogOpOfMaskedICmps(LHS, RHS, IsAnd, IsLogical, Builder,
3699 SQ.getWithInstruction(&I)))
3700 return V;
3701
3702 if (Value *Res = foldAndOrOfICmps(LHS, RHS, I, IsAnd, IsLogical))
3703 return Res;
3704
3705 if (auto *LHSCmp = dyn_cast<FCmpInst>(LHS))
3706 if (auto *RHSCmp = dyn_cast<FCmpInst>(RHS))
3707 if (Value *Res = foldLogicOfFCmps(LHSCmp, RHSCmp, IsAnd, IsLogical))
3708 return Res;
3709
3710 if (Value *Res = foldEqOfParts(LHS, RHS, IsAnd))
3711 return Res;
3712
3713 return nullptr;
3714}
3715
3717 InstCombiner::BuilderTy &Builder) {
3718 assert(I.getOpcode() == Instruction::Or &&
3719 "Simplification only supports or at the moment.");
3720
3721 Value *Cmp1, *Cmp2, *Cmp3, *Cmp4;
3722 if (!match(I.getOperand(0), m_And(m_Value(Cmp1), m_Value(Cmp2))) ||
3723 !match(I.getOperand(1), m_And(m_Value(Cmp3), m_Value(Cmp4))))
3724 return nullptr;
3725
3726 // Check if any two pairs of the and operations are inversions of each other.
3727 if (isKnownInversion(Cmp1, Cmp3) && isKnownInversion(Cmp2, Cmp4))
3728 return Builder.CreateXor(Cmp1, Cmp4);
3729 if (isKnownInversion(Cmp1, Cmp4) && isKnownInversion(Cmp2, Cmp3))
3730 return Builder.CreateXor(Cmp1, Cmp3);
3731
3732 return nullptr;
3733}
3734
3735/// Match \p V as "shufflevector -> bitcast" or "extractelement -> zext -> shl"
3736/// patterns, which extract vector elements and pack them in the same relative
3737/// positions.
3738///
3739/// \p Vec is the underlying vector being extracted from.
3740/// \p Mask is a bitmask identifying which packed elements are obtained from the
3741/// vector.
3742/// \p VecOffset is the vector element corresponding to index 0 of the
3743/// mask.
3745 int64_t &VecOffset,
3746 SmallBitVector &Mask,
3747 const DataLayout &DL) {
3748 // First try to match extractelement -> zext -> shl
3749 uint64_t VecIdx, ShlAmt;
3751 m_ConstantInt(VecIdx))),
3752 ShlAmt))) {
3753 auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
3754 if (!VecTy)
3755 return false;
3756 auto *EltTy = dyn_cast<IntegerType>(VecTy->getElementType());
3757 if (!EltTy)
3758 return false;
3759
3760 const unsigned EltBitWidth = EltTy->getBitWidth();
3761 const unsigned TargetBitWidth = V->getType()->getIntegerBitWidth();
3762 if (TargetBitWidth % EltBitWidth != 0 || ShlAmt % EltBitWidth != 0)
3763 return false;
3764 const unsigned TargetEltWidth = TargetBitWidth / EltBitWidth;
3765 const unsigned ShlEltAmt = ShlAmt / EltBitWidth;
3766
3767 const unsigned MaskIdx =
3768 DL.isLittleEndian() ? ShlEltAmt : TargetEltWidth - ShlEltAmt - 1;
3769
3770 VecOffset = static_cast<int64_t>(VecIdx) - static_cast<int64_t>(MaskIdx);
3771 Mask.resize(TargetEltWidth);
3772 Mask.set(MaskIdx);
3773 return true;
3774 }
3775
3776 // Now try to match a bitcasted subvector.
3777 Instruction *SrcVecI;
3778 if (!match(V, m_BitCast(m_Instruction(SrcVecI))))
3779 return false;
3780
3781 auto *SrcTy = dyn_cast<FixedVectorType>(SrcVecI->getType());
3782 if (!SrcTy)
3783 return false;
3784
3785 Mask.resize(SrcTy->getNumElements());
3786
3787 // First check for a subvector obtained from a shufflevector.
3788 if (isa<ShuffleVectorInst>(SrcVecI)) {
3789 Constant *ConstVec;
3790 ArrayRef<int> ShuffleMask;
3791 if (!match(SrcVecI, m_Shuffle(m_Value(Vec), m_Constant(ConstVec),
3792 m_Mask(ShuffleMask))))
3793 return false;
3794
3795 auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
3796 if (!VecTy)
3797 return false;
3798
3799 const unsigned NumVecElts = VecTy->getNumElements();
3800 bool FoundVecOffset = false;
3801 for (unsigned Idx = 0; Idx < ShuffleMask.size(); ++Idx) {
3802 if (ShuffleMask[Idx] == PoisonMaskElem)
3803 return false;
3804 const unsigned ShuffleIdx = ShuffleMask[Idx];
3805 if (ShuffleIdx >= NumVecElts) {
3806 const unsigned ConstIdx = ShuffleIdx - NumVecElts;
3807 auto *ConstElt =
3808 dyn_cast<ConstantInt>(ConstVec->getAggregateElement(ConstIdx));
3809 if (!ConstElt || !ConstElt->isNullValue())
3810 return false;
3811 continue;
3812 }
3813
3814 if (FoundVecOffset) {
3815 if (VecOffset + Idx != ShuffleIdx)
3816 return false;
3817 } else {
3818 if (ShuffleIdx < Idx)
3819 return false;
3820 VecOffset = ShuffleIdx - Idx;
3821 FoundVecOffset = true;
3822 }
3823 Mask.set(Idx);
3824 }
3825 return FoundVecOffset;
3826 }
3827
3828 // Check for a subvector obtained as an (insertelement V, 0, idx)
3829 uint64_t InsertIdx;
3830 if (!match(SrcVecI,
3831 m_InsertElt(m_Value(Vec), m_Zero(), m_ConstantInt(InsertIdx))))
3832 return false;
3833
3834 auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
3835 if (!VecTy)
3836 return false;
3837 VecOffset = 0;
3838 bool AlreadyInsertedMaskedElt = Mask.test(InsertIdx);
3839 Mask.set();
3840 if (!AlreadyInsertedMaskedElt)
3841 Mask.reset(InsertIdx);
3842 return true;
3843}
3844
3845/// Try to fold the join of two scalar integers whose contents are packed
3846/// elements of the same vector.
3848 InstCombiner::BuilderTy &Builder,
3849 const DataLayout &DL) {
3850 assert(I.getOpcode() == Instruction::Or);
3851 Value *LhsVec, *RhsVec;
3852 int64_t LhsVecOffset, RhsVecOffset;
3853 SmallBitVector Mask;
3854 if (!matchSubIntegerPackFromVector(I.getOperand(0), LhsVec, LhsVecOffset,
3855 Mask, DL))
3856 return nullptr;
3857 if (!matchSubIntegerPackFromVector(I.getOperand(1), RhsVec, RhsVecOffset,
3858 Mask, DL))
3859 return nullptr;
3860 if (LhsVec != RhsVec || LhsVecOffset != RhsVecOffset)
3861 return nullptr;
3862
3863 // Convert into shufflevector -> bitcast;
3864 const unsigned ZeroVecIdx =
3865 cast<FixedVectorType>(LhsVec->getType())->getNumElements();
3866 SmallVector<int> ShuffleMask(Mask.size(), ZeroVecIdx);
3867 for (unsigned Idx : Mask.set_bits()) {
3868 assert(LhsVecOffset + Idx >= 0);
3869 ShuffleMask[Idx] = LhsVecOffset + Idx;
3870 }
3871
3872 Value *MaskedVec = Builder.CreateShuffleVector(
3873 LhsVec, Constant::getNullValue(LhsVec->getType()), ShuffleMask,
3874 I.getName() + ".v");
3875 return CastInst::Create(Instruction::BitCast, MaskedVec, I.getType());
3876}
3877
3878/// Match \p V as "lshr -> mask -> zext -> shl".
3879///
3880/// \p Int is the underlying integer being extracted from.
3881/// \p Mask is a bitmask identifying which bits of the integer are being
3882/// extracted. \p Offset identifies which bit of the result \p V corresponds to
3883/// the least significant bit of \p Int
3885 uint64_t &Offset, bool &IsShlNUW,
3886 bool &IsShlNSW) {
3887 Value *ShlOp0;
3888 uint64_t ShlAmt = 0;
3889 if (!match(V, m_OneUse(m_Shl(m_Value(ShlOp0), m_ConstantInt(ShlAmt)))))
3890 return false;
3891
3892 IsShlNUW = cast<BinaryOperator>(V)->hasNoUnsignedWrap();
3893 IsShlNSW = cast<BinaryOperator>(V)->hasNoSignedWrap();
3894
3895 Value *ZExtOp0;
3896 if (!match(ShlOp0, m_OneUse(m_ZExt(m_Value(ZExtOp0)))))
3897 return false;
3898
3899 Value *MaskedOp0;
3900 const APInt *ShiftedMaskConst = nullptr;
3901 if (!match(ZExtOp0, m_CombineOr(m_OneUse(m_And(m_Value(MaskedOp0),
3902 m_APInt(ShiftedMaskConst))),
3903 m_Value(MaskedOp0))))
3904 return false;
3905
3906 uint64_t LShrAmt = 0;
3907 if (!match(MaskedOp0,
3909 m_Value(Int))))
3910 return false;
3911
3912 if (LShrAmt > ShlAmt)
3913 return false;
3914 Offset = ShlAmt - LShrAmt;
3915
3916 Mask = ShiftedMaskConst ? ShiftedMaskConst->shl(LShrAmt)
3918 Int->getType()->getScalarSizeInBits(), LShrAmt);
3919
3920 return true;
3921}
3922
3923/// Try to fold the join of two scalar integers whose bits are unpacked and
3924/// zexted from the same source integer.
3926 InstCombiner::BuilderTy &Builder) {
3927
3928 Value *LhsInt, *RhsInt;
3929 APInt LhsMask, RhsMask;
3930 uint64_t LhsOffset, RhsOffset;
3931 bool IsLhsShlNUW, IsLhsShlNSW, IsRhsShlNUW, IsRhsShlNSW;
3932 if (!matchZExtedSubInteger(Lhs, LhsInt, LhsMask, LhsOffset, IsLhsShlNUW,
3933 IsLhsShlNSW))
3934 return nullptr;
3935 if (!matchZExtedSubInteger(Rhs, RhsInt, RhsMask, RhsOffset, IsRhsShlNUW,
3936 IsRhsShlNSW))
3937 return nullptr;
3938 if (LhsInt != RhsInt || LhsOffset != RhsOffset)
3939 return nullptr;
3940
3941 APInt Mask = LhsMask | RhsMask;
3942
3943 Type *DestTy = Lhs->getType();
3944 Value *Res = Builder.CreateShl(
3945 Builder.CreateZExt(
3946 Builder.CreateAnd(LhsInt, Mask, LhsInt->getName() + ".mask"), DestTy,
3947 LhsInt->getName() + ".zext"),
3948 ConstantInt::get(DestTy, LhsOffset), "", IsLhsShlNUW && IsRhsShlNUW,
3949 IsLhsShlNSW && IsRhsShlNSW);
3950 Res->takeName(Lhs);
3951 return Res;
3952}
3953
3954// A decomposition of ((X & Mask) * Factor). The NUW / NSW bools
3955// track these properities for preservation. Note that we can decompose
3956// equivalent select form of this expression (e.g. (!(X & Mask) ? 0 : Mask *
3957// Factor))
3962 bool NUW;
3963 bool NSW;
3964
3966 return X == Other.X && !Mask.intersects(Other.Mask) &&
3967 Factor == Other.Factor;
3968 }
3969};
3970
3971static std::optional<DecomposedBitMaskMul> matchBitmaskMul(Value *V) {
3973 if (!Op)
3974 return std::nullopt;
3975
3976 // Decompose (A & N) * C) into BitMaskMul
3977 Value *Original = nullptr;
3978 const APInt *Mask = nullptr;
3979 const APInt *MulConst = nullptr;
3980 if (match(Op, m_Mul(m_And(m_Value(Original), m_APInt(Mask)),
3981 m_APInt(MulConst)))) {
3982 if (MulConst->isZero() || Mask->isZero())
3983 return std::nullopt;
3984
3985 return std::optional<DecomposedBitMaskMul>(
3986 {Original, *MulConst, *Mask,
3987 cast<BinaryOperator>(Op)->hasNoUnsignedWrap(),
3988 cast<BinaryOperator>(Op)->hasNoSignedWrap()});
3989 }
3990
3991 Value *Cond = nullptr;
3992 const APInt *EqZero = nullptr, *NeZero = nullptr;
3993
3994 // Decompose ((A & N) ? 0 : N * C) into BitMaskMul
3995 if (match(Op, m_Select(m_Value(Cond), m_APInt(EqZero), m_APInt(NeZero)))) {
3996 auto ICmpDecompose =
3997 decomposeBitTest(Cond, /*LookThroughTrunc=*/true,
3998 /*AllowNonZeroC=*/false, /*DecomposeBitMask=*/true);
3999 if (!ICmpDecompose.has_value())
4000 return std::nullopt;
4001
4002 // decomposeBitTest may provide a scalar bit test for a vector select.
4003 // Ensure the types match.
4004 if (ICmpDecompose->X->getType() != V->getType())
4005 return std::nullopt;
4006
4007 assert(ICmpInst::isEquality(ICmpDecompose->Pred) &&
4008 ICmpDecompose->C.isZero());
4009
4010 if (ICmpDecompose->Pred == ICmpInst::ICMP_NE)
4011 std::swap(EqZero, NeZero);
4012
4013 if (!EqZero->isZero() || NeZero->isZero())
4014 return std::nullopt;
4015
4016 if (!ICmpDecompose->Mask.isPowerOf2() || ICmpDecompose->Mask.isZero())
4017 return std::nullopt;
4018
4019 if (!NeZero->urem(ICmpDecompose->Mask).isZero())
4020 return std::nullopt;
4021
4022 return std::optional<DecomposedBitMaskMul>(
4023 {ICmpDecompose->X, NeZero->udiv(ICmpDecompose->Mask),
4024 ICmpDecompose->Mask, /*NUW=*/false, /*NSW=*/false});
4025 }
4026
4027 return std::nullopt;
4028}
4029
4030/// (A & N) * C + (A & M) * C -> (A & (N + M)) & C
4031/// This also accepts the equivalent select form of (A & N) * C
4032/// expressions i.e. !(A & N) ? 0 : N * C)
4033static Value *foldBitmaskMul(Value *Op0, Value *Op1,
4034 InstCombiner::BuilderTy &Builder) {
4035 auto Decomp1 = matchBitmaskMul(Op1);
4036 if (!Decomp1)
4037 return nullptr;
4038
4039 auto Decomp0 = matchBitmaskMul(Op0);
4040 if (!Decomp0)
4041 return nullptr;
4042
4043 if (Decomp0->isCombineableWith(*Decomp1)) {
4044 Value *NewAnd = Builder.CreateAnd(
4045 Decomp0->X,
4046 ConstantInt::get(Decomp0->X->getType(), Decomp0->Mask + Decomp1->Mask));
4047
4048 return Builder.CreateMul(
4049 NewAnd, ConstantInt::get(NewAnd->getType(), Decomp1->Factor), "",
4050 Decomp0->NUW && Decomp1->NUW, Decomp0->NSW && Decomp1->NSW);
4051 }
4052
4053 return nullptr;
4054}
4055
4056Value *InstCombinerImpl::foldDisjointOr(Value *LHS, Value *RHS) {
4057 if (Value *Res = foldBitmaskMul(LHS, RHS, Builder))
4058 return Res;
4060 return Res;
4061
4062 return nullptr;
4063}
4064
4065Value *InstCombinerImpl::reassociateDisjointOr(Value *LHS, Value *RHS) {
4066
4067 Value *X, *Y;
4069 if (Value *Res = foldDisjointOr(LHS, X))
4070 return Builder.CreateDisjointOr(Res, Y);
4071 if (Value *Res = foldDisjointOr(LHS, Y))
4072 return Builder.CreateDisjointOr(Res, X);
4073 }
4074
4076 if (Value *Res = foldDisjointOr(X, RHS))
4077 return Builder.CreateDisjointOr(Res, Y);
4078 if (Value *Res = foldDisjointOr(Y, RHS))
4079 return Builder.CreateDisjointOr(Res, X);
4080 }
4081
4082 return nullptr;
4083}
4084
4085/// Fold Res, Overflow = (umul.with.overflow x c1); (or Overflow (ugt Res c2))
4086/// --> (ugt x (c2/c1)). This code checks whether a multiplication of two
4087/// unsigned numbers (one is a constant) is mathematically greater than a
4088/// second constant.
4090 InstCombiner::BuilderTy &Builder,
4091 const DataLayout &DL) {
4092 Value *WOV, *X;
4093 const APInt *C1, *C2;
4094 if (match(&I,
4097 m_Value(X), m_APInt(C1)))),
4100 m_APInt(C2))))) &&
4101 !C1->isZero()) {
4102 Constant *NewC = ConstantInt::get(X->getType(), C2->udiv(*C1));
4103 return Builder.CreateICmp(ICmpInst::ICMP_UGT, X, NewC);
4104 }
4105 return nullptr;
4106}
4107
4108/// Fold select(X >s 0, 0, -X) | smax(X, 0) --> abs(X)
4109/// select(X <s 0, -X, 0) | smax(X, 0) --> abs(X)
4111 InstCombiner::BuilderTy &Builder) {
4112 Value *X;
4113 Value *Sel;
4114 if (match(&I,
4116 auto NegX = m_Neg(m_Specific(X));
4118 m_ZeroInt()),
4119 m_ZeroInt(), NegX)) ||
4121 m_ZeroInt()),
4122 NegX, m_ZeroInt())))
4123 return Builder.CreateBinaryIntrinsic(Intrinsic::abs, X,
4124 Builder.getFalse());
4125 }
4126 return nullptr;
4127}
4128
4130 Value *C, *A, *B;
4131 // (C && A) || (!C && B)
4132 // (C && A) || (B && !C)
4133 // (A && C) || (!C && B)
4134 // (A && C) || (B && !C) (may require freeze)
4135 //
4136 // => select C, A, B
4137 if (match(Op1, m_c_LogicalAnd(m_Not(m_Value(C)), m_Value(B))) &&
4139 auto *SelOp0 = dyn_cast<SelectInst>(Op0);
4140 auto *SelOp1 = dyn_cast<SelectInst>(Op1);
4141
4142 bool MayNeedFreeze = SelOp0 && SelOp1 &&
4143 match(SelOp1->getTrueValue(),
4144 m_Not(m_Specific(SelOp0->getTrueValue())));
4145 if (MayNeedFreeze)
4146 C = Builder.CreateFreeze(C);
4147 Value *C2 = nullptr, *A2 = nullptr, *B2 = nullptr;
4148 if (match(Op0, m_LogicalAnd(m_Specific(C), m_Value(A2))) && SelOp0)
4149 return SelectInst::Create(C, A, B, "", nullptr, SelOp0);
4150 if (match(Op1, m_LogicalAnd(m_Not(m_Value(C2)), m_Value(B2))) && SelOp1) {
4151 SelectInst *NewSI = SelectInst::Create(C, A, B, "", nullptr, SelOp1);
4152 NewSI->swapProfMetadata();
4153 return NewSI;
4154 }
4155 return createSelectInstWithUnknownProfile(C, A, B);
4156 }
4157
4158 // (!C && A) || (C && B)
4159 // (A && !C) || (C && B)
4160 // (!C && A) || (B && C)
4161 // (A && !C) || (B && C) (may require freeze)
4162 //
4163 // => select C, B, A
4164 if (match(Op0, m_c_LogicalAnd(m_Not(m_Value(C)), m_Value(A))) &&
4166 auto *SelOp0 = dyn_cast<SelectInst>(Op0);
4167 auto *SelOp1 = dyn_cast<SelectInst>(Op1);
4168 bool MayNeedFreeze = SelOp0 && SelOp1 &&
4169 match(SelOp0->getTrueValue(),
4170 m_Not(m_Specific(SelOp1->getTrueValue())));
4171 if (MayNeedFreeze)
4172 C = Builder.CreateFreeze(C);
4173 Value *C2 = nullptr, *A2 = nullptr, *B2 = nullptr;
4174 if (match(Op0, m_LogicalAnd(m_Not(m_Value(C2)), m_Value(A2))) && SelOp0) {
4175 SelectInst *NewSI = SelectInst::Create(C, B, A, "", nullptr, SelOp0);
4176 NewSI->swapProfMetadata();
4177 return NewSI;
4178 }
4179 if (match(Op1, m_LogicalAnd(m_Specific(C), m_Value(B2))) && SelOp1)
4180 return SelectInst::Create(C, B, A, "", nullptr, SelOp1);
4181 return createSelectInstWithUnknownProfile(C, B, A);
4182 }
4183
4184 return nullptr;
4185}
4186
4187// FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
4188// here. We should standardize that construct where it is needed or choose some
4189// other way to ensure that commutated variants of patterns are not missed.
4191 if (Value *V = simplifyOrInst(I.getOperand(0), I.getOperand(1),
4192 SQ.getWithInstruction(&I)))
4193 return replaceInstUsesWith(I, V);
4194
4196 return &I;
4197
4199 return X;
4200
4202 return Phi;
4203
4204 // See if we can simplify any instructions used by the instruction whose sole
4205 // purpose is to compute bits we don't care about.
4207 return &I;
4208
4209 // Do this before using distributive laws to catch simple and/or/not patterns.
4211 return Xor;
4212
4214 return X;
4215
4217 return X;
4218
4219 // (A & B) | (C & D) -> A ^ D where A == ~C && B == ~D
4220 // (A & B) | (C & D) -> A ^ C where A == ~D && B == ~C
4221 if (Value *V = foldOrOfInversions(I, Builder))
4222 return replaceInstUsesWith(I, V);
4223
4224 // (A&B)|(A&C) -> A&(B|C) etc
4226 return replaceInstUsesWith(I, V);
4227
4228 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
4229 Type *Ty = I.getType();
4230 if (Ty->isIntOrIntVectorTy(1)) {
4231 if (auto *SI0 = dyn_cast<SelectInst>(Op0)) {
4232 if (auto *R =
4233 foldAndOrOfSelectUsingImpliedCond(Op1, *SI0, /* IsAnd */ false))
4234 return R;
4235 }
4236 if (auto *SI1 = dyn_cast<SelectInst>(Op1)) {
4237 if (auto *R =
4238 foldAndOrOfSelectUsingImpliedCond(Op0, *SI1, /* IsAnd */ false))
4239 return R;
4240 }
4241 }
4242
4243 if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I))
4244 return FoldedLogic;
4245
4246 if (Instruction *FoldedLogic = foldBinOpSelectBinOp(I))
4247 return FoldedLogic;
4248
4249 if (Instruction *BitOp = matchBSwapOrBitReverse(I, /*MatchBSwaps*/ true,
4250 /*MatchBitReversals*/ true))
4251 return BitOp;
4252
4253 if (Instruction *Funnel = matchFunnelShift(I, *this))
4254 return Funnel;
4255
4257 return replaceInstUsesWith(I, Concat);
4258
4260 return R;
4261
4263 return R;
4264
4265 if (cast<PossiblyDisjointInst>(I).isDisjoint()) {
4266 if (Instruction *R =
4267 foldAddLikeCommutative(I.getOperand(0), I.getOperand(1),
4268 /*NSW=*/true, /*NUW=*/true))
4269 return R;
4270 if (Instruction *R =
4271 foldAddLikeCommutative(I.getOperand(1), I.getOperand(0),
4272 /*NSW=*/true, /*NUW=*/true))
4273 return R;
4274
4275 if (Value *Res = foldDisjointOr(I.getOperand(0), I.getOperand(1)))
4276 return replaceInstUsesWith(I, Res);
4277
4278 if (Value *Res = reassociateDisjointOr(I.getOperand(0), I.getOperand(1)))
4279 return replaceInstUsesWith(I, Res);
4280 }
4281
4282 Value *X, *Y;
4283 const APInt *CV;
4284 if (match(&I, m_c_Or(m_OneUse(m_Xor(m_Value(X), m_APInt(CV))), m_Value(Y))) &&
4285 !CV->isAllOnes() && MaskedValueIsZero(Y, *CV, &I)) {
4286 // (X ^ C) | Y -> (X | Y) ^ C iff Y & C == 0
4287 // The check for a 'not' op is for efficiency (if Y is known zero --> ~X).
4288 Value *Or = Builder.CreateOr(X, Y);
4289 return BinaryOperator::CreateXor(Or, ConstantInt::get(Ty, *CV));
4290 }
4291
4292 // If the operands have no common bits set:
4293 // or (mul X, Y), X --> add (mul X, Y), X --> mul X, (Y + 1)
4295 m_Deferred(X)))) {
4296 Value *IncrementY = Builder.CreateAdd(Y, ConstantInt::get(Ty, 1));
4297 return BinaryOperator::CreateMul(X, IncrementY);
4298 }
4299
4300 // Canonicalization to achieve lowering to Bit Manipulation Instructions (BMI)
4301 // ~X | (X-1) => ~(X & -X)
4302 Value *Op;
4305 Value *NegX = Builder.CreateNeg(Op);
4306 Value *And = Builder.CreateAnd(Op, NegX);
4308 }
4309
4310 // (C && A) || (C && B) => select C, A, B (and similar cases)
4311 //
4312 // Note: This is the same transformation used in `foldSelectOfBools`,
4313 // except that it's an `or` instead of `select`.
4314 if (I.getType()->isIntOrIntVectorTy(1) &&
4315 (Op0->hasOneUse() || Op1->hasOneUse())) {
4316 if (Instruction *V = FoldOrOfLogicalAnds(Op0, Op1)) {
4317 return V;
4318 }
4319 }
4320
4321 // (A & C) | (B & D)
4322 Value *A, *B, *C, *D;
4323 if (match(Op0, m_And(m_Value(A), m_Value(C))) &&
4324 match(Op1, m_And(m_Value(B), m_Value(D)))) {
4325
4326 // (A & C0) | (B & C1)
4327 const APInt *C0, *C1;
4328 if (match(C, m_APInt(C0)) && match(D, m_APInt(C1))) {
4329 Value *X;
4330 if (*C0 == ~*C1) {
4331 // ((X | B) & MaskC) | (B & ~MaskC) -> (X & MaskC) | B
4332 if (match(A, m_c_Or(m_Value(X), m_Specific(B))))
4333 return BinaryOperator::CreateOr(Builder.CreateAnd(X, *C0), B);
4334 // (A & MaskC) | ((X | A) & ~MaskC) -> (X & ~MaskC) | A
4335 if (match(B, m_c_Or(m_Specific(A), m_Value(X))))
4336 return BinaryOperator::CreateOr(Builder.CreateAnd(X, *C1), A);
4337
4338 // ((X ^ B) & MaskC) | (B & ~MaskC) -> (X & MaskC) ^ B
4339 if (match(A, m_c_Xor(m_Value(X), m_Specific(B))))
4340 return BinaryOperator::CreateXor(Builder.CreateAnd(X, *C0), B);
4341 // (A & MaskC) | ((X ^ A) & ~MaskC) -> (X & ~MaskC) ^ A
4342 if (match(B, m_c_Xor(m_Specific(A), m_Value(X))))
4343 return BinaryOperator::CreateXor(Builder.CreateAnd(X, *C1), A);
4344 }
4345
4346 if ((*C0 & *C1).isZero()) {
4347 // ((X | B) & C0) | (B & C1) --> (X | B) & (C0 | C1)
4348 // iff (C0 & C1) == 0 and (X & ~C0) == 0
4349 if (match(A, m_c_Or(m_Value(X), m_Specific(B))) &&
4350 MaskedValueIsZero(X, ~*C0, &I)) {
4351 Constant *C01 = ConstantInt::get(Ty, *C0 | *C1);
4352 return BinaryOperator::CreateAnd(A, C01);
4353 }
4354 // (A & C0) | ((X | A) & C1) --> (X | A) & (C0 | C1)
4355 // iff (C0 & C1) == 0 and (X & ~C1) == 0
4356 if (match(B, m_c_Or(m_Value(X), m_Specific(A))) &&
4357 MaskedValueIsZero(X, ~*C1, &I)) {
4358 Constant *C01 = ConstantInt::get(Ty, *C0 | *C1);
4359 return BinaryOperator::CreateAnd(B, C01);
4360 }
4361 // ((X | C2) & C0) | ((X | C3) & C1) --> (X | C2 | C3) & (C0 | C1)
4362 // iff (C0 & C1) == 0 and (C2 & ~C0) == 0 and (C3 & ~C1) == 0.
4363 const APInt *C2, *C3;
4364 if (match(A, m_Or(m_Value(X), m_APInt(C2))) &&
4365 match(B, m_Or(m_Specific(X), m_APInt(C3))) &&
4366 (*C2 & ~*C0).isZero() && (*C3 & ~*C1).isZero()) {
4367 Value *Or = Builder.CreateOr(X, *C2 | *C3, "bitfield");
4368 Constant *C01 = ConstantInt::get(Ty, *C0 | *C1);
4369 return BinaryOperator::CreateAnd(Or, C01);
4370 }
4371 }
4372
4373 // ((trunc (lshr X, S)) & C0) | ((lshr (trunc X), S) & C1)
4374 // --> ((trunc (lshr X, S)) & (C0 | C1)) (and similar cases)
4375 // A = trunc (lshr X, S) B = lshr (trunc X), S
4376 const APInt *ShiftAmt;
4377 if (match(A, m_Trunc(m_LShr(m_Value(X), m_APInt(ShiftAmt)))) &&
4378 match(B, m_LShr(m_Trunc(m_Specific(X)), m_SpecificInt(*ShiftAmt))) &&
4379 ShiftAmt->ult(A->getType()->getScalarSizeInBits()) &&
4380 C1->isIntN(A->getType()->getScalarSizeInBits() -
4381 ShiftAmt->getZExtValue())) {
4382 return BinaryOperator::CreateAnd(
4383 A, ConstantInt::get(I.getType(), *C0 | *C1));
4384 }
4385 // A = lshr (trunc X), S
4386 // B = trunc (lshr X, S)
4387 if (match(B, m_Trunc(m_LShr(m_Value(X), m_APInt(ShiftAmt)))) &&
4388 match(A, m_LShr(m_Trunc(m_Specific(X)), m_SpecificInt(*ShiftAmt))) &&
4389 ShiftAmt->ult(A->getType()->getScalarSizeInBits()) &&
4390 C0->isIntN(A->getType()->getScalarSizeInBits() -
4391 ShiftAmt->getZExtValue())) {
4392 return BinaryOperator::CreateAnd(
4393 B, ConstantInt::get(I.getType(), *C0 | *C1));
4394 }
4395 }
4396
4397 // Don't try to form a select if it's unlikely that we'll get rid of at
4398 // least one of the operands. A select is generally more expensive than the
4399 // 'or' that it is replacing.
4400 if (Op0->hasOneUse() || Op1->hasOneUse()) {
4401 // (Cond & C) | (~Cond & D) -> Cond ? C : D, and commuted variants.
4402 if (Value *V = matchSelectFromAndOr(A, C, B, D))
4403 return replaceInstUsesWith(I, V);
4404 if (Value *V = matchSelectFromAndOr(A, C, D, B))
4405 return replaceInstUsesWith(I, V);
4406 if (Value *V = matchSelectFromAndOr(C, A, B, D))
4407 return replaceInstUsesWith(I, V);
4408 if (Value *V = matchSelectFromAndOr(C, A, D, B))
4409 return replaceInstUsesWith(I, V);
4410 if (Value *V = matchSelectFromAndOr(B, D, A, C))
4411 return replaceInstUsesWith(I, V);
4412 if (Value *V = matchSelectFromAndOr(B, D, C, A))
4413 return replaceInstUsesWith(I, V);
4414 if (Value *V = matchSelectFromAndOr(D, B, A, C))
4415 return replaceInstUsesWith(I, V);
4416 if (Value *V = matchSelectFromAndOr(D, B, C, A))
4417 return replaceInstUsesWith(I, V);
4418 }
4419 }
4420
4421 if (match(Op0, m_And(m_Value(A), m_Value(C))) &&
4422 match(Op1, m_Not(m_Or(m_Value(B), m_Value(D)))) &&
4423 (Op0->hasOneUse() || Op1->hasOneUse())) {
4424 // (Cond & C) | ~(Cond | D) -> Cond ? C : ~D
4425 if (Value *V = matchSelectFromAndOr(A, C, B, D, true))
4426 return replaceInstUsesWith(I, V);
4427 if (Value *V = matchSelectFromAndOr(A, C, D, B, true))
4428 return replaceInstUsesWith(I, V);
4429 if (Value *V = matchSelectFromAndOr(C, A, B, D, true))
4430 return replaceInstUsesWith(I, V);
4431 if (Value *V = matchSelectFromAndOr(C, A, D, B, true))
4432 return replaceInstUsesWith(I, V);
4433 }
4434
4435 // (A ^ B) | ((B ^ C) ^ A) -> (A ^ B) | C
4436 if (match(Op0, m_Xor(m_Value(A), m_Value(B))))
4437 if (match(Op1,
4440 return BinaryOperator::CreateOr(Op0, C);
4441
4442 // ((B ^ C) ^ A) | (A ^ B) -> (A ^ B) | C
4443 if (match(Op1, m_Xor(m_Value(A), m_Value(B))))
4444 if (match(Op0,
4447 return BinaryOperator::CreateOr(Op1, C);
4448
4449 if (Instruction *DeMorgan = matchDeMorgansLaws(I, *this))
4450 return DeMorgan;
4451
4452 // Canonicalize xor to the RHS.
4453 bool SwappedForXor = false;
4454 if (match(Op0, m_Xor(m_Value(), m_Value()))) {
4455 std::swap(Op0, Op1);
4456 SwappedForXor = true;
4457 }
4458
4459 if (match(Op1, m_Xor(m_Value(A), m_Value(B)))) {
4460 // (A | ?) | (A ^ B) --> (A | ?) | B
4461 // (B | ?) | (A ^ B) --> (B | ?) | A
4462 if (match(Op0, m_c_Or(m_Specific(A), m_Value())))
4463 return BinaryOperator::CreateOr(Op0, B);
4464 if (match(Op0, m_c_Or(m_Specific(B), m_Value())))
4465 return BinaryOperator::CreateOr(Op0, A);
4466
4467 // (A & B) | (A ^ B) --> A | B
4468 // (B & A) | (A ^ B) --> A | B
4469 if (match(Op0, m_c_And(m_Specific(A), m_Specific(B))))
4470 return BinaryOperator::CreateOr(A, B);
4471
4472 // ~A | (A ^ B) --> ~(A & B)
4473 // ~B | (A ^ B) --> ~(A & B)
4474 // The swap above should always make Op0 the 'not'.
4475 if ((Op0->hasOneUse() || Op1->hasOneUse()) &&
4476 (match(Op0, m_Not(m_Specific(A))) || match(Op0, m_Not(m_Specific(B)))))
4477 return BinaryOperator::CreateNot(Builder.CreateAnd(A, B));
4478
4479 // Same as above, but peek through an 'and' to the common operand:
4480 // ~(A & ?) | (A ^ B) --> ~((A & ?) & B)
4481 // ~(B & ?) | (A ^ B) --> ~((B & ?) & A)
4483 if ((Op0->hasOneUse() || Op1->hasOneUse()) &&
4484 match(Op0,
4486 return BinaryOperator::CreateNot(Builder.CreateAnd(And, B));
4487 if ((Op0->hasOneUse() || Op1->hasOneUse()) &&
4488 match(Op0,
4490 return BinaryOperator::CreateNot(Builder.CreateAnd(And, A));
4491
4492 // (~A | C) | (A ^ B) --> ~(A & B) | C
4493 // (~B | C) | (A ^ B) --> ~(A & B) | C
4494 if (Op0->hasOneUse() && Op1->hasOneUse() &&
4495 (match(Op0, m_c_Or(m_Not(m_Specific(A)), m_Value(C))) ||
4496 match(Op0, m_c_Or(m_Not(m_Specific(B)), m_Value(C))))) {
4497 Value *Nand = Builder.CreateNot(Builder.CreateAnd(A, B), "nand");
4498 return BinaryOperator::CreateOr(Nand, C);
4499 }
4500 }
4501
4502 if (SwappedForXor)
4503 std::swap(Op0, Op1);
4504
4505 if (Value *Res =
4506 foldBooleanAndOr(Op0, Op1, I, /*IsAnd=*/false, /*IsLogical=*/false))
4507 return replaceInstUsesWith(I, Res);
4508
4509 if (match(Op1, m_OneUse(m_LogicalOr(m_Value(X), m_Value(Y))))) {
4510 bool IsLogical = isa<SelectInst>(Op1);
4511 if (auto *V = reassociateBooleanAndOr(Op0, X, Y, I, /*IsAnd=*/false,
4512 /*RHSIsLogical=*/IsLogical))
4513 return replaceInstUsesWith(I, V);
4514 }
4515 if (match(Op0, m_OneUse(m_LogicalOr(m_Value(X), m_Value(Y))))) {
4516 bool IsLogical = isa<SelectInst>(Op0);
4517 if (auto *V = reassociateBooleanAndOr(Op1, X, Y, I, /*IsAnd=*/false,
4518 /*RHSIsLogical=*/IsLogical))
4519 return replaceInstUsesWith(I, V);
4520 }
4521
4522 if (Instruction *FoldedFCmps = reassociateFCmps(I, Builder))
4523 return FoldedFCmps;
4524
4525 if (Instruction *CastedOr = foldCastedBitwiseLogic(I))
4526 return CastedOr;
4527
4528 if (Instruction *Sel = foldBinopOfSextBoolToSelect(I))
4529 return Sel;
4530
4531 // or(sext(A), B) / or(B, sext(A)) --> A ? -1 : B, where A is i1 or <N x i1>.
4532 // TODO: Move this into foldBinopOfSextBoolToSelect as a more generalized fold
4533 // with binop identity constant. But creating a select with non-constant
4534 // arm may not be reversible due to poison semantics. Is that a good
4535 // canonicalization?
4536 if (match(&I, m_c_Or(m_OneUse(m_SExt(m_Value(A))), m_Value(B))) &&
4537 A->getType()->isIntOrIntVectorTy(1))
4538 return createSelectInstWithUnknownProfile(
4540
4541 // Note: If we've gotten to the point of visiting the outer OR, then the
4542 // inner one couldn't be simplified. If it was a constant, then it won't
4543 // be simplified by a later pass either, so we try swapping the inner/outer
4544 // ORs in the hopes that we'll be able to simplify it this way.
4545 // (X|C) | V --> (X|V) | C
4546 // V | (X|C) --> (X|V) | C
4547 // Pass the disjoint flag in the following two patterns:
4548 // 1. or-disjoint (or-disjoint X, C), V -->
4549 // or-disjoint (or-disjoint X, V), C
4550 //
4551 // 2. or-disjoint (or X, C), V -->
4552 // or (or-disjoint X, V), C
4553 ConstantInt *CI;
4554 Instruction *InnerOr;
4555 Value *V;
4557 InnerOr, m_Or(m_Value(A), m_ConstantInt(CI)))),
4558 m_Value(V))) &&
4559 !match(V, m_ConstantInt())) {
4560 bool IsDisjointOuter = cast<PossiblyDisjointInst>(I).isDisjoint();
4561 bool IsDisjointInner = cast<PossiblyDisjointInst>(InnerOr)->isDisjoint();
4562 Value *Inner = Builder.CreateOr(A, V, "", /*IsDisjoint=*/IsDisjointOuter);
4563 Inner->takeName(InnerOr);
4564 return IsDisjointOuter && IsDisjointInner
4565 ? BinaryOperator::CreateDisjointOr(Inner, CI)
4566 : BinaryOperator::CreateOr(Inner, CI);
4567 }
4568
4569 // Change (or (bool?A:B),(bool?C:D)) --> (bool?(or A,C):(or B,D))
4570 // Since this OR statement hasn't been optimized further yet, we hope
4571 // that this transformation will allow the new ORs to be optimized.
4572 {
4573 Value *X = nullptr, *Y = nullptr;
4574 if (Op0->hasOneUse() && Op1->hasOneUse() &&
4575 match(Op0, m_Select(m_Value(X), m_Value(A), m_Value(B))) &&
4576 match(Op1, m_Select(m_Value(Y), m_Value(C), m_Value(D))) && X == Y) {
4577 Value *orTrue = Builder.CreateOr(A, C);
4578 Value *orFalse = Builder.CreateOr(B, D);
4579 return SelectInst::Create(X, orTrue, orFalse);
4580 }
4581 }
4582
4583 // or(ashr(subNSW(Y, X), ScalarSizeInBits(Y) - 1), X) --> X s> Y ? -1 : X.
4584 {
4585 Value *X, *Y;
4588 m_SpecificInt(Ty->getScalarSizeInBits() - 1))),
4589 m_Deferred(X)))) {
4590 Value *NewICmpInst = Builder.CreateICmpSGT(X, Y);
4592 return createSelectInstWithUnknownProfile(NewICmpInst, AllOnes, X);
4593 }
4594 }
4595
4596 {
4597 // ((A & B) ^ A) | ((A & B) ^ B) -> A ^ B
4598 // (A ^ (A & B)) | (B ^ (A & B)) -> A ^ B
4599 // ((A & B) ^ B) | ((A & B) ^ A) -> A ^ B
4600 // (B ^ (A & B)) | (A ^ (A & B)) -> A ^ B
4601 const auto TryXorOpt = [&](Value *Lhs, Value *Rhs) -> Instruction * {
4602 if (match(Lhs, m_c_Xor(m_And(m_Value(A), m_Value(B)), m_Deferred(A))) &&
4603 match(Rhs,
4605 return BinaryOperator::CreateXor(A, B);
4606 }
4607 return nullptr;
4608 };
4609
4610 if (Instruction *Result = TryXorOpt(Op0, Op1))
4611 return Result;
4612 if (Instruction *Result = TryXorOpt(Op1, Op0))
4613 return Result;
4614 }
4615
4616 if (Instruction *V =
4618 return V;
4619
4620 CmpPredicate Pred;
4621 Value *Mul, *Ov, *MulIsNotZero, *UMulWithOv;
4622 // Check if the OR weakens the overflow condition for umul.with.overflow by
4623 // treating any non-zero result as overflow. In that case, we overflow if both
4624 // umul.with.overflow operands are != 0, as in that case the result can only
4625 // be 0, iff the multiplication overflows.
4626 if (match(&I, m_c_Or(m_Value(Ov, m_ExtractValue<1>(m_Value(UMulWithOv))),
4627 m_Value(MulIsNotZero,
4631 m_Deferred(UMulWithOv))),
4632 m_ZeroInt())))) &&
4633 (Ov->hasOneUse() || (MulIsNotZero->hasOneUse() && Mul->hasOneUse()))) {
4634 Value *A, *B;
4636 m_Value(A), m_Value(B)))) {
4637 Value *NotNullA = Builder.CreateIsNotNull(A);
4638 Value *NotNullB = Builder.CreateIsNotNull(B);
4639 return BinaryOperator::CreateAnd(NotNullA, NotNullB);
4640 }
4641 }
4642
4643 /// Res, Overflow = xxx_with_overflow X, C1
4644 /// Try to canonicalize the pattern "Overflow | icmp pred Res, C2" into
4645 /// "Overflow | icmp pred X, C2 +/- C1".
4646 const WithOverflowInst *WO;
4647 const Value *WOV;
4648 const APInt *C1, *C2;
4650 m_Value(WOV, m_WithOverflowInst(WO)))),
4652 m_APInt(C2))))) &&
4653 (WO->getBinaryOp() == Instruction::Add ||
4654 WO->getBinaryOp() == Instruction::Sub) &&
4655 (ICmpInst::isEquality(Pred) ||
4656 WO->isSigned() == ICmpInst::isSigned(Pred)) &&
4657 match(WO->getRHS(), m_APInt(C1))) {
4658 bool Overflow;
4659 APInt NewC = WO->getBinaryOp() == Instruction::Add
4660 ? (ICmpInst::isSigned(Pred) ? C2->ssub_ov(*C1, Overflow)
4661 : C2->usub_ov(*C1, Overflow))
4662 : (ICmpInst::isSigned(Pred) ? C2->sadd_ov(*C1, Overflow)
4663 : C2->uadd_ov(*C1, Overflow));
4664 if (!Overflow || ICmpInst::isEquality(Pred)) {
4665 Value *NewCmp = Builder.CreateICmp(
4666 Pred, WO->getLHS(), ConstantInt::get(WO->getLHS()->getType(), NewC));
4667 return BinaryOperator::CreateOr(Ov, NewCmp);
4668 }
4669 }
4670
4671 // Try to fold the pattern "Overflow | icmp pred Res, C2" into a single
4672 // comparison instruction for umul.with.overflow.
4674 return replaceInstUsesWith(I, R);
4675
4676 // (~x) | y --> ~(x & (~y)) iff that gets rid of inversions
4678 return &I;
4679
4680 // Improve "get low bit mask up to and including bit X" pattern:
4681 // (1 << X) | ((1 << X) + -1) --> -1 l>> (bitwidth(x) - 1 - X)
4682 if (match(&I, m_c_Or(m_Add(m_Shl(m_One(), m_Value(X)), m_AllOnes()),
4683 m_Shl(m_One(), m_Deferred(X)))) &&
4684 match(&I, m_c_Or(m_OneUse(m_Value()), m_Value()))) {
4685 Value *Sub = Builder.CreateSub(
4686 ConstantInt::get(Ty, Ty->getScalarSizeInBits() - 1), X);
4687 return BinaryOperator::CreateLShr(Constant::getAllOnesValue(Ty), Sub);
4688 }
4689
4690 // An or recurrence w/loop invariant step is equivelent to (or start, step)
4691 PHINode *PN = nullptr;
4692 Value *Start = nullptr, *Step = nullptr;
4693 if (matchSimpleRecurrence(&I, PN, Start, Step) && DT.dominates(Step, PN))
4694 return replaceInstUsesWith(I, Builder.CreateOr(Start, Step));
4695
4696 // (A & B) | (C | D) or (C | D) | (A & B)
4697 // Can be combined if C or D is of type (A/B & X)
4699 m_OneUse(m_Or(m_Value(C), m_Value(D)))))) {
4700 // (A & B) | (C | ?) -> C | (? | (A & B))
4701 // (A & B) | (C | ?) -> C | (? | (A & B))
4702 // (A & B) | (C | ?) -> C | (? | (A & B))
4703 // (A & B) | (C | ?) -> C | (? | (A & B))
4704 // (C | ?) | (A & B) -> C | (? | (A & B))
4705 // (C | ?) | (A & B) -> C | (? | (A & B))
4706 // (C | ?) | (A & B) -> C | (? | (A & B))
4707 // (C | ?) | (A & B) -> C | (? | (A & B))
4708 if (match(D, m_OneUse(m_c_And(m_Specific(A), m_Value()))) ||
4710 return BinaryOperator::CreateOr(
4711 C, Builder.CreateOr(D, Builder.CreateAnd(A, B)));
4712 // (A & B) | (? | D) -> (? | (A & B)) | D
4713 // (A & B) | (? | D) -> (? | (A & B)) | D
4714 // (A & B) | (? | D) -> (? | (A & B)) | D
4715 // (A & B) | (? | D) -> (? | (A & B)) | D
4716 // (? | D) | (A & B) -> (? | (A & B)) | D
4717 // (? | D) | (A & B) -> (? | (A & B)) | D
4718 // (? | D) | (A & B) -> (? | (A & B)) | D
4719 // (? | D) | (A & B) -> (? | (A & B)) | D
4720 if (match(C, m_OneUse(m_c_And(m_Specific(A), m_Value()))) ||
4722 return BinaryOperator::CreateOr(
4723 Builder.CreateOr(C, Builder.CreateAnd(A, B)), D);
4724 }
4725
4727 return R;
4728
4729 if (Instruction *Canonicalized = canonicalizeLogicFirst(I, Builder))
4730 return Canonicalized;
4731
4732 if (Instruction *Folded = foldLogicOfIsFPClass(I, Op0, Op1))
4733 return Folded;
4734
4735 if (Instruction *Res = foldBinOpOfDisplacedShifts(I))
4736 return Res;
4737
4738 // If we are setting the sign bit of a floating-point value, convert
4739 // this to fneg(fabs), then cast back to integer.
4740 //
4741 // If the result isn't immediately cast back to a float, this will increase
4742 // the number of instructions. This is still probably a better canonical form
4743 // as it enables FP value tracking.
4744 //
4745 // Assumes any IEEE-represented type has the sign bit in the high bit.
4746 //
4747 // This is generous interpretation of noimplicitfloat, this is not a true
4748 // floating-point operation.
4749 Value *CastOp;
4750 if (match(Op0, m_ElementWiseBitCast(m_Value(CastOp))) &&
4751 match(Op1, m_SignMask()) &&
4752 !Builder.GetInsertBlock()->getParent()->hasFnAttribute(
4753 Attribute::NoImplicitFloat)) {
4754 Type *EltTy = CastOp->getType()->getScalarType();
4755 if (EltTy->isFloatingPointTy() &&
4757 Value *FAbs = Builder.CreateFAbs(CastOp);
4758 Value *FNegFAbs = Builder.CreateFNeg(FAbs);
4759 return new BitCastInst(FNegFAbs, I.getType());
4760 }
4761 }
4762
4763 // (X & C1) | C2 -> X & (C1 | C2) iff (X & C2) == C2
4764 if (match(Op0, m_OneUse(m_And(m_Value(X), m_APInt(C1)))) &&
4765 match(Op1, m_APInt(C2))) {
4766 KnownBits KnownX = computeKnownBits(X, &I);
4767 if ((KnownX.One & *C2) == *C2)
4768 return BinaryOperator::CreateAnd(X, ConstantInt::get(Ty, *C1 | *C2));
4769 }
4770
4772 return Res;
4773
4774 if (Value *V =
4776 /*SimplifyOnly*/ false, *this))
4777 return BinaryOperator::CreateOr(V, Op1);
4778 if (Value *V =
4780 /*SimplifyOnly*/ false, *this))
4781 return BinaryOperator::CreateOr(Op0, V);
4782
4783 if (cast<PossiblyDisjointInst>(I).isDisjoint())
4785 return replaceInstUsesWith(I, V);
4786
4788 return replaceInstUsesWith(I, Res);
4789
4790 // signum: or (ashr X, BW-1), zext (icmp ne|sgt X, 0) --> scmp(X, 0)
4791 // The ashr already supplies -1 for negative X, so any predicate that
4792 // produces 1 for positive X and 0 for X == 0 yields the same result here.
4793 {
4794 Value *X;
4795 CmpPredicate SignPred;
4796 unsigned BitWidth = Ty->getScalarSizeInBits();
4797 if (match(&I,
4799 m_ZExt(m_ICmp(SignPred, m_Deferred(X), m_ZeroInt())))) &&
4800 (SignPred == ICmpInst::ICMP_NE || SignPred == ICmpInst::ICMP_SGT) &&
4801 (Op0->hasOneUse() || Op1->hasOneUse()))
4802 return replaceInstUsesWith(
4803 I, Builder.CreateIntrinsic(Ty, Intrinsic::scmp,
4804 {X, Constant::getNullValue(Ty)}));
4805 }
4806
4807 return nullptr;
4808}
4809
4810/// A ^ B can be specified using other logic ops in a variety of patterns. We
4811/// can fold these early and efficiently by morphing an existing instruction.
4813 InstCombiner::BuilderTy &Builder) {
4814 assert(I.getOpcode() == Instruction::Xor);
4815 Value *Op0 = I.getOperand(0);
4816 Value *Op1 = I.getOperand(1);
4817 Value *A, *B;
4818
4819 // There are 4 commuted variants for each of the basic patterns.
4820
4821 // (A & B) ^ (A | B) -> A ^ B
4822 // (A & B) ^ (B | A) -> A ^ B
4823 // (A | B) ^ (A & B) -> A ^ B
4824 // (A | B) ^ (B & A) -> A ^ B
4825 if (match(&I, m_c_Xor(m_And(m_Value(A), m_Value(B)),
4827 return BinaryOperator::CreateXor(A, B);
4828
4829 // (A | ~B) ^ (~A | B) -> A ^ B
4830 // (~B | A) ^ (~A | B) -> A ^ B
4831 // (~A | B) ^ (A | ~B) -> A ^ B
4832 // (B | ~A) ^ (A | ~B) -> A ^ B
4833 if (match(&I, m_Xor(m_c_Or(m_Value(A), m_Not(m_Value(B))),
4835 return BinaryOperator::CreateXor(A, B);
4836
4837 // (A & ~B) ^ (~A & B) -> A ^ B
4838 // (~B & A) ^ (~A & B) -> A ^ B
4839 // (~A & B) ^ (A & ~B) -> A ^ B
4840 // (B & ~A) ^ (A & ~B) -> A ^ B
4841 if (match(&I, m_Xor(m_c_And(m_Value(A), m_Not(m_Value(B))),
4843 return BinaryOperator::CreateXor(A, B);
4844
4845 // For the remaining cases we need to get rid of one of the operands.
4846 if (!Op0->hasOneUse() && !Op1->hasOneUse())
4847 return nullptr;
4848
4849 // (A | B) ^ ~(A & B) -> ~(A ^ B)
4850 // (A | B) ^ ~(B & A) -> ~(A ^ B)
4851 // (A & B) ^ ~(A | B) -> ~(A ^ B)
4852 // (A & B) ^ ~(B | A) -> ~(A ^ B)
4853 // Complexity sorting ensures the not will be on the right side.
4854 if ((match(Op0, m_Or(m_Value(A), m_Value(B))) &&
4855 match(Op1, m_Not(m_c_And(m_Specific(A), m_Specific(B))))) ||
4856 (match(Op0, m_And(m_Value(A), m_Value(B))) &&
4858 return BinaryOperator::CreateNot(Builder.CreateXor(A, B));
4859
4860 return nullptr;
4861}
4862
4863Value *InstCombinerImpl::foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS,
4864 BinaryOperator &I) {
4865 assert(I.getOpcode() == Instruction::Xor && I.getOperand(0) == LHS &&
4866 I.getOperand(1) == RHS && "Should be 'xor' with these operands");
4867
4868 ICmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
4869 Value *LHS0 = LHS->getOperand(0), *LHS1 = LHS->getOperand(1);
4870 Value *RHS0 = RHS->getOperand(0), *RHS1 = RHS->getOperand(1);
4871
4872 if (predicatesFoldable(PredL, PredR)) {
4873 if (LHS0 == RHS1 && LHS1 == RHS0) {
4874 std::swap(LHS0, LHS1);
4875 PredL = ICmpInst::getSwappedPredicate(PredL);
4876 }
4877 if (LHS0 == RHS0 && LHS1 == RHS1) {
4878 // (icmp1 A, B) ^ (icmp2 A, B) --> (icmp3 A, B)
4879 unsigned Code = getICmpCode(PredL) ^ getICmpCode(PredR);
4880 bool IsSigned = LHS->isSigned() || RHS->isSigned();
4881 return getNewICmpValue(Code, IsSigned, LHS0, LHS1, Builder);
4882 }
4883 }
4884
4885 const APInt *LC, *RC;
4886 if (match(LHS1, m_APInt(LC)) && match(RHS1, m_APInt(RC)) &&
4887 LHS0->getType() == RHS0->getType() &&
4888 LHS0->getType()->isIntOrIntVectorTy()) {
4889 // Convert xor of signbit tests to signbit test of xor'd values:
4890 // (X > -1) ^ (Y > -1) --> (X ^ Y) < 0
4891 // (X < 0) ^ (Y < 0) --> (X ^ Y) < 0
4892 // (X > -1) ^ (Y < 0) --> (X ^ Y) > -1
4893 // (X < 0) ^ (Y > -1) --> (X ^ Y) > -1
4894 bool TrueIfSignedL, TrueIfSignedR;
4895 if ((LHS->hasOneUse() || RHS->hasOneUse()) &&
4896 isSignBitCheck(PredL, *LC, TrueIfSignedL) &&
4897 isSignBitCheck(PredR, *RC, TrueIfSignedR)) {
4898 Value *XorLR = Builder.CreateXor(LHS0, RHS0);
4899 return TrueIfSignedL == TrueIfSignedR ? Builder.CreateIsNeg(XorLR) :
4900 Builder.CreateIsNotNeg(XorLR);
4901 }
4902
4903 // Fold (icmp pred1 X, C1) ^ (icmp pred2 X, C2)
4904 // into a single comparison using range-based reasoning.
4905 if (LHS0 == RHS0) {
4906 ConstantRange CR1 = ConstantRange::makeExactICmpRegion(PredL, *LC);
4907 ConstantRange CR2 = ConstantRange::makeExactICmpRegion(PredR, *RC);
4908 auto CRUnion = CR1.exactUnionWith(CR2);
4909 auto CRIntersect = CR1.exactIntersectWith(CR2);
4910 if (CRUnion && CRIntersect)
4911 if (auto CR = CRUnion->exactIntersectWith(CRIntersect->inverse())) {
4912 if (CR->isFullSet())
4913 return ConstantInt::getTrue(I.getType());
4914 if (CR->isEmptySet())
4915 return ConstantInt::getFalse(I.getType());
4916
4917 CmpInst::Predicate NewPred;
4918 APInt NewC, Offset;
4919 CR->getEquivalentICmp(NewPred, NewC, Offset);
4920
4921 if ((Offset.isZero() && (LHS->hasOneUse() || RHS->hasOneUse())) ||
4922 (LHS->hasOneUse() && RHS->hasOneUse())) {
4923 Value *NewV = LHS0;
4924 Type *Ty = LHS0->getType();
4925 if (!Offset.isZero())
4926 NewV = Builder.CreateAdd(NewV, ConstantInt::get(Ty, Offset));
4927 return Builder.CreateICmp(NewPred, NewV,
4928 ConstantInt::get(Ty, NewC));
4929 }
4930 }
4931 }
4932
4933 // Fold (icmp eq/ne (X & Pow2), 0) ^ (icmp eq/ne (Y & Pow2), 0) into
4934 // (icmp eq/ne ((X ^ Y) & Pow2), 0)
4935 Value *X, *Y, *Pow2;
4936 if (ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) &&
4937 LC->isZero() && RC->isZero() && LHS->hasOneUse() && RHS->hasOneUse() &&
4938 match(LHS0, m_And(m_Value(X), m_Value(Pow2))) &&
4939 match(RHS0, m_And(m_Value(Y), m_Specific(Pow2))) &&
4940 isKnownToBeAPowerOfTwo(Pow2, /*OrZero=*/true, &I)) {
4941 Value *Xor = Builder.CreateXor(X, Y);
4942 Value *And = Builder.CreateAnd(Xor, Pow2);
4943 return Builder.CreateICmp(PredL == PredR ? ICmpInst::ICMP_NE
4945 And, ConstantInt::getNullValue(Xor->getType()));
4946 }
4947 }
4948
4949 // Instead of trying to imitate the folds for and/or, decompose this 'xor'
4950 // into those logic ops. That is, try to turn this into an and-of-icmps
4951 // because we have many folds for that pattern.
4952 //
4953 // This is based on a truth table definition of xor:
4954 // X ^ Y --> (X | Y) & !(X & Y)
4955 if (Value *OrICmp = simplifyBinOp(Instruction::Or, LHS, RHS, SQ)) {
4956 // TODO: If OrICmp is true, then the definition of xor simplifies to !(X&Y).
4957 // TODO: If OrICmp is false, the whole thing is false (InstSimplify?).
4958 if (Value *AndICmp = simplifyBinOp(Instruction::And, LHS, RHS, SQ)) {
4959 // TODO: Independently handle cases where the 'and' side is a constant.
4960 ICmpInst *X = nullptr, *Y = nullptr;
4961 if (OrICmp == LHS && AndICmp == RHS) {
4962 // (LHS | RHS) & !(LHS & RHS) --> LHS & !RHS --> X & !Y
4963 X = LHS;
4964 Y = RHS;
4965 }
4966 if (OrICmp == RHS && AndICmp == LHS) {
4967 // !(LHS & RHS) & (LHS | RHS) --> !LHS & RHS --> !Y & X
4968 X = RHS;
4969 Y = LHS;
4970 }
4971 if (X && Y && (Y->hasOneUse() || canFreelyInvertAllUsersOf(Y, &I))) {
4972 // Invert the predicate of 'Y', thus inverting its output.
4973 Y->setPredicate(Y->getInversePredicate());
4974 // So, are there other uses of Y?
4975 if (!Y->hasOneUse()) {
4976 // We need to adapt other uses of Y though. Get a value that matches
4977 // the original value of Y before inversion. While this increases
4978 // immediate instruction count, we have just ensured that all the
4979 // users are freely-invertible, so that 'not' *will* get folded away.
4981 // Set insertion point to right after the Y.
4982 Builder.SetInsertPoint(++(Y->getIterator()));
4983 Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not");
4984 // Replace all uses of Y (excluding the one in NotY!) with NotY.
4985 Worklist.pushUsersToWorkList(*Y);
4986 Y->replaceUsesWithIf(NotY,
4987 [NotY](Use &U) { return U.getUser() != NotY; });
4988 }
4989 // All done.
4990 return Builder.CreateAnd(LHS, RHS);
4991 }
4992 }
4993 }
4994
4995 return nullptr;
4996}
4997
4998/// If we have a masked merge, in the canonical form of:
4999/// (assuming that A only has one use.)
5000/// | A | |B|
5001/// ((x ^ y) & M) ^ y
5002/// | D |
5003/// * If M is inverted:
5004/// | D |
5005/// ((x ^ y) & ~M) ^ y
5006/// We can canonicalize by swapping the final xor operand
5007/// to eliminate the 'not' of the mask.
5008/// ((x ^ y) & M) ^ x
5009/// * If M is a constant, and D has one use, we transform to 'and' / 'or' ops
5010/// because that shortens the dependency chain and improves analysis:
5011/// (x & M) | (y & ~M)
5013 InstCombiner::BuilderTy &Builder) {
5014 Value *B, *X, *D;
5015 Value *M;
5016 if (!match(&I, m_c_Xor(m_Value(B),
5019 m_Value(M))))))
5020 return nullptr;
5021
5022 Value *NotM;
5023 if (match(M, m_Not(m_Value(NotM)))) {
5024 // De-invert the mask and swap the value in B part.
5025 Value *NewA = Builder.CreateAnd(D, NotM);
5026 return BinaryOperator::CreateXor(NewA, X);
5027 }
5028
5029 Constant *C;
5030 if (D->hasOneUse() && match(M, m_Constant(C))) {
5031 // Propagating undef is unsafe. Clamp undef elements to -1.
5032 Type *EltTy = C->getType()->getScalarType();
5034 // Unfold.
5035 Value *LHS = Builder.CreateAnd(X, C);
5036 Value *NotC = Builder.CreateNot(C);
5037 Value *RHS = Builder.CreateAnd(B, NotC);
5038 return BinaryOperator::CreateOr(LHS, RHS);
5039 }
5040
5041 return nullptr;
5042}
5043
5045 InstCombiner::BuilderTy &Builder) {
5046 Value *X, *Y;
5047 // FIXME: one-use check is not needed in general, but currently we are unable
5048 // to fold 'not' into 'icmp', if that 'icmp' has multiple uses. (D35182)
5049 if (!match(&I, m_Not(m_OneUse(m_Xor(m_Value(X), m_Value(Y))))))
5050 return nullptr;
5051
5052 auto hasCommonOperand = [](Value *A, Value *B, Value *C, Value *D) {
5053 return A == C || A == D || B == C || B == D;
5054 };
5055
5056 Value *A, *B, *C, *D;
5057 // Canonicalize ~((A & B) ^ (A | ?)) -> (A & B) | ~(A | ?)
5058 // 4 commuted variants
5059 if (match(X, m_And(m_Value(A), m_Value(B))) &&
5060 match(Y, m_Or(m_Value(C), m_Value(D))) && hasCommonOperand(A, B, C, D)) {
5061 Value *NotY = Builder.CreateNot(Y);
5062 return BinaryOperator::CreateOr(X, NotY);
5063 };
5064
5065 // Canonicalize ~((A | ?) ^ (A & B)) -> (A & B) | ~(A | ?)
5066 // 4 commuted variants
5067 if (match(Y, m_And(m_Value(A), m_Value(B))) &&
5068 match(X, m_Or(m_Value(C), m_Value(D))) && hasCommonOperand(A, B, C, D)) {
5069 Value *NotX = Builder.CreateNot(X);
5070 return BinaryOperator::CreateOr(Y, NotX);
5071 };
5072
5073 return nullptr;
5074}
5075
5076/// Canonicalize a shifty way to code absolute value to the more common pattern
5077/// that uses negation and select.
5079 InstCombiner::BuilderTy &Builder) {
5080 assert(Xor.getOpcode() == Instruction::Xor && "Expected an xor instruction.");
5081
5082 // There are 4 potential commuted variants. Move the 'ashr' candidate to Op1.
5083 // We're relying on the fact that we only do this transform when the shift has
5084 // exactly 2 uses and the add has exactly 1 use (otherwise, we might increase
5085 // instructions).
5086 Value *Op0 = Xor.getOperand(0), *Op1 = Xor.getOperand(1);
5087 if (Op0->hasNUses(2))
5088 std::swap(Op0, Op1);
5089
5090 Type *Ty = Xor.getType();
5091 Value *A;
5092 const APInt *ShAmt;
5093 if (match(Op1, m_AShr(m_Value(A), m_APInt(ShAmt))) &&
5094 Op1->hasNUses(2) && *ShAmt == Ty->getScalarSizeInBits() - 1 &&
5095 match(Op0, m_OneUse(m_c_Add(m_Specific(A), m_Specific(Op1))))) {
5096 // Op1 = ashr i32 A, 31 ; smear the sign bit
5097 // xor (add A, Op1), Op1 ; add -1 and flip bits if negative
5098 // --> (A < 0) ? -A : A
5099 Value *IsNeg = Builder.CreateIsNeg(A);
5100 // Copy the nsw flags from the add to the negate.
5101 auto *Add = cast<BinaryOperator>(Op0);
5102 Value *NegA = Add->hasNoUnsignedWrap()
5103 ? Constant::getNullValue(A->getType())
5104 : Builder.CreateNeg(A, "", Add->hasNoSignedWrap());
5105 return SelectInst::Create(IsNeg, NegA, A);
5106 }
5107 return nullptr;
5108}
5109
5111 Instruction *IgnoredUser) {
5112 auto *I = dyn_cast<Instruction>(Op);
5113 return I && I->getInsertionPointAfterDef() &&
5114 IC.isFreeToInvert(I, /*WillInvertAllUses=*/true) &&
5115 IC.canFreelyInvertAllUsersOf(I, IgnoredUser);
5116}
5117
5119 Instruction *IgnoredUser) {
5120 auto *I = cast<Instruction>(Op);
5121 auto InsertPt = I->getInsertionPointAfterDef();
5122 assert(InsertPt &&
5123 "freelyInvert requires an instruction with a valid insertion point");
5124 IC.Builder.SetInsertPoint(*InsertPt);
5125 Value *NotOp = IC.Builder.CreateNot(Op, Op->getName() + ".not");
5126 Op->replaceUsesWithIf(NotOp,
5127 [NotOp](Use &U) { return U.getUser() != NotOp; });
5128 IC.freelyInvertAllUsersOf(NotOp, IgnoredUser);
5129 return NotOp;
5130}
5131
5132// Transform
5133// z = ~(x &/| y)
5134// into:
5135// z = ((~x) |/& (~y))
5136// iff both x and y are free to invert and all uses of z can be freely updated.
5138 Value *Op0, *Op1;
5139 if (!match(&I, m_LogicalOp(m_Value(Op0), m_Value(Op1))))
5140 return false;
5141
5142 // If this logic op has not been simplified yet, just bail out and let that
5143 // happen first. Otherwise, the code below may wrongly invert.
5144 if (Op0 == Op1)
5145 return false;
5146
5147 // If one of the operands is a user of the other,
5148 // freelyInvert->freelyInvertAllUsersOf will change the operands of I, which
5149 // may cause miscompilation.
5150 if (match(Op0, m_Not(m_Specific(Op1))) || match(Op1, m_Not(m_Specific(Op0))))
5151 return false;
5152
5153 Instruction::BinaryOps NewOpc =
5154 match(&I, m_LogicalAnd()) ? Instruction::Or : Instruction::And;
5155 bool IsBinaryOp = isa<BinaryOperator>(I);
5156
5157 // Can our users be adapted?
5158 if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr))
5159 return false;
5160
5161 // And can the operands be adapted?
5162 if (!canFreelyInvert(*this, Op0, &I) || !canFreelyInvert(*this, Op1, &I))
5163 return false;
5164
5165 Op0 = freelyInvert(*this, Op0, &I);
5166 Op1 = freelyInvert(*this, Op1, &I);
5167
5168 auto InsertPt = I.getInsertionPointAfterDef();
5169 assert(InsertPt && "sinkNotIntoLogicalOp requires an instruction with a "
5170 "valid insertion point");
5171 Builder.SetInsertPoint(*InsertPt);
5172 Value *NewLogicOp;
5173 if (IsBinaryOp) {
5174 NewLogicOp = Builder.CreateBinOp(NewOpc, Op0, Op1, I.getName() + ".not");
5175 } else {
5176 NewLogicOp =
5177 Builder.CreateLogicalOp(NewOpc, Op0, Op1, I.getName() + ".not", &I);
5178 if (SelectInst *SI = dyn_cast<SelectInst>(NewLogicOp))
5179 SI->swapProfMetadata();
5180 }
5181
5182 replaceInstUsesWith(I, NewLogicOp);
5183 // We can not just create an outer `not`, it will most likely be immediately
5184 // folded back, reconstructing our initial pattern, and causing an
5185 // infinite combine loop, so immediately manually fold it away.
5186 freelyInvertAllUsersOf(NewLogicOp);
5187 return true;
5188}
5189
5190// Transform
5191// z = (~x) &/| y
5192// into:
5193// z = ~(x |/& (~y))
5194// iff y is free to invert and all uses of z can be freely updated.
5196 Value *Op0, *Op1;
5197 if (!match(&I, m_LogicalOp(m_Value(Op0), m_Value(Op1))))
5198 return false;
5199 Instruction::BinaryOps NewOpc =
5200 match(&I, m_LogicalAnd()) ? Instruction::Or : Instruction::And;
5201 bool IsBinaryOp = isa<BinaryOperator>(I);
5202
5203 Value *NotOp0 = nullptr;
5204 Value *NotOp1 = nullptr;
5205 Value **OpToInvert = nullptr;
5206 if (match(Op0, m_Not(m_Value(NotOp0))) && canFreelyInvert(*this, Op1, &I)) {
5207 Op0 = NotOp0;
5208 OpToInvert = &Op1;
5209 } else if (match(Op1, m_Not(m_Value(NotOp1))) &&
5210 canFreelyInvert(*this, Op0, &I)) {
5211 Op1 = NotOp1;
5212 OpToInvert = &Op0;
5213 } else
5214 return false;
5215
5216 // If the kept operand is defined as NOT(OpToInvert), freelyInvert(OpToInvert)
5217 // will also flip the kept operand as a side effect of updating its uses,
5218 // invalidating the assumption that it stays fixed while I is rewriten.
5219 if (match(*OpToInvert == Op1 ? Op0 : Op1, m_Not(m_Specific(*OpToInvert))))
5220 return false;
5221
5222 // And can our users be adapted?
5223 if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr))
5224 return false;
5225
5226 *OpToInvert = freelyInvert(*this, *OpToInvert, &I);
5227
5228 Builder.SetInsertPoint(*I.getInsertionPointAfterDef());
5229 Value *NewBinOp;
5230 if (IsBinaryOp) {
5231 NewBinOp = Builder.CreateBinOp(NewOpc, Op0, Op1, I.getName() + ".not");
5232 } else {
5233 NewBinOp =
5234 Builder.CreateLogicalOp(NewOpc, Op0, Op1, I.getName() + ".not",
5235 ProfcheckDisableMetadataFixes ? nullptr : &I);
5236 if (auto *NewSI = dyn_cast<SelectInst>(NewBinOp))
5237 NewSI->swapProfMetadata();
5238 }
5239 replaceInstUsesWith(I, NewBinOp);
5240 // We can not just create an outer `not`, it will most likely be immediately
5241 // folded back, reconstructing our initial pattern, and causing an
5242 // infinite combine loop, so immediately manually fold it away.
5243 freelyInvertAllUsersOf(NewBinOp);
5244 return true;
5245}
5246
5247Instruction *InstCombinerImpl::foldNot(BinaryOperator &I) {
5248 Value *NotOp;
5249 if (!match(&I, m_Not(m_Value(NotOp))))
5250 return nullptr;
5251
5252 // Apply DeMorgan's Law for 'nand' / 'nor' logic with an inverted operand.
5253 // We must eliminate the and/or (one-use) for these transforms to not increase
5254 // the instruction count.
5255 //
5256 // ~(~X & Y) --> (X | ~Y)
5257 // ~(Y & ~X) --> (X | ~Y)
5258 //
5259 // Note: The logical matches do not check for the commuted patterns because
5260 // those are handled via SimplifySelectsFeedingBinaryOp().
5261 Type *Ty = I.getType();
5262 Value *X, *Y;
5263 if (match(NotOp, m_OneUse(m_c_And(m_Not(m_Value(X)), m_Value(Y))))) {
5264 Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not");
5265 return BinaryOperator::CreateOr(X, NotY);
5266 }
5267 if (match(NotOp, m_OneUse(m_LogicalAnd(m_Not(m_Value(X)), m_Value(Y))))) {
5268 Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not");
5270 nullptr, cast<Instruction>(NotOp));
5271 SI->swapProfMetadata();
5272 return SI;
5273 }
5274
5275 // ~(~X | Y) --> (X & ~Y)
5276 // ~(Y | ~X) --> (X & ~Y)
5277 if (match(NotOp, m_OneUse(m_c_Or(m_Not(m_Value(X)), m_Value(Y))))) {
5278 Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not");
5279 return BinaryOperator::CreateAnd(X, NotY);
5280 }
5281 if (match(NotOp, m_OneUse(m_LogicalOr(m_Not(m_Value(X)), m_Value(Y))))) {
5282 Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not");
5283 SelectInst *SI = SelectInst::Create(X, NotY, ConstantInt::getFalse(Ty), "",
5284 nullptr, cast<Instruction>(NotOp));
5285 SI->swapProfMetadata();
5286 return SI;
5287 }
5288
5289 // Is this a 'not' (~) fed by a binary operator?
5290 BinaryOperator *NotVal;
5291 if (match(NotOp, m_BinOp(NotVal))) {
5292 // ~((-X) | Y) --> (X - 1) & (~Y)
5293 if (match(NotVal,
5295 Value *DecX = Builder.CreateAdd(X, ConstantInt::getAllOnesValue(Ty));
5296 Value *NotY = Builder.CreateNot(Y);
5297 return BinaryOperator::CreateAnd(DecX, NotY);
5298 }
5299
5300 // ~(~X >>s Y) --> (X >>s Y)
5301 if (match(NotVal, m_AShr(m_Not(m_Value(X)), m_Value(Y))))
5302 return BinaryOperator::CreateAShr(X, Y);
5303
5304 // Treat lshr with non-negative operand as ashr.
5305 // ~(~X >>u Y) --> (X >>s Y) iff X is known negative
5306 if (match(NotVal, m_LShr(m_Not(m_Value(X)), m_Value(Y))) &&
5307 isKnownNegative(X, SQ.getWithInstruction(NotVal)))
5308 return BinaryOperator::CreateAShr(X, Y);
5309
5310 // Bit-hack form of a signbit test for iN type:
5311 // ~(X >>s (N - 1)) --> sext i1 (X > -1) to iN
5312 unsigned FullShift = Ty->getScalarSizeInBits() - 1;
5313 if (match(NotVal, m_OneUse(m_AShr(m_Value(X), m_SpecificInt(FullShift))))) {
5314 Value *IsNotNeg = Builder.CreateIsNotNeg(X, "isnotneg");
5315 return new SExtInst(IsNotNeg, Ty);
5316 }
5317
5318 // If we are inverting a right-shifted constant, we may be able to eliminate
5319 // the 'not' by inverting the constant and using the opposite shift type.
5320 // Canonicalization rules ensure that only a negative constant uses 'ashr',
5321 // but we must check that in case that transform has not fired yet.
5322
5323 // ~(C >>s Y) --> ~C >>u Y (when inverting the replicated sign bits)
5324 Constant *C;
5325 if (match(NotVal, m_AShr(m_Constant(C), m_Value(Y))) &&
5326 match(C, m_Negative()))
5327 return BinaryOperator::CreateLShr(ConstantExpr::getNot(C), Y);
5328
5329 // ~(C >>u Y) --> ~C >>s Y (when inverting the replicated sign bits)
5330 if (match(NotVal, m_LShr(m_Constant(C), m_Value(Y))) &&
5331 match(C, m_NonNegative()))
5332 return BinaryOperator::CreateAShr(ConstantExpr::getNot(C), Y);
5333
5334 // ~(X + C) --> ~C - X
5335 if (match(NotVal, m_Add(m_Value(X), m_ImmConstant(C))))
5336 return BinaryOperator::CreateSub(ConstantExpr::getNot(C), X);
5337
5338 // ~(X - Y) --> ~X + Y
5339 // FIXME: is it really beneficial to sink the `not` here?
5340 if (match(NotVal, m_Sub(m_Value(X), m_Value(Y))))
5341 if (isa<Constant>(X) || NotVal->hasOneUse())
5342 return BinaryOperator::CreateAdd(Builder.CreateNot(X), Y);
5343
5344 // ~(~X + Y) --> X - Y
5345 if (match(NotVal, m_c_Add(m_Not(m_Value(X)), m_Value(Y))))
5346 return BinaryOperator::CreateWithCopiedFlags(Instruction::Sub, X, Y,
5347 NotVal);
5348 }
5349
5350 // not (cmp A, B) = !cmp A, B
5351 CmpPredicate Pred;
5352 if (match(NotOp, m_Cmp(Pred, m_Value(), m_Value())) &&
5353 (NotOp->hasOneUse() ||
5355 /*IgnoredUser=*/nullptr))) {
5356 cast<CmpInst>(NotOp)->setPredicate(CmpInst::getInversePredicate(Pred));
5358 return &I;
5359 }
5360
5361 // not (bitcast (cmp A, B) --> bitcast (!cmp A, B)
5362 if (match(NotOp, m_OneUse(m_BitCast(m_Value(X)))) &&
5363 match(X, m_OneUse(m_Cmp(Pred, m_Value(), m_Value())))) {
5365 return new BitCastInst(X, Ty);
5366 }
5367
5368 // Move a 'not' ahead of casts of a bool to enable logic reduction:
5369 // not (bitcast (sext i1 X)) --> bitcast (sext (not i1 X))
5370 if (match(NotOp, m_OneUse(m_BitCast(m_OneUse(m_SExt(m_Value(X)))))) &&
5371 X->getType()->isIntOrIntVectorTy(1)) {
5372 Type *SextTy = cast<BitCastOperator>(NotOp)->getSrcTy();
5373 Value *NotX = Builder.CreateNot(X);
5374 Value *Sext = Builder.CreateSExt(NotX, SextTy);
5375 return new BitCastInst(Sext, Ty);
5376 }
5377
5378 if (auto *NotOpI = dyn_cast<Instruction>(NotOp))
5379 if (sinkNotIntoLogicalOp(*NotOpI))
5380 return &I;
5381
5382 // Eliminate a bitwise 'not' op of 'not' min/max by inverting the min/max:
5383 // ~min(~X, ~Y) --> max(X, Y)
5384 // ~max(~X, Y) --> min(X, ~Y)
5385 auto *II = dyn_cast<IntrinsicInst>(NotOp);
5386 if (II && II->hasOneUse()) {
5387 if (match(NotOp, m_c_MaxOrMin(m_Not(m_Value(X)), m_Value(Y)))) {
5388 Intrinsic::ID InvID = getInverseMinMaxIntrinsic(II->getIntrinsicID());
5389 Value *NotY = Builder.CreateNot(Y);
5390 Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, X, NotY);
5391 return replaceInstUsesWith(I, InvMaxMin);
5392 }
5393
5394 if (II->getIntrinsicID() == Intrinsic::is_fpclass) {
5395 ConstantInt *ClassMask = cast<ConstantInt>(II->getArgOperand(1));
5396 II->setArgOperand(
5397 1, ConstantInt::get(ClassMask->getType(),
5398 ~ClassMask->getZExtValue() & fcAllFlags));
5399 return replaceInstUsesWith(I, II);
5400 }
5401 }
5402
5403 if (NotOp->hasOneUse()) {
5404 // Pull 'not' into operands of select if both operands are one-use compares
5405 // or one is one-use compare and the other one is a constant.
5406 // Inverting the predicates eliminates the 'not' operation.
5407 // Example:
5408 // not (select ?, (cmp TPred, ?, ?), (cmp FPred, ?, ?) -->
5409 // select ?, (cmp InvTPred, ?, ?), (cmp InvFPred, ?, ?)
5410 // not (select ?, (cmp TPred, ?, ?), true -->
5411 // select ?, (cmp InvTPred, ?, ?), false
5412 if (auto *Sel = dyn_cast<SelectInst>(NotOp)) {
5413 Value *TV = Sel->getTrueValue();
5414 Value *FV = Sel->getFalseValue();
5415 auto *CmpT = dyn_cast<CmpInst>(TV);
5416 auto *CmpF = dyn_cast<CmpInst>(FV);
5417 bool InvertibleT = (CmpT && CmpT->hasOneUse()) || isa<Constant>(TV);
5418 bool InvertibleF = (CmpF && CmpF->hasOneUse()) || isa<Constant>(FV);
5419 if (InvertibleT && InvertibleF) {
5420 if (CmpT)
5421 CmpT->setPredicate(CmpT->getInversePredicate());
5422 else
5423 Sel->setTrueValue(ConstantExpr::getNot(cast<Constant>(TV)));
5424 if (CmpF)
5425 CmpF->setPredicate(CmpF->getInversePredicate());
5426 else
5427 Sel->setFalseValue(ConstantExpr::getNot(cast<Constant>(FV)));
5428 return replaceInstUsesWith(I, Sel);
5429 }
5430 }
5431 }
5432
5433 if (Instruction *NewXor = foldNotXor(I, Builder))
5434 return NewXor;
5435
5436 // TODO: Could handle multi-use better by checking if all uses of NotOp (other
5437 // than I) can be inverted.
5438 if (Value *R = getFreelyInverted(NotOp, NotOp->hasOneUse(), &Builder))
5439 return replaceInstUsesWith(I, R);
5440
5441 return nullptr;
5442}
5443
5444// ((X + C) & M) ^ M --> (~C − X) & M
5446 InstCombiner::BuilderTy &Builder) {
5447 Value *X, *Mask;
5448 Constant *AddC;
5449 BinaryOperator *AddInst;
5450 if (match(&I,
5452 m_BinOp(AddInst),
5453 m_Add(m_Value(X), m_ImmConstant(AddC)))),
5454 m_Value(Mask))),
5455 m_Deferred(Mask)))) {
5456 Value *NotC = Builder.CreateNot(AddC);
5457 Value *NewSub = Builder.CreateSub(NotC, X, "", AddInst->hasNoUnsignedWrap(),
5458 AddInst->hasNoSignedWrap());
5459 return BinaryOperator::CreateAnd(NewSub, Mask);
5460 }
5461
5462 return nullptr;
5463}
5464
5465// FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
5466// here. We should standardize that construct where it is needed or choose some
5467// other way to ensure that commutated variants of patterns are not missed.
5469 if (Value *V = simplifyXorInst(I.getOperand(0), I.getOperand(1),
5470 SQ.getWithInstruction(&I)))
5471 return replaceInstUsesWith(I, V);
5472
5474 return &I;
5475
5477 return X;
5478
5480 return Phi;
5481
5482 if (Instruction *NewXor = foldXorToXor(I, Builder))
5483 return NewXor;
5484
5485 // (A&B)^(A&C) -> A&(B^C) etc
5487 return replaceInstUsesWith(I, V);
5488
5489 // See if we can simplify any instructions used by the instruction whose sole
5490 // purpose is to compute bits we don't care about.
5492 return &I;
5493
5494 if (Instruction *R = foldNot(I))
5495 return R;
5496
5498 return R;
5499
5500 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
5501 Value *X, *Y, *M;
5502
5503 // (X | Y) ^ M -> (X ^ M) ^ Y
5504 // (X | Y) ^ M -> (Y ^ M) ^ X
5506 m_Value(M)))) {
5507 if (Value *XorAC = simplifyXorInst(X, M, SQ.getWithInstruction(&I)))
5508 return BinaryOperator::CreateXor(XorAC, Y);
5509
5510 if (Value *XorBC = simplifyXorInst(Y, M, SQ.getWithInstruction(&I)))
5511 return BinaryOperator::CreateXor(XorBC, X);
5512 }
5513
5514 // Fold (X & M) ^ (Y & ~M) -> (X & M) | (Y & ~M)
5515 // This it a special case in haveNoCommonBitsSet, but the computeKnownBits
5516 // calls in there are unnecessary as SimplifyDemandedInstructionBits should
5517 // have already taken care of those cases.
5518 if (match(&I, m_c_Xor(m_c_And(m_Not(m_Value(M)), m_Value()),
5519 m_c_And(m_Deferred(M), m_Value())))) {
5521 return BinaryOperator::CreateDisjointOr(Op0, Op1);
5522 else
5523 return BinaryOperator::CreateOr(Op0, Op1);
5524 }
5525
5527 return Xor;
5528
5529 Constant *C1;
5530 if (match(Op1, m_Constant(C1))) {
5531 Constant *C2;
5532
5533 if (match(Op0, m_OneUse(m_Or(m_Value(X), m_ImmConstant(C2)))) &&
5534 match(C1, m_ImmConstant())) {
5535 // (X | C2) ^ C1 --> (X & ~C2) ^ (C1^C2)
5538 Value *And = Builder.CreateAnd(
5540 return BinaryOperator::CreateXor(
5542 }
5543
5544 // Use DeMorgan and reassociation to eliminate a 'not' op.
5545 if (match(Op0, m_OneUse(m_Or(m_Not(m_Value(X)), m_Constant(C2))))) {
5546 // (~X | C2) ^ C1 --> ((X & ~C2) ^ -1) ^ C1 --> (X & ~C2) ^ ~C1
5547 Value *And = Builder.CreateAnd(X, ConstantExpr::getNot(C2));
5548 return BinaryOperator::CreateXor(And, ConstantExpr::getNot(C1));
5549 }
5550 if (match(Op0, m_OneUse(m_And(m_Not(m_Value(X)), m_Constant(C2))))) {
5551 // (~X & C2) ^ C1 --> ((X | ~C2) ^ -1) ^ C1 --> (X | ~C2) ^ ~C1
5552 Value *Or = Builder.CreateOr(X, ConstantExpr::getNot(C2));
5553 return BinaryOperator::CreateXor(Or, ConstantExpr::getNot(C1));
5554 }
5555
5556 // Convert xor ([trunc] (ashr X, BW-1)), C =>
5557 // select(X >s -1, C, ~C)
5558 // The ashr creates "AllZeroOrAllOne's", which then optionally inverses the
5559 // constant depending on whether this input is less than 0.
5560 const APInt *CA;
5561 if (match(Op0, m_OneUse(m_TruncOrSelf(
5562 m_AShr(m_Value(X), m_APIntAllowPoison(CA))))) &&
5563 *CA == X->getType()->getScalarSizeInBits() - 1 &&
5564 !match(C1, m_AllOnes())) {
5565 assert(!C1->isNullValue() && "Unexpected xor with 0");
5566 Value *IsNotNeg = Builder.CreateIsNotNeg(X);
5567 return createSelectInstWithUnknownProfile(IsNotNeg, Op1,
5568 Builder.CreateNot(Op1));
5569 }
5570 }
5571
5572 Type *Ty = I.getType();
5573 {
5574 const APInt *RHSC;
5575 if (match(Op1, m_APInt(RHSC))) {
5576 Value *X;
5577 const APInt *C;
5578 // (C - X) ^ signmaskC --> (C + signmaskC) - X
5579 if (RHSC->isSignMask() && match(Op0, m_Sub(m_APInt(C), m_Value(X))))
5580 return BinaryOperator::CreateSub(ConstantInt::get(Ty, *C + *RHSC), X);
5581
5582 // (X + C) ^ signmaskC --> X + (C + signmaskC)
5583 if (RHSC->isSignMask() && match(Op0, m_Add(m_Value(X), m_APInt(C))))
5584 return BinaryOperator::CreateAdd(X, ConstantInt::get(Ty, *C + *RHSC));
5585
5586 // (X | C) ^ RHSC --> X ^ (C ^ RHSC) iff X & C == 0
5587 if (match(Op0, m_Or(m_Value(X), m_APInt(C))) &&
5588 MaskedValueIsZero(X, *C, &I))
5589 return BinaryOperator::CreateXor(X, ConstantInt::get(Ty, *C ^ *RHSC));
5590
5591 // When X is a power-of-two or zero and zero input is poison:
5592 // ctlz(i32 X) ^ 31 --> cttz(X)
5593 // cttz(i32 X) ^ 31 --> ctlz(X)
5594 // The bit width must be a power of two so BW - 1 is an all-ones mask.
5595 auto *II = dyn_cast<IntrinsicInst>(Op0);
5596 if (II && II->hasOneUse() && isPowerOf2_32(Ty->getScalarSizeInBits()) &&
5597 *RHSC == Ty->getScalarSizeInBits() - 1) {
5598 Intrinsic::ID IID = II->getIntrinsicID();
5599 if ((IID == Intrinsic::ctlz || IID == Intrinsic::cttz) &&
5600 match(II->getArgOperand(1), m_One()) &&
5601 isKnownToBeAPowerOfTwo(II->getArgOperand(0), /*OrZero */ true)) {
5602 IID = (IID == Intrinsic::ctlz) ? Intrinsic::cttz : Intrinsic::ctlz;
5603 Function *F =
5604 Intrinsic::getOrInsertDeclaration(II->getModule(), IID, Ty);
5605 return CallInst::Create(F, {II->getArgOperand(0), Builder.getTrue()});
5606 }
5607 }
5608
5609 // If RHSC is inverting the remaining bits of shifted X,
5610 // canonicalize to a 'not' before the shift to help SCEV and codegen:
5611 // (X << C) ^ RHSC --> ~X << C
5612 if (match(Op0, m_OneUse(m_Shl(m_Value(X), m_APInt(C)))) &&
5613 *RHSC == APInt::getAllOnes(Ty->getScalarSizeInBits()).shl(*C)) {
5614 Value *NotX = Builder.CreateNot(X);
5615 return BinaryOperator::CreateShl(NotX, ConstantInt::get(Ty, *C));
5616 }
5617 // (X >>u C) ^ RHSC --> ~X >>u C
5618 if (match(Op0, m_OneUse(m_LShr(m_Value(X), m_APInt(C)))) &&
5619 *RHSC == APInt::getAllOnes(Ty->getScalarSizeInBits()).lshr(*C)) {
5620 Value *NotX = Builder.CreateNot(X);
5621 return BinaryOperator::CreateLShr(NotX, ConstantInt::get(Ty, *C));
5622 }
5623 // TODO: We could handle 'ashr' here as well. That would be matching
5624 // a 'not' op and moving it before the shift. Doing that requires
5625 // preventing the inverse fold in canShiftBinOpWithConstantRHS().
5626 }
5627
5628 // If we are XORing the sign bit of a floating-point value, convert
5629 // this to fneg, then cast back to integer.
5630 //
5631 // This is generous interpretation of noimplicitfloat, this is not a true
5632 // floating-point operation.
5633 //
5634 // Assumes any IEEE-represented type has the sign bit in the high bit.
5635 // TODO: Unify with APInt matcher. This version allows undef unlike m_APInt
5636 Value *CastOp;
5637 if (match(Op0, m_ElementWiseBitCast(m_Value(CastOp))) &&
5638 match(Op1, m_SignMask()) &&
5639 !Builder.GetInsertBlock()->getParent()->hasFnAttribute(
5640 Attribute::NoImplicitFloat)) {
5641 Type *EltTy = CastOp->getType()->getScalarType();
5642 if (EltTy->isFloatingPointTy() &&
5644 Value *FNeg = Builder.CreateFNeg(CastOp);
5645 return new BitCastInst(FNeg, I.getType());
5646 }
5647 }
5648 }
5649
5650 // FIXME: This should not be limited to scalar (pull into APInt match above).
5651 {
5652 Value *X;
5653 ConstantInt *C1, *C2, *C3;
5654 // ((X^C1) >> C2) ^ C3 -> (X>>C2) ^ ((C1>>C2)^C3)
5655 if (match(Op1, m_ConstantInt(C3)) &&
5657 m_ConstantInt(C2))) &&
5658 Op0->hasOneUse()) {
5659 // fold (C1 >> C2) ^ C3
5660 APInt FoldConst = C1->getValue().lshr(C2->getValue());
5661 FoldConst ^= C3->getValue();
5662 // Prepare the two operands.
5663 auto *Opnd0 = Builder.CreateLShr(X, C2);
5664 Opnd0->takeName(Op0);
5665 return BinaryOperator::CreateXor(Opnd0, ConstantInt::get(Ty, FoldConst));
5666 }
5667 }
5668
5669 if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I))
5670 return FoldedLogic;
5671
5672 if (Instruction *FoldedLogic = foldBinOpSelectBinOp(I))
5673 return FoldedLogic;
5674
5675 // Y ^ (X | Y) --> X & ~Y
5676 // Y ^ (Y | X) --> X & ~Y
5677 if (match(Op1, m_OneUse(m_c_Or(m_Value(X), m_Specific(Op0)))))
5678 return BinaryOperator::CreateAnd(X, Builder.CreateNot(Op0));
5679 // (X | Y) ^ Y --> X & ~Y
5680 // (Y | X) ^ Y --> X & ~Y
5681 if (match(Op0, m_OneUse(m_c_Or(m_Value(X), m_Specific(Op1)))))
5682 return BinaryOperator::CreateAnd(X, Builder.CreateNot(Op1));
5683
5684 // Y ^ (X & Y) --> ~X & Y
5685 // Y ^ (Y & X) --> ~X & Y
5686 if (match(Op1, m_OneUse(m_c_And(m_Value(X), m_Specific(Op0)))))
5687 return BinaryOperator::CreateAnd(Op0, Builder.CreateNot(X));
5688 // (X & Y) ^ Y --> ~X & Y
5689 // (Y & X) ^ Y --> ~X & Y
5690 // Canonical form is (X & C) ^ C; don't touch that.
5691 // TODO: A 'not' op is better for analysis and codegen, but demanded bits must
5692 // be fixed to prefer that (otherwise we get infinite looping).
5693 if (!match(Op1, m_Constant()) &&
5694 match(Op0, m_OneUse(m_c_And(m_Value(X), m_Specific(Op1)))))
5695 return BinaryOperator::CreateAnd(Op1, Builder.CreateNot(X));
5696
5697 Value *A, *B, *C;
5698 // (A ^ B) ^ (A | C) --> (~A & C) ^ B -- There are 4 commuted variants.
5701 return BinaryOperator::CreateXor(
5702 Builder.CreateAnd(Builder.CreateNot(A), C), B);
5703
5704 // (A ^ B) ^ (B | C) --> (~B & C) ^ A -- There are 4 commuted variants.
5707 return BinaryOperator::CreateXor(
5708 Builder.CreateAnd(Builder.CreateNot(B), C), A);
5709
5710 // (A & B) ^ (A ^ B) -> (A | B)
5711 if (match(Op0, m_And(m_Value(A), m_Value(B))) &&
5713 return BinaryOperator::CreateOr(A, B);
5714 // (A ^ B) ^ (A & B) -> (A | B)
5715 if (match(Op0, m_Xor(m_Value(A), m_Value(B))) &&
5717 return BinaryOperator::CreateOr(A, B);
5718
5719 // (A & ~B) ^ ~A -> ~(A & B)
5720 // (~B & A) ^ ~A -> ~(A & B)
5721 if (match(Op0, m_c_And(m_Value(A), m_Not(m_Value(B)))) &&
5722 match(Op1, m_Not(m_Specific(A))))
5723 return BinaryOperator::CreateNot(Builder.CreateAnd(A, B));
5724
5725 // (~A & B) ^ A --> A | B -- There are 4 commuted variants.
5727 return BinaryOperator::CreateOr(A, B);
5728
5729 // (~A | B) ^ A --> ~(A & B)
5730 if (match(Op0, m_OneUse(m_c_Or(m_Not(m_Specific(Op1)), m_Value(B)))))
5731 return BinaryOperator::CreateNot(Builder.CreateAnd(Op1, B));
5732
5733 // A ^ (~A | B) --> ~(A & B)
5734 if (match(Op1, m_OneUse(m_c_Or(m_Not(m_Specific(Op0)), m_Value(B)))))
5735 return BinaryOperator::CreateNot(Builder.CreateAnd(Op0, B));
5736
5737 // (A | B) ^ (A | C) --> (B ^ C) & ~A -- There are 4 commuted variants.
5738 // TODO: Loosen one-use restriction if common operand is a constant.
5739 Value *D;
5740 if (match(Op0, m_OneUse(m_Or(m_Value(A), m_Value(B)))) &&
5741 match(Op1, m_OneUse(m_Or(m_Value(C), m_Value(D))))) {
5742 if (B == C || B == D)
5743 std::swap(A, B);
5744 if (A == C)
5745 std::swap(C, D);
5746 if (A == D) {
5747 Value *NotA = Builder.CreateNot(A);
5748 return BinaryOperator::CreateAnd(Builder.CreateXor(B, C), NotA);
5749 }
5750 }
5751
5752 // (A & B) ^ (A | C) --> A ? ~B : C -- There are 4 commuted variants.
5753 if (I.getType()->isIntOrIntVectorTy(1) &&
5756 bool NeedFreeze = isa<SelectInst>(Op0) && isa<SelectInst>(Op1) && B == D;
5757 if (B == C || B == D) {
5758 std::swap(A, B);
5759 }
5760 if (A == C)
5761 std::swap(C, D);
5762 if (A == D) {
5763 if (NeedFreeze)
5764 A = Builder.CreateFreeze(A);
5765 Value *NotB = Builder.CreateNot(B);
5766 Instruction *MDFrom = nullptr;
5767 // If one of the operands has the same condition as we will use for the
5768 // select we are going to create, pull the metadata from it (primarily the
5769 // profile info).
5770 if (auto *Op0SI = dyn_cast<SelectInst>(Op0)) {
5771 if (Op0SI->getCondition() == A)
5772 MDFrom = Op0SI;
5773 }
5774 if (auto *Op1SI = dyn_cast<SelectInst>(Op1)) {
5775 if (Op1SI->getCondition() == A)
5776 MDFrom = Op1SI;
5777 }
5778 return (MDFrom == nullptr || ProfcheckDisableMetadataFixes)
5779 ? createSelectInstWithUnknownProfile(A, NotB, C)
5780 : SelectInst::Create(A, NotB, C, "", nullptr, MDFrom);
5781 }
5782 }
5783
5784 if (auto *LHS = dyn_cast<ICmpInst>(I.getOperand(0)))
5785 if (auto *RHS = dyn_cast<ICmpInst>(I.getOperand(1)))
5786 if (Value *V = foldXorOfICmps(LHS, RHS, I))
5787 return replaceInstUsesWith(I, V);
5788
5789 if (Instruction *CastedXor = foldCastedBitwiseLogic(I))
5790 return CastedXor;
5791
5792 if (Instruction *Abs = canonicalizeAbs(I, Builder))
5793 return Abs;
5794
5795 // Otherwise, if all else failed, try to hoist the xor-by-constant:
5796 // (X ^ C) ^ Y --> (X ^ Y) ^ C
5797 // Just like we do in other places, we completely avoid the fold
5798 // for constantexprs, at least to avoid endless combine loop.
5800 m_ImmConstant(C1))),
5801 m_Value(Y))))
5802 return BinaryOperator::CreateXor(Builder.CreateXor(X, Y), C1);
5803
5805 return R;
5806
5807 if (Instruction *Canonicalized = canonicalizeLogicFirst(I, Builder))
5808 return Canonicalized;
5809
5810 if (Instruction *Folded = foldLogicOfIsFPClass(I, Op0, Op1))
5811 return Folded;
5812
5813 if (Instruction *Folded = canonicalizeConditionalNegationViaMathToSelect(I))
5814 return Folded;
5815
5816 if (Instruction *Res = foldBinOpOfDisplacedShifts(I))
5817 return Res;
5818
5820 return Res;
5821
5823 return Res;
5824
5825 return nullptr;
5826}
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool isSigned(unsigned Opcode)
#define DEBUG_TYPE
static Value * foldBitmaskMul(Value *Op0, Value *Op1, InstCombiner::BuilderTy &Builder)
(A & N) * C + (A & M) * C -> (A & (N + M)) & C This also accepts the equivalent select form of (A & N...
static unsigned conjugateICmpMask(unsigned Mask)
Convert an analysis of a masked ICmp into its equivalent if all boolean operations had the opposite s...
static Instruction * foldNotXor(BinaryOperator &I, InstCombiner::BuilderTy &Builder)
static Value * foldLogOpOfMaskedICmps(Value *LHS, Value *RHS, bool IsAnd, bool IsLogical, InstCombiner::BuilderTy &Builder, const SimplifyQuery &Q)
Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E) into a single (icmp(A & X) ==/!...
static Value * getFCmpValue(unsigned Code, Value *LHS, Value *RHS, InstCombiner::BuilderTy &Builder, FMFSource FMF)
This is the complement of getFCmpCode, which turns an opcode and two operands into either a FCmp inst...
static bool matchIsFPClassLikeFCmp(Value *Op, Value *&ClassVal, uint64_t &ClassMask)
Match an fcmp against a special value that performs a test possible by llvm.is.fpclass.
static Instruction * visitMaskedMerge(BinaryOperator &I, InstCombiner::BuilderTy &Builder)
If we have a masked merge, in the canonical form of: (assuming that A only has one use....
static Instruction * canonicalizeAbs(BinaryOperator &Xor, InstCombiner::BuilderTy &Builder)
Canonicalize a shifty way to code absolute value to the more common pattern that uses negation and se...
static Value * foldAndOrOfICmpEqConstantAndICmp(CmpPredicate PredL, Value *LHS0, Value *LHS1, bool LHSOneUse, CmpPredicate PredR, Value *RHS0, Value *RHS1, bool RHSOneUse, bool IsAnd, bool IsLogical, IRBuilderBase &Builder)
static Instruction * foldOrToXor(BinaryOperator &I, InstCombiner::BuilderTy &Builder)
static Value * simplifyAndOrWithOpReplaced(Value *V, Value *Op, Value *RepOp, bool SimplifyOnly, InstCombinerImpl &IC, unsigned Depth=0)
static Instruction * matchDeMorgansLaws(BinaryOperator &I, InstCombiner &IC)
Match variations of De Morgan's Laws: (~A & ~B) == (~(A | B)) (~A | ~B) == (~(A & B))
static Value * FoldOrOfSelectSmaxToAbs(BinaryOperator &I, InstCombiner::BuilderTy &Builder)
Fold select(X >s 0, 0, -X) | smax(X, 0) --> abs(X) select(X <s 0, -X, 0) | smax(X,...
static Value * foldLogOpOfMaskedICmpsAsymmetric(Value *LHS, Value *RHS, bool IsAnd, bool IsLogical, Value *A, Value *B, Value *C, Value *D, Value *E, ICmpInst::Predicate PredL, ICmpInst::Predicate PredR, unsigned LHSMask, unsigned RHSMask, InstCombiner::BuilderTy &Builder)
Try to fold (icmp(A & B) ==/!= 0) &/| (icmp(A & D) ==/!= E) into a single (icmp(A & X) ==/!...
static Instruction * foldAndToXor(BinaryOperator &I, InstCombiner::BuilderTy &Builder)
static unsigned getMaskedICmpType(Value *A, Value *B, Value *C, ICmpInst::Predicate Pred)
Return the set of patterns (from MaskedICmpType) that (icmp SCC (A & B), C) satisfies.
static Instruction * foldXorToXor(BinaryOperator &I, InstCombiner::BuilderTy &Builder)
A ^ B can be specified using other logic ops in a variety of patterns.
static bool canNarrowShiftAmt(Constant *C, unsigned BitWidth)
Return true if a constant shift amount is always less than the specified bit-width.
static Value * foldIsPowerOf2(CmpPredicate PredL, Value *LHS0, Value *LHS1, CmpPredicate PredR, Value *RHS0, Value *RHS1, bool JoinedByAnd, InstCombiner::BuilderTy &Builder, InstCombinerImpl &IC)
Reduce a pair of compares that check if a value has exactly 1 bit set.
static Value * foldIsPowerOf2OrZero(CmpPredicate PredL, Value *LHS0, Value *LHS1, CmpPredicate PredR, Value *RHS0, Value *RHS1, bool IsAnd, InstCombiner::BuilderTy &Builder, InstCombinerImpl &IC)
Fold (icmp eq ctpop(X) 1) | (icmp eq X 0) into (icmp ult ctpop(X) 2) and fold (icmp ne ctpop(X) 1) & ...
static Instruction * foldLogicCastConstant(BinaryOperator &Logic, CastInst *Cast, InstCombinerImpl &IC)
Fold {and,or,xor} (cast X), C.
static Value * foldPowerOf2AndShiftedMask(Value *Cmp0, Value *Cmp1, bool JoinedByAnd, InstCombiner::BuilderTy &Builder)
Try to fold ((icmp X u< P) & (icmp(X & M) != M)) or ((icmp X s> -1) & (icmp(X & M) !...
static bool canFreelyInvert(InstCombiner &IC, Value *Op, Instruction *IgnoredUser)
static Value * foldNegativePower2AndShiftedMask(Value *A, Value *B, Value *D, Value *E, ICmpInst::Predicate PredL, ICmpInst::Predicate PredR, InstCombiner::BuilderTy &Builder)
Try to fold (icmp(A & B) == 0) & (icmp(A & D) != E) into (icmp A u< D) iff B is a contiguous set of o...
static Value * matchIsFiniteTest(InstCombiner::BuilderTy &Builder, FCmpInst *LHS, FCmpInst *RHS)
and (fcmp ord x, 0), (fcmp u* x, inf) -> fcmp o* x, inf
static Value * foldSignedTruncationCheck(CmpPredicate PredL, Value *LHS0, Value *LHS1, CmpPredicate PredR, Value *RHS0, Value *RHS1, Instruction &CtxI, InstCombiner::BuilderTy &Builder)
General pattern: X & Y.
static Value * foldOrUnsignedUMulOverflowICmp(BinaryOperator &I, InstCombiner::BuilderTy &Builder, const DataLayout &DL)
Fold Res, Overflow = (umul.with.overflow x c1); (or Overflow (ugt Res c2)) --> (ugt x (c2/c1)).
static Value * freelyInvert(InstCombinerImpl &IC, Value *Op, Instruction *IgnoredUser)
static std::optional< IntPart > matchIntPart(Value *V)
Match an extraction of bits from an integer.
static Instruction * canonicalizeLogicFirst(BinaryOperator &I, InstCombiner::BuilderTy &Builder)
static Instruction * reassociateFCmps(BinaryOperator &BO, InstCombiner::BuilderTy &Builder)
This a limited reassociation for a special case (see above) where we are checking if two values are e...
static Value * getNewICmpValue(unsigned Code, bool Sign, Value *LHS, Value *RHS, InstCombiner::BuilderTy &Builder)
This is the complement of getICmpCode, which turns an opcode and two operands into either a constant ...
static Value * extractIntPart(const IntPart &P, IRBuilderBase &Builder)
Materialize an extraction of bits from an integer in IR.
static bool matchUnorderedInfCompare(FCmpInst::Predicate P, Value *LHS, Value *RHS)
Matches fcmp u__ x, +/-inf.
static bool matchIsNotNaN(FCmpInst::Predicate P, Value *LHS, Value *RHS)
Matches canonical form of isnan, fcmp ord x, 0.
static Value * getReconstructedMaskedICmp(IRBuilderBase &Builder, bool IsLogical, Value *RHS, Value *A, Value *D, Value *E, ICmpInst::Predicate PredR)
When foldLogOpOfMaskedICmps returns RHS for a logical and/or, we need to make sure that RHS is not mo...
static bool areInverseVectorBitmasks(Constant *C1, Constant *C2)
If all elements of two constant vectors are 0/-1 and inverses, return true.
MaskedICmpType
Classify (icmp eq (A & B), C) and (icmp ne (A & B), C) as matching patterns that can be simplified.
@ BMask_NotAllOnes
@ AMask_NotAllOnes
@ Mask_NotAllZeros
static Instruction * foldComplexAndOrPatterns(BinaryOperator &I, InstCombiner::BuilderTy &Builder)
Try folding relatively complex patterns for both And and Or operations with all And and Or swapped.
static bool matchZExtedSubInteger(Value *V, Value *&Int, APInt &Mask, uint64_t &Offset, bool &IsShlNUW, bool &IsShlNSW)
Match V as "lshr -> mask -> zext -> shl".
static Value * foldAndOrOfICmpsWithPow2AndWithZero(InstCombiner::BuilderTy &Builder, CmpPredicate PredL, Value *LHS0, Value *LHS1, bool LHSOneUse, CmpPredicate PredR, Value *RHS0, Value *RHS1, bool RHSOneUse, bool IsAnd, const SimplifyQuery &Q)
static Value * foldUnsignedUnderflowCheck(CmpPredicate PredL, Value *LHS0, Value *LHS1, bool LHSOneUse, CmpPredicate PredR, Value *RHS0, Value *RHS1, bool RHSOneUse, bool IsAnd, const SimplifyQuery &Q, InstCombiner::BuilderTy &Builder)
Commuted variants are assumed to be handled by calling this function again with the parameters swappe...
static Value * foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed(Value *LHS, Value *RHS, bool IsAnd, bool IsLogical, Value *A, Value *B, Value *D, Value *E, ICmpInst::Predicate PredL, ICmpInst::Predicate PredR, InstCombiner::BuilderTy &Builder)
Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E) into a single (icmp(A & X) ==/!...
static Instruction * foldRoundUpToPow2Alignment(BinaryOperator &I, InstCombiner::BuilderTy &Builder)
The pattern div_ceil(X, P) * P, where P is a power of 2, lowers to the following conditional round-up...
static std::optional< DecomposedBitMaskMul > matchBitmaskMul(Value *V)
static Value * foldOrOfInversions(BinaryOperator &I, InstCombiner::BuilderTy &Builder)
static bool matchSubIntegerPackFromVector(Value *V, Value *&Vec, int64_t &VecOffset, SmallBitVector &Mask, const DataLayout &DL)
Match V as "shufflevector -> bitcast" or "extractelement -> zext -> shl" patterns,...
static Instruction * matchFunnelShift(Instruction &Or, InstCombinerImpl &IC)
Match UB-safe variants of the funnel shift intrinsic.
static Instruction * reassociateForUses(BinaryOperator &BO, InstCombinerImpl::BuilderTy &Builder)
Try to reassociate a pair of binops so that values with one use only are part of the same instruction...
static Value * matchOrConcat(Instruction &Or, InstCombiner::BuilderTy &Builder)
Attempt to combine or(zext(x),shl(zext(y),bw/2) concat packing patterns.
static Instruction * foldMaskedAddXorPattern(BinaryOperator &I, InstCombiner::BuilderTy &Builder)
static Instruction * foldBitwiseLogicWithIntrinsics(BinaryOperator &I, InstCombiner::BuilderTy &Builder)
static std::optional< std::pair< unsigned, unsigned > > getMaskedTypeForICmpPair(Value *&A, Value *&B, Value *&C, Value *&D, Value *&E, Value *LHS, Value *RHS, ICmpInst::Predicate &PredL, ICmpInst::Predicate &PredR)
Handle (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E).
static Value * foldAndOrOfICmpsWithConstEq(CmpPredicate PredL, Value *LHS0, Value *LHS1, Value *LHS, CmpPredicate PredR, Value *RHS0, Value *RHS1, bool RHSOneUse, bool IsAnd, bool IsLogical, InstCombiner::BuilderTy &Builder, const SimplifyQuery &Q, Instruction &I)
Reduce logic-of-compares with equality to a constant by substituting a common operand with the consta...
static Instruction * foldIntegerPackFromVector(Instruction &I, InstCombiner::BuilderTy &Builder, const DataLayout &DL)
Try to fold the join of two scalar integers whose contents are packed elements of the same vector.
static Value * foldIntegerRepackThroughZExt(Value *Lhs, Value *Rhs, InstCombiner::BuilderTy &Builder)
Try to fold the join of two scalar integers whose bits are unpacked and zexted from the same source i...
This file provides internal interfaces used to implement the InstCombine.
This file provides the interface for the instcombine pass implementation.
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:550
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define R2(n)
uint64_t High
uint64_t IntrinsicInst * II
#define P(N)
This file contains the declarations for profiling metadata utility functions.
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
int64_t C1
RISCVCC::CondCode CC
int64_t C0
MachineInstr *& Lo
This file implements the SmallBitVector class.
static unsigned getScalarSizeInBits(Type *Ty)
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static constexpr int Concat[]
Value * RHS
Value * LHS
The Input class is used to parse a yaml document into in-memory structs and vectors.
static LLVM_ABI bool hasSignBitInMSB(const fltSemantics &)
Definition APFloat.cpp:381
bool bitwiseIsEqual(const APFloat &RHS) const
Definition APFloat.h:1548
bool isZero() const
Definition APFloat.h:1579
APInt bitcastToAPInt() const
Definition APFloat.h:1475
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition APFloat.h:1202
Class for arbitrary precision integers.
Definition APInt.h:78
LLVM_ABI APInt udiv(const APInt &RHS) const
Unsigned division operation.
Definition APInt.cpp:1602
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1057
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
unsigned countLeadingOnes() const
Definition APInt.h:1644
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition APInt.h:367
LLVM_ABI APInt usub_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1986
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition APInt.h:1186
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
Definition APInt.h:462
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition APInt.h:1115
LLVM_ABI APInt sadd_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1966
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition APInt.h:1253
int32_t exactLogBase2() const
Definition APInt.h:1803
LLVM_ABI APInt reverseBits() const
Definition APInt.cpp:786
LLVM_ABI APInt uadd_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1973
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition APInt.h:1659
unsigned countLeadingZeros() const
Definition APInt.h:1626
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition APInt.h:1154
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition APInt.h:875
LLVM_ABI APInt byteSwap() const
Definition APInt.cpp:764
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition APInt.h:1261
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:436
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
LLVM_ABI APInt ssub_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1979
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition APInt.h:282
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition APInt.h:1225
void clearSignBit()
Set the sign bit to 0.
Definition APInt.h:1469
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
LLVM_ABI bool isSigned() const
Whether the intrinsic is signed or unsigned.
LLVM_ABI Instruction::BinaryOps getBinaryOp() const
Returns the binary operation underlying the intrinsic.
BinaryOps getOpcode() const
Definition InstrTypes.h:409
static LLVM_ABI BinaryOperator * CreateNot(Value *Op, const Twine &Name="", InsertPosition InsertBefore=nullptr)
static LLVM_ABI BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
static BinaryOperator * CreateWithCopiedFlags(BinaryOps Opc, Value *V1, Value *V2, Value *CopyO, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Definition InstrTypes.h:254
This class represents a no-op cast from one type to another.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
This is the base class for all instructions that perform data casts.
Definition InstrTypes.h:512
Type * getSrcTy() const
Return the source type, as a convenience.
Definition InstrTypes.h:679
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Definition InstrTypes.h:674
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Type * getDestTy() const
Return the destination type, as a convenience.
Definition InstrTypes.h:681
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
bool isSigned() const
Definition InstrTypes.h:993
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition InstrTypes.h:890
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition InstrTypes.h:852
static LLVM_ABI bool isUnordered(Predicate predicate)
Determine if the predicate is an unordered operation.
static Predicate getOrderedPredicate(Predicate Pred)
Returns the ordered variant of a floating point compare.
Definition InstrTypes.h:859
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
static LLVM_ABI CmpPredicate getInverse(CmpPredicate P)
Get the inverse predicate of a CmpPredicate.
CmpInst::Predicate dropSameSign() const
Drops samesign information.
bool hasSameSign() const
Query samesign information, for optimizations.
static LLVM_ABI CmpPredicate getSwapped(CmpPredicate P)
Get the swapped predicate of a CmpPredicate.
static LLVM_ABI Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static LLVM_ABI Constant * getNot(Constant *C)
static LLVM_ABI Constant * getXor(Constant *C1, Constant *C2)
static LLVM_ABI Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static LLVM_ABI Constant * getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static LLVM_ABI Constant * getExactLogBase2(Constant *C)
If C is a scalar/fixed width vector of known powers of 2, then this function returns a new scalar/fix...
static LLVM_ABI ConstantFP * getZero(Type *Ty, bool Negative=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
Definition Constants.h:231
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
LLVM_ABI std::optional< ConstantRange > exactUnionWith(const ConstantRange &CR) const
Union the two ranges and return the result if it can be represented exactly, otherwise return std::nu...
LLVM_ABI ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
static LLVM_ABI ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
LLVM_ABI std::optional< ConstantRange > exactIntersectWith(const ConstantRange &CR) const
Intersect the two ranges and return the result if it can be represented exactly, otherwise return std...
This is an important base class in LLVM.
Definition Constant.h:44
static LLVM_ABI Constant * replaceUndefsWith(Constant *C, Constant *Replacement)
Try to replace undefined constant C or undefined elements in C with Replacement.
static LLVM_ABI Constant * mergeUndefsWith(Constant *C, Constant *Other)
Merges undefs of a Constant with another Constant, along with the undefs already present.
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
LLVM_ABI Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
This instruction compares its operands according to the predicate given to the constructor.
This provides a helper for copying FMF from an instruction or setting specified flags.
Definition IRBuilder.h:90
static FMFSource intersect(Value *A, Value *B)
Intersect the FMF from two instructions.
Definition IRBuilder.h:104
void setNoNaNs(bool B=true)
Definition FMF.h:78
void setNoInfs(bool B=true)
Definition FMF.h:81
This instruction compares its operands according to the predicate given to the constructor.
Predicate getSignedPredicate() const
For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
bool isEquality() const
Return true if this predicate is either EQ or NE.
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
Common base class shared among various IRBuilders.
Definition IRBuilder.h:111
Value * CreateNot(Value *V, const Twine &Name="")
Definition IRBuilder.h:1856
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition IRBuilder.h:1733
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition IRBuilder.h:197
Instruction * canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract(BinaryOperator &I)
Instruction * foldBinOpIntoSelectOrPhi(BinaryOperator &I)
This is a convenience wrapper function for the above two functions.
Instruction * visitOr(BinaryOperator &I)
bool SimplifyAssociativeOrCommutative(BinaryOperator &I)
Performs a few simplifications for operators which are associative or commutative.
Value * foldUsingDistributiveLaws(BinaryOperator &I)
Tries to simplify binary operations which some other binary operation distributes over.
Instruction * foldBinOpShiftWithShift(BinaryOperator &I)
Value * insertRangeTest(Value *V, const APInt &Lo, const APInt &Hi, bool isSigned, bool Inside)
Emit a computation of: (V >= Lo && V < Hi) if Inside is true, otherwise (V < Lo || V >= Hi).
Instruction * foldBinOpSelectBinOp(BinaryOperator &Op)
In some cases it is beneficial to fold a select into a binary operator.
bool sinkNotIntoLogicalOp(Instruction &I)
std::optional< std::pair< Intrinsic::ID, SmallVector< Value *, 3 > > > convertOrOfShiftsToFunnelShift(Instruction &Or)
Instruction * visitAnd(BinaryOperator &I)
bool sinkNotIntoOtherHandOfLogicalOp(Instruction &I)
Instruction * foldBinopWithPhiOperands(BinaryOperator &BO)
For a binary operator with 2 phi operands, try to hoist the binary operation before the phi.
Instruction * foldAddLikeCommutative(Value *LHS, Value *RHS, bool NSW, bool NUW)
Common transforms for add / disjoint or.
Instruction * tryFoldInstWithCtpopWithNot(Instruction *I)
Instruction * FoldOrOfLogicalAnds(Value *Op0, Value *Op1)
Value * SimplifyAddWithRemainder(BinaryOperator &I)
Tries to simplify add operations using the definition of remainder.
Instruction * visitXor(BinaryOperator &I)
bool SimplifyDemandedInstructionBits(Instruction &Inst)
Tries to simplify operands to an integer instruction based on its demanded bits.
Value * simplifyRangeCheck(CmpPredicate PredL, Value *LHS0, Value *LHS1, CmpPredicate PredR, Value *RHS0, Value *RHS1, Instruction *CtxI, bool Inverted)
Try to fold a signed range checked with lower bound 0 to an unsigned icmp.
Instruction * foldVectorBinop(BinaryOperator &Inst)
Canonicalize the position of binops relative to shufflevector.
Instruction * matchBSwapOrBitReverse(Instruction &I, bool MatchBSwaps, bool MatchBitReversals)
Given an initial instruction, check to see if it is the root of a bswap/bitreverse idiom.
void freelyInvertAllUsersOf(Value *V, Value *IgnoredUser=nullptr)
Freely adapt every user of V as-if V was changed to !V.
The core instruction combiner logic.
SimplifyQuery SQ
const DataLayout & getDataLayout() const
bool isFreeToInvert(Value *V, bool WillInvertAllUses, bool &DoesConsume)
Return true if the specified value is free to invert (apply ~ to).
bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero=false, const Instruction *CtxI=nullptr, unsigned Depth=0)
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
InstructionWorklist & Worklist
A worklist of the instructions that need to be simplified.
const DataLayout & DL
unsigned ComputeNumSignBits(const Value *Op, const Instruction *CtxI=nullptr, unsigned Depth=0) const
bool MaskedValueIsZero(const Value *V, const APInt &Mask, const Instruction *CtxI=nullptr, unsigned Depth=0) const
static Value * peekThroughBitcast(Value *V, bool OneUseOnly=false)
Return the source operand of a potentially bitcasted value while optionally checking if it has one us...
IRBuilder< TargetFolder, IRBuilderInstCombineInserter > BuilderTy
An IRBuilder that automatically inserts new instructions into the worklist.
bool canFreelyInvertAllUsersOf(Instruction *V, Value *IgnoredUser)
Given i1 V, can every user of V be freely adapted if V is changed to !V ?
void addToWorklist(Instruction *I)
static Value * stripSignOnlyFPOps(Value *Val)
Ignore all operations which only change the sign of a value, returning the underlying magnitude value...
DominatorTree & DT
void computeKnownBits(const Value *V, KnownBits &Known, const Instruction *CtxI, unsigned Depth=0) const
Value * getFreelyInverted(Value *V, bool WillInvertAllUses, BuilderTy *Builder, bool &DoesConsume)
const SimplifyQuery & getSimplifyQuery() const
LLVM_ABI void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
LLVM_ABI bool hasNoUnsignedWrap() const LLVM_READONLY
Determine whether the no unsigned wrap flag is set.
LLVM_ABI bool hasNoSignedWrap() const LLVM_READONLY
Determine whether the no signed wrap flag is set.
LLVM_ABI void swapProfMetadata()
If the instruction has "branch_weights" MD_prof metadata and the MDNode has three operands (including...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
A wrapper class for inspecting calls to intrinsic functions.
This class represents a sign extension of integer types.
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, const Instruction *MDFrom=nullptr)
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition Type.h:258
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
LLVM_ABI const fltSemantics & getFltSemantics() const
Definition Type.cpp:96
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
Value * getOperand(unsigned i) const
Definition User.h:207
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
iterator_range< user_iterator > users()
Definition Value.h:428
LLVM_ABI bool hasNUsesOrMore(unsigned N) const
Return true if this value has N uses or more.
Definition Value.cpp:150
LLVM_ABI bool hasNUses(unsigned N) const
Return true if this Value has exactly N uses.
Definition Value.cpp:142
bool use_empty() const
Definition Value.h:348
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:395
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Represents an op.with.overflow intrinsic.
This class represents zero extension of integer types.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be unsigned.
Definition APInt.h:2284
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
auto m_PosZeroFP()
Matches a floating-point positive zero.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
AllOnesConstantMatch m_AllOnes()
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
match_unless< Pattern > m_Unless(const Pattern &P)
Match if the inner matcher does NOT match.
match_combine_or< Ty... > m_CombineOr(const Ty &...Ps)
Combine pattern matchers matching any of Ps patterns.
match_combine_and< Ty... > m_CombineAnd(const Ty &...Ps)
Combine pattern matchers matching all of Ps patterns.
cst_pred_ty< is_lowbit_mask > m_LowBitMask()
Match an integer or vector with only the low bit(s) set.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
auto m_BSwap(const Opnd0 &Op0)
cst_pred_ty< is_negative > m_Negative()
Match an integer or vector of negative values.
auto m_Cmp()
Matches any compare instruction and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
auto m_BitReverse(const Opnd0 &Op0)
CmpClass_match< LHS, RHS, FCmpInst > m_FCmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
cst_pred_ty< is_sign_mask > m_SignMask()
Match an integer or vector with only the sign bit(s) set.
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
cstfp_pred_ty< is_inf > m_Inf()
Match a positive or negative infinity FP constant.
cst_pred_ty< is_power2 > m_Power2()
Match an integer or vector power-of-2.
match_combine_or< CastInst_match< OpTy, TruncInst >, OpTy > m_TruncOrSelf(const OpTy &Op)
auto m_LogicalOp()
Matches either L && R or L || R where L and R are arbitrary values.
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)
Matches Trunc.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
ap_match< APInt > m_APIntAllowPoison(const APInt *&Res)
Match APInt while allowing poison in splat vector constants.
auto m_ConstantExpr()
Match a constant expression or a constant that contains a constant expression.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Sub, OverflowingBinaryOperator::NoSignedWrap > m_NSWSub(const LHS &L, const RHS &R)
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
match_combine_or< CastInst_match< OpTy, ZExtInst >, OpTy > m_ZExtOrSelf(const OpTy &Op)
bool match(Val *V, const Pattern &P)
match_bind< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
cst_pred_ty< is_shifted_mask > m_ShiftedMask()
match_deferred< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
cstfp_pred_ty< is_any_zero_fp > m_AnyZeroFP()
Match a floating-point negative zero or positive zero.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
DisjointOr_match< LHS, RHS > m_DisjointOr(const LHS &L, const RHS &R)
BinOpPred_match< LHS, RHS, is_right_shift_op > m_Shr(const LHS &L, const RHS &R)
Matches logical shift operations.
specific_intval< true > m_SpecificIntAllowPoison(const APInt &V)
ap_match< APFloat > m_APFloatAllowPoison(const APFloat *&Res)
Match APFloat while allowing poison in splat vector constants.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
cst_pred_ty< is_nonnegative > m_NonNegative()
Match an integer or vector of non-negative values.
auto m_SMax(const Opnd0 &Op0, const Opnd1 &Op1)
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
auto m_BinOp()
Match an arbitrary binary operation and ignore it.
match_combine_or< CastInst_match< OpTy, SExtInst >, OpTy > m_SExtOrSelf(const OpTy &Op)
ExtractValue_match< Ind, Val_t > m_ExtractValue(const Val_t &V)
Match a single index ExtractValue instruction.
BinOpPred_match< LHS, RHS, is_logical_shift_op > m_LogicalShift(const LHS &L, const RHS &R)
Matches logical shift operations.
ICmpLike_match< LHS, RHS > m_ICmpLike(CmpPredicate &Pred, const LHS &L, const RHS &R)
auto m_Value()
Match an arbitrary value and ignore it.
ShiftLike_match< LHS, Instruction::Shl > m_ShlOrSelf(const LHS &L, uint64_t &R)
Matches shl L, ConstShAmt or L itself (R will be set to zero in this case).
BinaryOp_match< LHS, RHS, Instruction::Xor, true > m_c_Xor(const LHS &L, const RHS &R)
Matches an Xor with LHS and RHS in either order.
auto m_Ctpop(const Opnd0 &Op0)
SpecificCmpClass_match< LHS, RHS, CmpInst > m_SpecificCmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
auto m_Constant()
Match an arbitrary Constant and ignore it.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
match_bind< WithOverflowInst > m_WithOverflowInst(WithOverflowInst *&I)
Match a with overflow intrinsic, capturing it if we match.
SpecificCmpClass_match< LHS, RHS, ICmpInst > m_SpecificICmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
is_zero< true > m_Zero()
Match any null constant or a vector with all elements equal to 0.
cst_pred_ty< is_negated_power2 > m_NegatedPower2()
Match a integer or vector negated power-of-2.
match_immconstant_ty m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
DisjointOr_match< LHS, RHS, true > m_c_DisjointOr(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
SpecificCmpClass_match< LHS, RHS, FCmpInst > m_SpecificFCmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
match_combine_or< BinaryOp_match< LHS, RHS, Instruction::Add >, DisjointOr_match< LHS, RHS > > m_AddLike(const LHS &L, const RHS &R)
Match either "add" or "or disjoint".
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
match_combine_or< CastInst_match< OpTy, SExtInst >, NNegZExt_match< OpTy > > m_SExtLike(const OpTy &Op)
Match either "sext" or "zext nneg".
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
auto m_c_MaxOrMin(const LHS &L, const RHS &R)
cst_pred_ty< is_maxsignedvalue > m_MaxSignedValue()
Match an integer or vector with values having all bits except for the high bit set (0x7f....
AnyBinaryOp_match< LHS, RHS, true > m_c_BinOp(const LHS &L, const RHS &R)
Matches a BinaryOperator with LHS and RHS in either order.
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
BinOpPred_match< LHS, RHS, is_shift_op > m_Shift(const LHS &L, const RHS &R)
Matches shift operations.
LogicalOp_match< LHS, RHS, Instruction::And, true > m_c_LogicalAnd(const LHS &L, const RHS &R)
Matches L && R with LHS and RHS in either order.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
BinaryOp_match< LHS, RHS, Instruction::Or, true > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
ElementWiseBitCast_match< OpTy > m_ElementWiseBitCast(const OpTy &Op)
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
cst_pred_ty< icmp_pred_with_threshold > m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold)
Match an integer or vector with every element comparing 'pred' (eg/ne/...) to Threshold.
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
NodeAddr< CodeNode * > Code
Definition RDFGraph.h:388
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI Intrinsic::ID getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID)
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
@ Offset
Definition DWP.cpp:577
LLVM_ABI Constant * getPredForFCmpCode(unsigned Code, Type *OpTy, CmpInst::Predicate &Pred)
This is the complement of getFCmpCode.
LLVM_ABI cl::opt< bool > ProfcheckDisableMetadataFixes
LLVM_ABI bool isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS, bool &TrueIfSigned)
Given an exploded icmp instruction, return true if the comparison only checks the sign bit.
@ Known
Known to have no common set bits.
LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, StringRef PassName, const Function *F=nullptr)
Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch weights in the new instruct...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI bool predicatesFoldable(CmpInst::Predicate P1, CmpInst::Predicate P2)
Return true if both predicates match sign or if at least one of them is an equality comparison (which...
LLVM_ABI bool isKnownNegative(const WithCache< const Value * > &V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the given value is known be negative (i.e.
LLVM_ABI Value * simplifyOrInst(Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for an Or, fold the result or return null.
LLVM_ABI Value * simplifyXorInst(Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for an Xor, fold the result or return null.
LLVM_ABI bool isGuaranteedNotToBeUndef(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Returns true if V cannot be undef, but may be poison.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
LLVM_ABI bool isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL, bool OrZero=false, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Return true if the given value is known to have exactly one bit set when defined.
LLVM_ABI bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start, Value *&Step)
Attempt to match a simple first order recurrence cycle of the form: iv = phi Ty [Start,...
LLVM_ABI Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Function *CtxF=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
LLVM_ABI Constant * getLosslessUnsignedTrunc(Constant *C, Type *DestTy, const DataLayout &DL, PreservedCastFlags *Flags=nullptr)
LLVM_ABI bool recognizeBSwapOrBitReverseIdiom(Instruction *I, bool MatchBSwaps, bool MatchBitReversals, SmallVectorImpl< Instruction * > &InsertedInsts)
Try to match a bswap or bitreverse idiom.
Definition Local.cpp:3860
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
LLVM_ABI Value * simplifyICmpInst(CmpPredicate Pred, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for an ICmpInst, fold the result or return null.
LLVM_ABI Constant * getLosslessSignedTrunc(Constant *C, Type *DestTy, const DataLayout &DL, PreservedCastFlags *Flags=nullptr)
LLVM_ABI Value * simplifyAndInst(Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for an And, fold the result or return null.
LLVM_ABI bool isKnownInversion(const Value *X, const Value *Y)
Return true iff:
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
constexpr int PoisonMaskElem
@ Other
Any other memory.
Definition ModRef.h:68
LLVM_ABI Value * simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a BinaryOperator, fold the result or return null.
LLVM_ABI std::optional< DecomposedBitTest > decomposeBitTest(Value *Cond, bool LookThroughTrunc=true, bool AllowNonZeroC=false, bool DecomposeAnd=false)
Decompose an icmp into the form ((X & Mask) pred C) if possible.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
DWARFExpression::Operation Op
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
APFloat neg(APFloat X)
Returns the negated value of the argument.
Definition APFloat.h:1742
LLVM_ABI unsigned getICmpCode(CmpInst::Predicate Pred)
Encode a icmp predicate into a three bit mask.
LLVM_ABI bool isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Returns true if V cannot be poison, but may be undef.
std::pair< Value *, FPClassTest > fcmpToClassTest(FCmpInst::Predicate Pred, const Function &F, Value *LHS, Value *RHS, bool LookThroughSrc=true)
Returns a pair of values, which if passed to llvm.is.fpclass, returns the same result as an fcmp with...
unsigned getFCmpCode(CmpInst::Predicate CC)
Similar to getICmpCode but for FCmpInst.
LLVM_ABI std::optional< DecomposedBitTest > decomposeBitTestICmp(Value *LHS, Value *RHS, CmpInst::Predicate Pred, bool LookThroughTrunc=true, bool AllowNonZeroC=false, bool DecomposeAnd=false)
Decompose an icmp into the form ((X & Mask) pred C) if possible.
LLVM_ABI Constant * getPredForICmpCode(unsigned Code, bool Sign, Type *OpTy, CmpInst::Predicate &Pred)
This is the complement of getICmpCode.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
bool isCombineableWith(const DecomposedBitMaskMul Other)
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition KnownBits.h:146
Matching combinators.
SimplifyQuery getWithInstruction(const Instruction *I) const