LLVM 24.0.0git
InstCombineCompares.cpp
Go to the documentation of this file.
1//===- InstCombineCompares.cpp --------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visitICmp and visitFCmp functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/ADT/APFloat.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/APSInt.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/Statistic.h"
23#include "llvm/Analysis/Loads.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/DataLayout.h"
30#include "llvm/IR/InstrTypes.h"
31#include "llvm/IR/Instruction.h"
37#include <bitset>
38
39using namespace llvm;
40using namespace PatternMatch;
41
42#define DEBUG_TYPE "instcombine"
43
44// How many times is a select replaced by one of its operands?
45STATISTIC(NumSel, "Number of select opts");
46
47/// Compute Result = In1+In2, returning true if the result overflowed for this
48/// type.
49static bool addWithOverflow(APInt &Result, const APInt &In1, const APInt &In2,
50 bool IsSigned = false) {
51 bool Overflow;
52 if (IsSigned)
53 Result = In1.sadd_ov(In2, Overflow);
54 else
55 Result = In1.uadd_ov(In2, Overflow);
56
57 return Overflow;
58}
59
60/// Compute Result = In1-In2, returning true if the result overflowed for this
61/// type.
62static bool subWithOverflow(APInt &Result, const APInt &In1, const APInt &In2,
63 bool IsSigned = false) {
64 bool Overflow;
65 if (IsSigned)
66 Result = In1.ssub_ov(In2, Overflow);
67 else
68 Result = In1.usub_ov(In2, Overflow);
69
70 return Overflow;
71}
72
73/// Given an icmp instruction, return true if any use of this comparison is a
74/// branch on sign bit comparison.
75static bool hasBranchUse(ICmpInst &I) {
76 for (auto *U : I.users())
77 if (isa<CondBrInst>(U))
78 return true;
79 return false;
80}
81
82/// Returns true if the exploded icmp can be expressed as a signed comparison
83/// to zero and updates the predicate accordingly.
84/// The signedness of the comparison is preserved.
85/// TODO: Refactor with decomposeBitTestICmp()?
86static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) {
88 return false;
89
90 if (C.isZero())
92
93 if (C.isOne()) {
94 if (Pred == ICmpInst::ICMP_SLT) {
96 return true;
97 }
98 } else if (C.isAllOnes()) {
99 if (Pred == ICmpInst::ICMP_SGT) {
101 return true;
102 }
103 }
104
105 return false;
106}
107
108/// Try to fold
109/// cmp pred (load (gep GV, ...)), cmpcst
110/// where GV is a global variable with a constant initializer. Try to simplify
111/// this into some simple computation that does not need the load. For example
112/// we can optimize "icmp eq (load (gep "foo", 0, i)), 0" into "icmp eq i, 3".
113///
114/// If AndCst is non-null, then the loaded value is masked with that constant
115/// before doing the comparison. This handles cases like "A[i]&4 == 0".
116///
117/// We allow multi-use cases in this fold, even though it can increase
118/// instruction count, because it appears to be mostly beneficial in practice.
119/// Even if there are multiple uses, they can often be sunk into the block
120/// guarded by the icmp.
122 LoadInst *LI, CmpPredicate Pred, Constant *RHSC, Instruction &CtxI,
123 ConstantInt *AndCst) {
124 Value *Ptr = LI->getPointerOperand();
126 if (LI->isVolatile() || !GV || !GV->isConstant() ||
127 !GV->hasDefinitiveInitializer())
128 return nullptr;
129
130 Type *EltTy = LI->getType();
131 TypeSize EltSize = DL.getTypeStoreSize(EltTy);
132 if (EltSize.isScalable())
133 return nullptr;
134
136 if (!Expr.Index || Expr.BasePtr != GV || Expr.Offset.getBitWidth() > 64)
137 return nullptr;
138
139 Constant *Init = GV->getInitializer();
140 TypeSize GlobalSize = DL.getTypeAllocSize(Init->getType());
141
142 Value *Idx = Expr.Index;
143 const APInt &Stride = Expr.Scale;
144 const APInt &ConstOffset = Expr.Offset;
145
146 // Allow an additional context offset, but only within the stride.
147 if (!ConstOffset.ult(Stride))
148 return nullptr;
149
150 // Don't handle overlapping loads for now.
151 if (!Stride.uge(EltSize.getFixedValue()))
152 return nullptr;
153
154 // - With nuw: `Idx * Stride` does not wrap and all strides are valid.
155 // - With nusw: `Idx * Stride` does not wrap as signed, and the following
156 // enumeration over ArrayElementCount is valid so long as the stride is
157 // positive.
158 // - Without nusw or nuw: `Idx * Stride` is computed modulo 2^N (see MaskIdx
159 // below), and the masking in MaskIdx below is only exact when the stride
160 // is a power of two.
161 bool StrideIsValid =
162 Expr.Flags.hasNoUnsignedWrap() ||
163 (Expr.Flags.hasNoUnsignedSignedWrap() ? !Stride.isNegative()
164 : Stride.isPowerOf2());
165 if (!StrideIsValid)
166 return nullptr;
167
168 // Don't blow up on huge arrays.
169 uint64_t ArrayElementCount =
170 divideCeil((GlobalSize.getFixedValue() - ConstOffset.getZExtValue()),
171 Stride.getZExtValue());
172 if (ArrayElementCount > CLOpts.maxarray_size)
173 return nullptr;
174
175 enum { Overdefined = -3, Undefined = -2 };
176
177 // Variables for our state machines.
178
179 // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form
180 // "i == 47 | i == 87", where 47 is the first index the condition is true for,
181 // and 87 is the second (and last) index. FirstTrueElement is -2 when
182 // undefined, otherwise set to the first true element. SecondTrueElement is
183 // -2 when undefined, -3 when overdefined and >= 0 when that index is true.
184 int FirstTrueElement = Undefined, SecondTrueElement = Undefined;
185
186 // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the
187 // form "i != 47 & i != 87". Same state transitions as for true elements.
188 int FirstFalseElement = Undefined, SecondFalseElement = Undefined;
189
190 /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these
191 /// define a state machine that triggers for ranges of values that the index
192 /// is true or false for. This triggers on things like "abbbbc"[i] == 'b'.
193 /// This is -2 when undefined, -3 when overdefined, and otherwise the last
194 /// index in the range (inclusive). We use -2 for undefined here because we
195 /// use relative comparisons and don't want 0-1 to match -1.
196 int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined;
197
198 // MagicBitvector - This is a magic bitvector where we set a bit if the
199 // comparison is true for element 'i'. If there are 64 elements or less in
200 // the array, this will fully represent all the comparison results.
201 uint64_t MagicBitvector = 0;
202
203 // Scan the array and see if one of our patterns matches.
204 APInt Offset = ConstOffset;
205 for (unsigned i = 0, e = ArrayElementCount; i != e; ++i, Offset += Stride) {
207 if (!Elt)
208 return nullptr;
209
210 // If the element is masked, handle it.
211 if (AndCst) {
212 Elt = ConstantFoldBinaryOpOperands(Instruction::And, Elt, AndCst, DL);
213 if (!Elt)
214 return nullptr;
215 }
216
217 // Find out if the comparison would be true or false for the i'th element.
218 Constant *C = ConstantFoldCompareInstOperands(Pred, Elt, RHSC, DL, &TLI);
219 if (!C)
220 return nullptr;
221
222 // If the result is undef for this element, ignore it.
223 if (isa<UndefValue>(C)) {
224 // Extend range state machines to cover this element in case there is an
225 // undef in the middle of the range.
226 if (TrueRangeEnd == (int)i - 1)
227 TrueRangeEnd = i;
228 if (FalseRangeEnd == (int)i - 1)
229 FalseRangeEnd = i;
230 continue;
231 }
232
233 // If we can't compute the result for any of the elements, we have to give
234 // up evaluating the entire conditional.
235 if (!isa<ConstantInt>(C))
236 return nullptr;
237
238 // Otherwise, we know if the comparison is true or false for this element,
239 // update our state machines.
240 bool IsTrueForElt = !cast<ConstantInt>(C)->isZero();
241
242 // State machine for single/double/range index comparison.
243 if (IsTrueForElt) {
244 // Update the TrueElement state machine.
245 if (FirstTrueElement == Undefined)
246 FirstTrueElement = TrueRangeEnd = i; // First true element.
247 else {
248 // Update double-compare state machine.
249 if (SecondTrueElement == Undefined)
250 SecondTrueElement = i;
251 else
252 SecondTrueElement = Overdefined;
253
254 // Update range state machine.
255 if (TrueRangeEnd == (int)i - 1)
256 TrueRangeEnd = i;
257 else
258 TrueRangeEnd = Overdefined;
259 }
260 } else {
261 // Update the FalseElement state machine.
262 if (FirstFalseElement == Undefined)
263 FirstFalseElement = FalseRangeEnd = i; // First false element.
264 else {
265 // Update double-compare state machine.
266 if (SecondFalseElement == Undefined)
267 SecondFalseElement = i;
268 else
269 SecondFalseElement = Overdefined;
270
271 // Update range state machine.
272 if (FalseRangeEnd == (int)i - 1)
273 FalseRangeEnd = i;
274 else
275 FalseRangeEnd = Overdefined;
276 }
277 }
278
279 // If this element is in range, update our magic bitvector.
280 if (i < 64 && IsTrueForElt)
281 MagicBitvector |= 1ULL << i;
282
283 // If all of our states become overdefined and we need more than 64 bits in
284 // the bit vector, bail out.
285 if (i >= 64 && SecondTrueElement == Overdefined &&
286 SecondFalseElement == Overdefined && TrueRangeEnd == Overdefined &&
287 FalseRangeEnd == Overdefined)
288 return nullptr;
289 }
290
291 // Now that we've scanned the entire array, emit our new comparison(s). We
292 // order the state machines in complexity of the generated code.
293
294 // If none of nusw (implied by inbounds) or nuw is present, Idx * Stride can
295 // overflow.
296 // Let's assume that Stride is 2 and the wanted value is at offset 0.
297 // Then, there are two possible values for Idx to match offset 0:
298 // 0x00..00, 0x80..00.
299 // Emitting 'icmp eq Idx, 0' isn't correct in this case because the
300 // comparison is false if Idx was 0x80..00.
301 // We need to erase the highest countTrailingZeros(Stride) bits of Idx.
302 auto MaskIdx = [&](Value *Idx) {
303 if (!Expr.Flags.hasNoUnsignedWrap() &&
304 !Expr.Flags.hasNoUnsignedSignedWrap() && Stride.countr_zero() != 0) {
305 Value *Mask = Constant::getAllOnesValue(Idx->getType());
306 Mask = Builder.CreateLShr(Mask, Stride.countr_zero());
307 Idx = Builder.CreateAnd(Idx, Mask);
308 }
309 return Idx;
310 };
311
312 // If the comparison is only true for one or two elements, emit direct
313 // comparisons.
314 if (SecondTrueElement != Overdefined) {
315 Idx = MaskIdx(Idx);
316 // None true -> false.
317 if (FirstTrueElement == Undefined)
318 return replaceInstUsesWith(CtxI, Builder.getFalse());
319
320 Value *FirstTrueIdx = ConstantInt::get(Idx->getType(), FirstTrueElement);
321
322 // True for one element -> 'i == 47'.
323 if (SecondTrueElement == Undefined)
324 return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx);
325
326 // True for two elements -> 'i == 47 | i == 72'.
327 Value *C1 = Builder.CreateICmpEQ(Idx, FirstTrueIdx);
328 Value *SecondTrueIdx = ConstantInt::get(Idx->getType(), SecondTrueElement);
329 Value *C2 = Builder.CreateICmpEQ(Idx, SecondTrueIdx);
330 return BinaryOperator::CreateOr(C1, C2);
331 }
332
333 // If the comparison is only false for one or two elements, emit direct
334 // comparisons.
335 if (SecondFalseElement != Overdefined) {
336 Idx = MaskIdx(Idx);
337 // None false -> true.
338 if (FirstFalseElement == Undefined)
339 return replaceInstUsesWith(CtxI, Builder.getTrue());
340
341 Value *FirstFalseIdx = ConstantInt::get(Idx->getType(), FirstFalseElement);
342
343 // False for one element -> 'i != 47'.
344 if (SecondFalseElement == Undefined)
345 return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx);
346
347 // False for two elements -> 'i != 47 & i != 72'.
348 Value *C1 = Builder.CreateICmpNE(Idx, FirstFalseIdx);
349 Value *SecondFalseIdx =
350 ConstantInt::get(Idx->getType(), SecondFalseElement);
351 Value *C2 = Builder.CreateICmpNE(Idx, SecondFalseIdx);
352 return BinaryOperator::CreateAnd(C1, C2);
353 }
354
355 // If the comparison can be replaced with a range comparison for the elements
356 // where it is true, emit the range check.
357 if (TrueRangeEnd != Overdefined) {
358 assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare");
359 Idx = MaskIdx(Idx);
360
361 // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1).
362 if (FirstTrueElement) {
363 Value *Offs = ConstantInt::getSigned(Idx->getType(), -FirstTrueElement);
364 Idx = Builder.CreateAdd(Idx, Offs);
365 }
366
367 Value *End =
368 ConstantInt::get(Idx->getType(), TrueRangeEnd - FirstTrueElement + 1);
369 return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End);
370 }
371
372 // False range check.
373 if (FalseRangeEnd != Overdefined) {
374 assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare");
375 Idx = MaskIdx(Idx);
376 // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse).
377 if (FirstFalseElement) {
378 Value *Offs = ConstantInt::getSigned(Idx->getType(), -FirstFalseElement);
379 Idx = Builder.CreateAdd(Idx, Offs);
380 }
381
382 Value *End =
383 ConstantInt::get(Idx->getType(), FalseRangeEnd - FirstFalseElement);
384 return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End);
385 }
386
387 // If a magic bitvector captures the entire comparison state
388 // of this load, replace it with computation that does:
389 // ((magic_cst >> i) & 1) != 0
390 {
391 Type *Ty = nullptr;
392
393 // Look for an appropriate type:
394 // - The type of Idx if the magic fits
395 // - The smallest fitting legal type
396 if (ArrayElementCount <= Idx->getType()->getIntegerBitWidth())
397 Ty = Idx->getType();
398 else
399 Ty = DL.getSmallestLegalIntType(Init->getContext(), ArrayElementCount);
400
401 if (Ty) {
402 Idx = MaskIdx(Idx);
403 Value *V = Builder.CreateIntCast(Idx, Ty, false);
404 V = Builder.CreateLShr(ConstantInt::get(Ty, MagicBitvector), V);
405 V = Builder.CreateAnd(ConstantInt::get(Ty, 1), V);
406 return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, 0));
407 }
408 }
409
410 return nullptr;
411}
412
413/// Returns true if we can rewrite Start as a GEP with pointer Base
414/// and some integer offset. The nodes that need to be re-written
415/// for this transformation will be added to Explored.
417 const DataLayout &DL,
418 SetVector<Value *> &Explored) {
419 SmallVector<Value *, 16> WorkList(1, Start);
420 Explored.insert(Base);
421
422 // The following traversal gives us an order which can be used
423 // when doing the final transformation. Since in the final
424 // transformation we create the PHI replacement instructions first,
425 // we don't have to get them in any particular order.
426 //
427 // However, for other instructions we will have to traverse the
428 // operands of an instruction first, which means that we have to
429 // do a post-order traversal.
430 while (!WorkList.empty()) {
432
433 while (!WorkList.empty()) {
434 if (Explored.size() >= 100)
435 return false;
436
437 Value *V = WorkList.back();
438
439 if (Explored.contains(V)) {
440 WorkList.pop_back();
441 continue;
442 }
443
445 // We've found some value that we can't explore which is different from
446 // the base. Therefore we can't do this transformation.
447 return false;
448
449 if (auto *GEP = dyn_cast<GEPOperator>(V)) {
450 // Only allow inbounds GEPs with at most one variable offset.
451 auto IsNonConst = [](Value *V) { return !isa<ConstantInt>(V); };
452 if (!GEP->isInBounds() || count_if(GEP->indices(), IsNonConst) > 1)
453 return false;
454
455 NW = NW.intersectForOffsetAdd(GEP->getNoWrapFlags());
456 if (!Explored.contains(GEP->getOperand(0)))
457 WorkList.push_back(GEP->getOperand(0));
458 }
459
460 if (WorkList.back() == V) {
461 WorkList.pop_back();
462 // We've finished visiting this node, mark it as such.
463 Explored.insert(V);
464 }
465
466 if (auto *PN = dyn_cast<PHINode>(V)) {
467 // We cannot transform PHIs on unsplittable basic blocks.
468 if (isa<CatchSwitchInst>(PN->getParent()->getTerminator()))
469 return false;
470 Explored.insert(PN);
471 PHIs.insert(PN);
472 }
473 }
474
475 // Explore the PHI nodes further.
476 for (auto *PN : PHIs)
477 for (Value *Op : PN->incoming_values())
478 if (!Explored.contains(Op))
479 WorkList.push_back(Op);
480 }
481
482 // Make sure that we can do this. Since we can't insert GEPs in a basic
483 // block before a PHI node, we can't easily do this transformation if
484 // we have PHI node users of transformed instructions.
485 for (Value *Val : Explored) {
486 for (Value *Use : Val->uses()) {
487
488 auto *PHI = dyn_cast<PHINode>(Use);
489 auto *Inst = dyn_cast<Instruction>(Val);
490
491 if (Inst == Base || Inst == PHI || !Inst || !PHI ||
492 !Explored.contains(PHI))
493 continue;
494
495 if (PHI->getParent() == Inst->getParent())
496 return false;
497 }
498 }
499 return true;
500}
501
502// Sets the appropriate insert point on Builder where we can add
503// a replacement Instruction for V (if that is possible).
504static void setInsertionPoint(IRBuilder<> &Builder, Value *V,
505 bool Before = true) {
506 if (auto *PHI = dyn_cast<PHINode>(V)) {
507 BasicBlock *Parent = PHI->getParent();
508 Builder.SetInsertPoint(Parent->getFirstInsertionPt());
509 return;
510 }
511 if (auto *I = dyn_cast<Instruction>(V)) {
512 if (!Before)
513 I = &*std::next(I->getIterator());
514 Builder.SetInsertPoint(I);
515 return;
516 }
517 if (auto *A = dyn_cast<Argument>(V)) {
518 // Set the insertion point in the entry block.
519 BasicBlock &Entry = A->getParent()->getEntryBlock();
520 Builder.SetInsertPoint(Entry.getFirstInsertionPt());
521 return;
522 }
523 // Otherwise, this is a constant and we don't need to set a new
524 // insertion point.
525 assert(isa<Constant>(V) && "Setting insertion point for unknown value!");
526}
527
528/// Returns a re-written value of Start as an indexed GEP using Base as a
529/// pointer.
531 const DataLayout &DL,
532 SetVector<Value *> &Explored,
533 InstCombiner &IC) {
534 // Perform all the substitutions. This is a bit tricky because we can
535 // have cycles in our use-def chains.
536 // 1. Create the PHI nodes without any incoming values.
537 // 2. Create all the other values.
538 // 3. Add the edges for the PHI nodes.
539 // 4. Emit GEPs to get the original pointers.
540 // 5. Remove the original instructions.
541 Type *IndexType = IntegerType::get(
542 Base->getContext(), DL.getIndexTypeSizeInBits(Start->getType()));
543
545 NewInsts[Base] = ConstantInt::getNullValue(IndexType);
546
547 // Create the new PHI nodes, without adding any incoming values.
548 for (Value *Val : Explored) {
549 if (Val == Base)
550 continue;
551 // Create empty phi nodes. This avoids cyclic dependencies when creating
552 // the remaining instructions.
553 if (auto *PHI = dyn_cast<PHINode>(Val))
554 NewInsts[PHI] =
555 PHINode::Create(IndexType, PHI->getNumIncomingValues(),
556 PHI->getName() + ".idx", PHI->getIterator());
557 }
558 IRBuilder<> Builder(IC.getModule());
559
560 // Create all the other instructions.
561 for (Value *Val : Explored) {
562 if (NewInsts.contains(Val))
563 continue;
564
565 if (auto *GEP = dyn_cast<GEPOperator>(Val)) {
566 setInsertionPoint(Builder, GEP);
567 Value *Op = NewInsts[GEP->getOperand(0)];
568 Value *OffsetV = emitGEPOffset(&Builder, DL, GEP);
570 NewInsts[GEP] = OffsetV;
571 else
572 NewInsts[GEP] = Builder.CreateAdd(
573 Op, OffsetV, GEP->getOperand(0)->getName() + ".add",
574 /*NUW=*/NW.hasNoUnsignedWrap(),
575 /*NSW=*/NW.hasNoUnsignedSignedWrap());
576 continue;
577 }
578 if (isa<PHINode>(Val))
579 continue;
580
581 llvm_unreachable("Unexpected instruction type");
582 }
583
584 // Add the incoming values to the PHI nodes.
585 for (Value *Val : Explored) {
586 if (Val == Base)
587 continue;
588 // All the instructions have been created, we can now add edges to the
589 // phi nodes.
590 if (auto *PHI = dyn_cast<PHINode>(Val)) {
591 PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]);
592 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) {
593 Value *NewIncoming = PHI->getIncomingValue(I);
594
595 auto It = NewInsts.find(NewIncoming);
596 if (It != NewInsts.end())
597 NewIncoming = It->second;
598
599 NewPhi->addIncoming(NewIncoming, PHI->getIncomingBlock(I));
600 }
601 }
602 }
603
604 for (Value *Val : Explored) {
605 if (Val == Base)
606 continue;
607
608 setInsertionPoint(Builder, Val, false);
609 // Create GEP for external users.
610 Value *NewVal = Builder.CreateGEP(Builder.getInt8Ty(), Base, NewInsts[Val],
611 Val->getName() + ".ptr", NW);
612 IC.replaceInstUsesWith(*cast<Instruction>(Val), NewVal);
613 // Add old instruction to worklist for DCE. We don't directly remove it
614 // here because the original compare is one of the users.
616 }
617
618 return NewInsts[Start];
619}
620
621/// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant.
622/// We can look through PHIs, GEPs and casts in order to determine a common base
623/// between GEPLHS and RHS.
626 const DataLayout &DL,
627 InstCombiner &IC) {
628 // FIXME: Support vector of pointers.
629 if (GEPLHS->getType()->isVectorTy())
630 return nullptr;
631
632 if (!GEPLHS->hasAllConstantIndices())
633 return nullptr;
634
635 APInt Offset(DL.getIndexTypeSizeInBits(GEPLHS->getType()), 0);
636 Value *PtrBase =
638 /*AllowNonInbounds*/ false);
639
640 // Bail if we looked through addrspacecast.
641 if (PtrBase->getType() != GEPLHS->getType())
642 return nullptr;
643
644 // The set of nodes that will take part in this transformation.
645 SetVector<Value *> Nodes;
646 GEPNoWrapFlags NW = GEPLHS->getNoWrapFlags();
647 if (!canRewriteGEPAsOffset(RHS, PtrBase, NW, DL, Nodes))
648 return nullptr;
649
650 // We know we can re-write this as
651 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2)
652 // Since we've only looked through inbouds GEPs we know that we
653 // can't have overflow on either side. We can therefore re-write
654 // this as:
655 // OFFSET1 cmp OFFSET2
656 Value *NewRHS = rewriteGEPAsOffset(RHS, PtrBase, NW, DL, Nodes, IC);
657
658 // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written
659 // GEP having PtrBase as the pointer base, and has returned in NewRHS the
660 // offset. Since Index is the offset of LHS to the base pointer, we will now
661 // compare the offsets instead of comparing the pointers.
663 IC.Builder.getInt(Offset), NewRHS);
664}
665
666/// Fold comparisons between a GEP instruction and something else. At this point
667/// we know that the GEP is on the LHS of the comparison.
670 // Don't transform signed compares of GEPs into index compares. Even if the
671 // GEP is inbounds, the final add of the base pointer can have signed overflow
672 // and would change the result of the icmp.
673 // e.g. "&foo[0] <s &foo[1]" can't be folded to "true" because "foo" could be
674 // the maximum signed value for the pointer type.
676 return nullptr;
677
678 // Look through bitcasts and addrspacecasts. We do not however want to remove
679 // 0 GEPs.
680 if (!isa<GetElementPtrInst>(RHS))
681 RHS = RHS->stripPointerCasts();
682
683 auto CanFold = [Cond](GEPNoWrapFlags NW) {
685 return true;
686
687 // Unsigned predicates can be folded if the GEPs have *any* nowrap flags.
689 return NW != GEPNoWrapFlags::none();
690 };
691
692 auto NewICmp = [Cond](GEPNoWrapFlags NW, Value *Op1, Value *Op2) {
693 if (!NW.hasNoUnsignedWrap()) {
694 // Convert signed to unsigned comparison.
695 return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Op1, Op2);
696 }
697
698 auto *I = new ICmpInst(Cond, Op1, Op2);
699 I->setSameSign(NW.hasNoUnsignedSignedWrap());
700 return I;
701 };
702
704 if (Base.Ptr == RHS && CanFold(Base.LHSNW) && !Base.isExpensive()) {
705 // ((gep Ptr, OFFSET) cmp Ptr) ---> (OFFSET cmp 0).
706 Type *IdxTy = DL.getIndexType(GEPLHS->getType());
707 Value *Offset =
708 EmitGEPOffsets(Base.LHSGEPs, Base.LHSNW, IdxTy, /*RewriteGEPs=*/true);
709 return NewICmp(Base.LHSNW, Offset,
710 Constant::getNullValue(Offset->getType()));
711 }
712
713 if (GEPLHS->isInBounds() && ICmpInst::isEquality(Cond) &&
715 !NullPointerIsDefined(I.getFunction(),
716 RHS->getType()->getPointerAddressSpace())) {
717 // For most address spaces, an allocation can't be placed at null, but null
718 // itself is treated as a 0 size allocation in the in bounds rules. Thus,
719 // the only valid inbounds address derived from null, is null itself.
720 // Thus, we have four cases to consider:
721 // 1) Base == nullptr, Offset == 0 -> inbounds, null
722 // 2) Base == nullptr, Offset != 0 -> poison as the result is out of bounds
723 // 3) Base != nullptr, Offset == (-base) -> poison (crossing allocations)
724 // 4) Base != nullptr, Offset != (-base) -> nonnull (and possibly poison)
725 //
726 // (Note if we're indexing a type of size 0, that simply collapses into one
727 // of the buckets above.)
728 //
729 // In general, we're allowed to make values less poison (i.e. remove
730 // sources of full UB), so in this case, we just select between the two
731 // non-poison cases (1 and 4 above).
732 //
733 // For vectors, we apply the same reasoning on a per-lane basis.
734 auto *Base = GEPLHS->getPointerOperand();
735 if (GEPLHS->getType()->isVectorTy() && Base->getType()->isPointerTy()) {
736 auto EC = cast<VectorType>(GEPLHS->getType())->getElementCount();
737 Base = Builder.CreateVectorSplat(EC, Base);
738 }
739 return new ICmpInst(Cond, Base,
741 cast<Constant>(RHS), Base->getType()));
742 } else if (GEPOperator *GEPRHS = dyn_cast<GEPOperator>(RHS)) {
743 GEPNoWrapFlags NW = GEPLHS->getNoWrapFlags() & GEPRHS->getNoWrapFlags();
744
745 // If the base pointers are different, but the indices are the same, just
746 // compare the base pointer.
747 if (GEPLHS->getOperand(0) != GEPRHS->getOperand(0)) {
748 bool IndicesTheSame =
749 GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
750 GEPLHS->getPointerOperand()->getType() ==
751 GEPRHS->getPointerOperand()->getType() &&
752 GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType();
753 if (IndicesTheSame)
754 for (unsigned i = 1, e = GEPLHS->getNumOperands(); i != e; ++i)
755 if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) {
756 IndicesTheSame = false;
757 break;
758 }
759
760 // If all indices are the same, just compare the base pointers.
761 Type *BaseType = GEPLHS->getOperand(0)->getType();
762 if (IndicesTheSame &&
763 CmpInst::makeCmpResultType(BaseType) == I.getType() && CanFold(NW))
764 return new ICmpInst(Cond, GEPLHS->getOperand(0), GEPRHS->getOperand(0));
765
766 // If we're comparing GEPs with two base pointers that only differ in type
767 // and both GEPs have only constant indices or just one use, then fold
768 // the compare with the adjusted indices.
769 // FIXME: Support vector of pointers.
770 if (GEPLHS->isInBounds() && GEPRHS->isInBounds() &&
771 (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) &&
772 (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse()) &&
773 GEPLHS->getOperand(0)->stripPointerCasts() ==
774 GEPRHS->getOperand(0)->stripPointerCasts() &&
775 !GEPLHS->getType()->isVectorTy()) {
776 Value *LOffset = EmitGEPOffset(GEPLHS);
777 Value *ROffset = EmitGEPOffset(GEPRHS);
778
779 // If we looked through an addrspacecast between different sized address
780 // spaces, the LHS and RHS pointers are different sized
781 // integers. Truncate to the smaller one.
782 Type *LHSIndexTy = LOffset->getType();
783 Type *RHSIndexTy = ROffset->getType();
784 if (LHSIndexTy != RHSIndexTy) {
785 if (LHSIndexTy->getPrimitiveSizeInBits().getFixedValue() <
786 RHSIndexTy->getPrimitiveSizeInBits().getFixedValue()) {
787 ROffset = Builder.CreateTrunc(ROffset, LHSIndexTy);
788 } else
789 LOffset = Builder.CreateTrunc(LOffset, RHSIndexTy);
790 }
791
793 LOffset, ROffset);
794 return replaceInstUsesWith(I, Cmp);
795 }
796 }
797
798 if (GEPLHS->getOperand(0) == GEPRHS->getOperand(0) &&
799 GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
800 GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType()) {
801 // If the GEPs only differ by one index, compare it.
802 unsigned NumDifferences = 0; // Keep track of # differences.
803 unsigned DiffOperand = 0; // The operand that differs.
804 for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i)
805 if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) {
806 Type *LHSType = GEPLHS->getOperand(i)->getType();
807 Type *RHSType = GEPRHS->getOperand(i)->getType();
808 // FIXME: Better support for vector of pointers.
809 if (LHSType->getPrimitiveSizeInBits() !=
810 RHSType->getPrimitiveSizeInBits() ||
811 (GEPLHS->getType()->isVectorTy() &&
812 (!LHSType->isVectorTy() || !RHSType->isVectorTy()))) {
813 // Irreconcilable differences.
814 NumDifferences = 2;
815 break;
816 }
817
818 if (NumDifferences++)
819 break;
820 DiffOperand = i;
821 }
822
823 if (NumDifferences == 0) // SAME GEP?
824 return replaceInstUsesWith(
825 I, // No comparison is needed here.
826 ConstantInt::get(I.getType(), ICmpInst::isTrueWhenEqual(Cond)));
827 // If two GEPs only differ by an index, compare them.
828 // Note that nowrap flags are always needed when comparing two indices.
829 else if (NumDifferences == 1 && NW != GEPNoWrapFlags::none()) {
830 Value *LHSV = GEPLHS->getOperand(DiffOperand);
831 Value *RHSV = GEPRHS->getOperand(DiffOperand);
832 return NewICmp(NW, LHSV, RHSV);
833 }
834 }
835
836 if (Base.Ptr && !Base.isExpensive()) {
837 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2)
838 bool DoFold = CanFold(Base.LHSNW & Base.RHSNW);
839
840 if (!DoFold && Base.Ptr->getType()->isPointerTy()) {
841 // Without the flags, we can still fold if the offsets are constant and
842 // they cross the base's alignment boundary the same number of times, so
843 // either both arguments will wrap, or none of them will.
844 unsigned BW = DL.getIndexTypeSizeInBits(GEPLHS->getType());
845 APInt Alignment = APInt(BW, Base.Ptr->getPointerAlignment(DL).value());
846 APInt LOff(BW, 0);
847 APInt ROff(BW, 0);
849 DL, LOff, /*AllowNonInbounds=*/true) == Base.Ptr &&
850 RHS->stripAndAccumulateConstantOffsets(
851 DL, ROff, /*AllowNonInbounds=*/true) == Base.Ptr)
852 DoFold =
855 }
856
857 if (DoFold) {
858 Type *IdxTy = DL.getIndexType(GEPLHS->getType());
859 Value *L = EmitGEPOffsets(Base.LHSGEPs, Base.LHSNW, IdxTy,
860 /*RewriteGEP=*/true);
861 Value *R = EmitGEPOffsets(Base.RHSGEPs, Base.RHSNW, IdxTy,
862 /*RewriteGEP=*/true);
863 return NewICmp(Base.LHSNW & Base.RHSNW, L, R);
864 }
865 }
866 }
867
868 // Try convert this to an indexed compare by looking through PHIs/casts as a
869 // last resort.
870 return transformToIndexedCompare(GEPLHS, RHS, Cond, DL, *this);
871}
872
874 // It would be tempting to fold away comparisons between allocas and any
875 // pointer not based on that alloca (e.g. an argument). However, even
876 // though such pointers cannot alias, they can still compare equal.
877 //
878 // But LLVM doesn't specify where allocas get their memory, so if the alloca
879 // doesn't escape we can argue that it's impossible to guess its value, and we
880 // can therefore act as if any such guesses are wrong.
881 //
882 // However, we need to ensure that this folding is consistent: We can't fold
883 // one comparison to false, and then leave a different comparison against the
884 // same value alone (as it might evaluate to true at runtime, leading to a
885 // contradiction). As such, this code ensures that all comparisons are folded
886 // at the same time, and there are no other escapes.
887
888 struct CmpCaptureTracker : public CaptureTracker {
889 AllocaInst *Alloca;
890 bool Captured = false;
891 /// The value of the map is a bit mask of which icmp operands the alloca is
892 /// used in.
894
895 CmpCaptureTracker(AllocaInst *Alloca) : Alloca(Alloca) {}
896
897 void tooManyUses() override { Captured = true; }
898
899 Action captured(const Use *U, UseCaptureInfo CI) override {
900 // TODO(captures): Use UseCaptureInfo.
901 auto *ICmp = dyn_cast<ICmpInst>(U->getUser());
902 // We need to check that U is based *only* on the alloca, and doesn't
903 // have other contributions from a select/phi operand.
904 // TODO: We could check whether getUnderlyingObjects() reduces to one
905 // object, which would allow looking through phi nodes.
906 if (ICmp && ICmp->isEquality() && getUnderlyingObject(*U) == Alloca) {
907 // Collect equality icmps of the alloca, and don't treat them as
908 // captures.
909 ICmps[ICmp] |= 1u << U->getOperandNo();
910 return Continue;
911 }
912
913 Captured = true;
914 return Stop;
915 }
916 };
917
918 CmpCaptureTracker Tracker(Alloca);
919 PointerMayBeCaptured(Alloca, &Tracker);
920 if (Tracker.Captured)
921 return false;
922
923 bool Changed = false;
924 for (auto [ICmp, Operands] : Tracker.ICmps) {
925 switch (Operands) {
926 case 1:
927 case 2: {
928 // The alloca is only used in one icmp operand. Assume that the
929 // equality is false.
930 auto *Res = ConstantInt::get(ICmp->getType(),
931 ICmp->getPredicate() == ICmpInst::ICMP_NE);
932 replaceInstUsesWith(*ICmp, Res);
934 Changed = true;
935 break;
936 }
937 case 3:
938 // Both icmp operands are based on the alloca, so this is comparing
939 // pointer offsets, without leaking any information about the address
940 // of the alloca. Ignore such comparisons.
941 break;
942 default:
943 llvm_unreachable("Cannot happen");
944 }
945 }
946
947 return Changed;
948}
949
950/// Fold "icmp pred (X+C), X".
952 CmpPredicate Pred) {
953 // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0,
954 // so the values can never be equal. Similarly for all other "or equals"
955 // operators.
956 assert(!!C && "C should not be zero!");
957
958 // (X+1) <u X --> X >u (MAXUINT-1) --> X == 255
959 // (X+2) <u X --> X >u (MAXUINT-2) --> X > 253
960 // (X+MAXUINT) <u X --> X >u (MAXUINT-MAXUINT) --> X != 0
961 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
962 Constant *R =
963 ConstantInt::get(X->getType(), APInt::getMaxValue(C.getBitWidth()) - C);
964 return new ICmpInst(ICmpInst::ICMP_UGT, X, R);
965 }
966
967 // (X+1) >u X --> X <u (0-1) --> X != 255
968 // (X+2) >u X --> X <u (0-2) --> X <u 254
969 // (X+MAXUINT) >u X --> X <u (0-MAXUINT) --> X <u 1 --> X == 0
970 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
971 return new ICmpInst(ICmpInst::ICMP_ULT, X,
972 ConstantInt::get(X->getType(), -C));
973
974 APInt SMax = APInt::getSignedMaxValue(C.getBitWidth());
975
976 // (X+ 1) <s X --> X >s (MAXSINT-1) --> X == 127
977 // (X+ 2) <s X --> X >s (MAXSINT-2) --> X >s 125
978 // (X+MAXSINT) <s X --> X >s (MAXSINT-MAXSINT) --> X >s 0
979 // (X+MINSINT) <s X --> X >s (MAXSINT-MINSINT) --> X >s -1
980 // (X+ -2) <s X --> X >s (MAXSINT- -2) --> X >s 126
981 // (X+ -1) <s X --> X >s (MAXSINT- -1) --> X != 127
982 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
983 return new ICmpInst(ICmpInst::ICMP_SGT, X,
984 ConstantInt::get(X->getType(), SMax - C));
985
986 // (X+ 1) >s X --> X <s (MAXSINT-(1-1)) --> X != 127
987 // (X+ 2) >s X --> X <s (MAXSINT-(2-1)) --> X <s 126
988 // (X+MAXSINT) >s X --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1
989 // (X+MINSINT) >s X --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2
990 // (X+ -2) >s X --> X <s (MAXSINT-(-2-1)) --> X <s -126
991 // (X+ -1) >s X --> X <s (MAXSINT-(-1-1)) --> X == -128
992
993 assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE);
994 return new ICmpInst(ICmpInst::ICMP_SLT, X,
995 ConstantInt::get(X->getType(), SMax - (C - 1)));
996}
997
998/// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" ->
999/// (icmp eq/ne A, Log2(AP2/AP1)) ->
1000/// (icmp eq/ne A, Log2(AP2) - Log2(AP1)).
1002 const APInt &AP1,
1003 const APInt &AP2) {
1004 assert(I.isEquality() && "Cannot fold icmp gt/lt");
1005
1006 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
1007 if (I.getPredicate() == I.ICMP_NE)
1008 Pred = CmpInst::getInversePredicate(Pred);
1009 return new ICmpInst(Pred, LHS, RHS);
1010 };
1011
1012 // Don't bother doing any work for cases which InstSimplify handles.
1013 if (AP2.isZero())
1014 return nullptr;
1015
1016 bool IsAShr = isa<AShrOperator>(I.getOperand(0));
1017 if (IsAShr) {
1018 if (AP2.isAllOnes())
1019 return nullptr;
1020 if (AP2.isNegative() != AP1.isNegative())
1021 return nullptr;
1022 if (AP2.sgt(AP1))
1023 return nullptr;
1024 }
1025
1026 if (!AP1)
1027 // 'A' must be large enough to shift out the highest set bit.
1028 return getICmp(I.ICMP_UGT, A,
1029 ConstantInt::get(A->getType(), AP2.logBase2()));
1030
1031 if (AP1 == AP2)
1032 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType()));
1033
1034 int Shift;
1035 if (IsAShr && AP1.isNegative())
1036 Shift = AP1.countl_one() - AP2.countl_one();
1037 else
1038 Shift = AP1.countl_zero() - AP2.countl_zero();
1039
1040 if (Shift > 0) {
1041 if (IsAShr && AP1 == AP2.ashr(Shift)) {
1042 // There are multiple solutions if we are comparing against -1 and the LHS
1043 // of the ashr is not a power of two.
1044 if (AP1.isAllOnes() && !AP2.isPowerOf2())
1045 return getICmp(I.ICMP_UGE, A, ConstantInt::get(A->getType(), Shift));
1046 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1047 } else if (AP1 == AP2.lshr(Shift)) {
1048 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1049 }
1050 }
1051
1052 // Shifting const2 will never be equal to const1.
1053 // FIXME: This should always be handled by InstSimplify?
1054 auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE);
1055 return replaceInstUsesWith(I, TorF);
1056}
1057
1058/// Handle "(icmp eq/ne (shl AP2, A), AP1)" ->
1059/// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)).
1061 const APInt &AP1,
1062 const APInt &AP2) {
1063 assert(I.isEquality() && "Cannot fold icmp gt/lt");
1064
1065 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
1066 if (I.getPredicate() == I.ICMP_NE)
1067 Pred = CmpInst::getInversePredicate(Pred);
1068 return new ICmpInst(Pred, LHS, RHS);
1069 };
1070
1071 // Don't bother doing any work for cases which InstSimplify handles.
1072 if (AP2.isZero())
1073 return nullptr;
1074
1075 unsigned AP2TrailingZeros = AP2.countr_zero();
1076
1077 if (!AP1 && AP2TrailingZeros != 0)
1078 return getICmp(
1079 I.ICMP_UGE, A,
1080 ConstantInt::get(A->getType(), AP2.getBitWidth() - AP2TrailingZeros));
1081
1082 if (AP1 == AP2)
1083 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType()));
1084
1085 // Get the distance between the lowest bits that are set.
1086 int Shift = AP1.countr_zero() - AP2TrailingZeros;
1087
1088 if (Shift > 0 && AP2.shl(Shift) == AP1)
1089 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1090
1091 // Shifting const2 will never be equal to const1.
1092 // FIXME: This should always be handled by InstSimplify?
1093 auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE);
1094 return replaceInstUsesWith(I, TorF);
1095}
1096
1097/// The caller has matched a pattern of the form:
1098/// I = icmp ugt (add (add A, B), CI2), CI1
1099/// If this is of the form:
1100/// sum = a + b
1101/// if (sum+128 >u 255)
1102/// Then replace it with llvm.sadd.with.overflow.i8.
1103///
1105 ConstantInt *CI2, ConstantInt *CI1,
1106 InstCombinerImpl &IC) {
1107 // The transformation we're trying to do here is to transform this into an
1108 // llvm.sadd.with.overflow. To do this, we have to replace the original add
1109 // with a narrower add, and discard the add-with-constant that is part of the
1110 // range check (if we can't eliminate it, this isn't profitable).
1111
1112 // In order to eliminate the add-with-constant, the compare can be its only
1113 // use.
1114 Instruction *AddWithCst = cast<Instruction>(I.getOperand(0));
1115 if (!AddWithCst->hasOneUse())
1116 return nullptr;
1117
1118 // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow.
1119 if (!CI2->getValue().isPowerOf2())
1120 return nullptr;
1121 unsigned NewWidth = CI2->getValue().countr_zero();
1122 if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31)
1123 return nullptr;
1124
1125 // The width of the new add formed is 1 more than the bias.
1126 ++NewWidth;
1127
1128 // Check to see that CI1 is an all-ones value with NewWidth bits.
1129 if (CI1->getBitWidth() == NewWidth ||
1130 CI1->getValue() != APInt::getLowBitsSet(CI1->getBitWidth(), NewWidth))
1131 return nullptr;
1132
1133 // This is only really a signed overflow check if the inputs have been
1134 // sign-extended; check for that condition. For example, if CI2 is 2^31 and
1135 // the operands of the add are 64 bits wide, we need at least 33 sign bits.
1136 if (IC.ComputeMaxSignificantBits(A, &I) > NewWidth ||
1137 IC.ComputeMaxSignificantBits(B, &I) > NewWidth)
1138 return nullptr;
1139
1140 // In order to replace the original add with a narrower
1141 // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant
1142 // and truncates that discard the high bits of the add. Verify that this is
1143 // the case.
1144 Instruction *OrigAdd = cast<Instruction>(AddWithCst->getOperand(0));
1145 for (User *U : OrigAdd->users()) {
1146 if (U == AddWithCst)
1147 continue;
1148
1149 // Only accept truncates for now. We would really like a nice recursive
1150 // predicate like SimplifyDemandedBits, but which goes downwards the use-def
1151 // chain to see which bits of a value are actually demanded. If the
1152 // original add had another add which was then immediately truncated, we
1153 // could still do the transformation.
1155 if (!TI || TI->getType()->getPrimitiveSizeInBits() > NewWidth)
1156 return nullptr;
1157 }
1158
1159 // If the pattern matches, truncate the inputs to the narrower type and
1160 // use the sadd_with_overflow intrinsic to efficiently compute both the
1161 // result and the overflow bit.
1162 Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth);
1164 I.getModule(), Intrinsic::sadd_with_overflow, NewType);
1165
1166 InstCombiner::BuilderTy &Builder = IC.Builder;
1167
1168 // Put the new code above the original add, in case there are any uses of the
1169 // add between the add and the compare.
1170 Builder.SetInsertPoint(OrigAdd);
1171
1172 Value *TruncA = Builder.CreateTrunc(A, NewType, A->getName() + ".trunc");
1173 Value *TruncB = Builder.CreateTrunc(B, NewType, B->getName() + ".trunc");
1174 CallInst *Call = Builder.CreateCall(F, {TruncA, TruncB}, "sadd");
1175 Value *Add = Builder.CreateExtractValue(Call, 0, "sadd.result");
1176 Value *ZExt = Builder.CreateZExt(Add, OrigAdd->getType());
1177
1178 // The inner add was the result of the narrow add, zero extended to the
1179 // wider type. Replace it with the result computed by the intrinsic.
1180 IC.replaceInstUsesWith(*OrigAdd, ZExt);
1181 IC.eraseInstFromFunction(*OrigAdd);
1182
1183 // The original icmp gets replaced with the overflow value.
1184 return ExtractValueInst::Create(Call, 1, "sadd.overflow");
1185}
1186
1187/// If we have:
1188/// icmp eq/ne (urem/srem %x, %y), 0
1189/// iff %y is a power-of-two, we can replace this with a bit test:
1190/// icmp eq/ne (and %x, (add %y, -1)), 0
1192 // This fold is only valid for equality predicates.
1193 if (!I.isEquality())
1194 return nullptr;
1195 CmpPredicate Pred;
1196 Value *X, *Y, *Zero;
1197 if (!match(&I, m_ICmp(Pred, m_OneUse(m_IRem(m_Value(X), m_Value(Y))),
1199 return nullptr;
1200 if (!isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, &I))
1201 return nullptr;
1202 // This may increase instruction count, we don't enforce that Y is a constant.
1203 Value *Mask = Builder.CreateAdd(Y, Constant::getAllOnesValue(Y->getType()));
1204 Value *Masked = Builder.CreateAnd(X, Mask);
1205 return ICmpInst::Create(Instruction::ICmp, Pred, Masked, Zero);
1206}
1207
1208/// Fold equality-comparison between zero and any (maybe truncated) right-shift
1209/// by one-less-than-bitwidth into a sign test on the original value.
1211 Instruction *Val;
1212 CmpPredicate Pred;
1213 if (!I.isEquality() || !match(&I, m_ICmp(Pred, m_Instruction(Val), m_Zero())))
1214 return nullptr;
1215
1216 Value *X;
1217 Type *XTy;
1218
1219 Constant *C;
1220 if (match(Val, m_TruncOrSelf(m_Shr(m_Value(X), m_Constant(C))))) {
1221 XTy = X->getType();
1222 unsigned XBitWidth = XTy->getScalarSizeInBits();
1224 APInt(XBitWidth, XBitWidth - 1))))
1225 return nullptr;
1226 } else if (isa<BinaryOperator>(Val) &&
1228 cast<BinaryOperator>(Val), SQ.getWithInstruction(Val),
1229 /*AnalyzeForSignBitExtraction=*/true))) {
1230 XTy = X->getType();
1231 } else
1232 return nullptr;
1233
1234 return ICmpInst::Create(Instruction::ICmp,
1238}
1239
1240// Handle icmp pred X, 0
1242 CmpInst::Predicate Pred = Cmp.getPredicate();
1243 if (!match(Cmp.getOperand(1), m_Zero()))
1244 return nullptr;
1245
1246 // (icmp sgt smin(PosA, B) 0) -> (icmp sgt B 0)
1247 if (Pred == ICmpInst::ICMP_SGT) {
1248 Value *A, *B;
1249 if (match(Cmp.getOperand(0), m_SMin(m_Value(A), m_Value(B)))) {
1250 if (isKnownPositive(A, SQ.getWithInstruction(&Cmp)))
1251 return new ICmpInst(Pred, B, Cmp.getOperand(1));
1252 if (isKnownPositive(B, SQ.getWithInstruction(&Cmp)))
1253 return new ICmpInst(Pred, A, Cmp.getOperand(1));
1254 }
1255 }
1256
1258 return New;
1259
1260 // Given:
1261 // icmp eq/ne (urem %x, %y), 0
1262 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
1263 // icmp eq/ne %x, 0
1264 Value *X, *Y;
1265 if (match(Cmp.getOperand(0), m_URem(m_Value(X), m_Value(Y))) &&
1266 ICmpInst::isEquality(Pred)) {
1267 KnownBits XKnown = computeKnownBits(X, &Cmp);
1268 KnownBits YKnown = computeKnownBits(Y, &Cmp);
1269 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
1270 return new ICmpInst(Pred, X, Cmp.getOperand(1));
1271 }
1272
1273 // (icmp eq/ne (mul X Y)) -> (icmp eq/ne X/Y) if we know about whether X/Y are
1274 // odd/non-zero/there is no overflow.
1275 if (match(Cmp.getOperand(0), m_Mul(m_Value(X), m_Value(Y))) &&
1276 ICmpInst::isEquality(Pred)) {
1277
1278 KnownBits XKnown = computeKnownBits(X, &Cmp);
1279 // if X % 2 != 0
1280 // (icmp eq/ne Y)
1281 if (XKnown.countMaxTrailingZeros() == 0)
1282 return new ICmpInst(Pred, Y, Cmp.getOperand(1));
1283
1284 KnownBits YKnown = computeKnownBits(Y, &Cmp);
1285 // if Y % 2 != 0
1286 // (icmp eq/ne X)
1287 if (YKnown.countMaxTrailingZeros() == 0)
1288 return new ICmpInst(Pred, X, Cmp.getOperand(1));
1289
1290 auto *BO0 = cast<OverflowingBinaryOperator>(Cmp.getOperand(0));
1291 if (BO0->hasNoUnsignedWrap() || BO0->hasNoSignedWrap()) {
1292 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
1293 // `isKnownNonZero` does more analysis than just `!KnownBits.One.isZero()`
1294 // but to avoid unnecessary work, first just if this is an obvious case.
1295
1296 // if X non-zero and NoOverflow(X * Y)
1297 // (icmp eq/ne Y)
1298 if (!XKnown.One.isZero() || isKnownNonZero(X, Q))
1299 return new ICmpInst(Pred, Y, Cmp.getOperand(1));
1300
1301 // if Y non-zero and NoOverflow(X * Y)
1302 // (icmp eq/ne X)
1303 if (!YKnown.One.isZero() || isKnownNonZero(Y, Q))
1304 return new ICmpInst(Pred, X, Cmp.getOperand(1));
1305 }
1306 // Note, we are skipping cases:
1307 // if Y % 2 != 0 AND X % 2 != 0
1308 // (false/true)
1309 // if X non-zero and Y non-zero and NoOverflow(X * Y)
1310 // (false/true)
1311 // Those can be simplified later as we would have already replaced the (icmp
1312 // eq/ne (mul X, Y)) with (icmp eq/ne X/Y) and if X/Y is known non-zero that
1313 // will fold to a constant elsewhere.
1314 }
1315
1316 // (icmp eq/ne f(X), 0) -> (icmp eq/ne X, 0)
1317 // where f(X) == 0 if and only if X == 0
1318 if (ICmpInst::isEquality(Pred))
1319 if (Value *Stripped = stripNullTest(Cmp.getOperand(0)))
1320 return new ICmpInst(Pred, Stripped,
1321 Constant::getNullValue(Stripped->getType()));
1322
1323 return nullptr;
1324}
1325
1326/// Fold icmp eq (num + mask) & ~mask, num
1327/// to
1328/// icmp eq (and num, mask), 0
1329/// Where mask is a low bit mask.
1331 Value *Num;
1332 CmpPredicate Pred;
1333 const APInt *Mask, *Neg;
1334
1335 if (!match(&Cmp,
1336 m_c_ICmp(Pred, m_Value(Num),
1338 m_LowBitMask(Mask))),
1339 m_APInt(Neg))))))
1340 return nullptr;
1341
1342 if (*Neg != ~*Mask)
1343 return nullptr;
1344
1345 if (!ICmpInst::isEquality(Pred))
1346 return nullptr;
1347
1348 // Create new icmp eq (num & mask), 0
1349 auto *NewAnd = Builder.CreateAnd(Num, *Mask);
1350 auto *Zero = Constant::getNullValue(Num->getType());
1351
1352 return new ICmpInst(Pred, NewAnd, Zero);
1353}
1354
1355/// Fold icmp Pred X, C.
1356/// TODO: This code structure does not make sense. The saturating add fold
1357/// should be moved to some other helper and extended as noted below (it is also
1358/// possible that code has been made unnecessary - do we canonicalize IR to
1359/// overflow/saturating intrinsics or not?).
1361 // Match the following pattern, which is a common idiom when writing
1362 // overflow-safe integer arithmetic functions. The source performs an addition
1363 // in wider type and explicitly checks for overflow using comparisons against
1364 // INT_MIN and INT_MAX. Simplify by using the sadd_with_overflow intrinsic.
1365 //
1366 // TODO: This could probably be generalized to handle other overflow-safe
1367 // operations if we worked out the formulas to compute the appropriate magic
1368 // constants.
1369 //
1370 // sum = a + b
1371 // if (sum+128 >u 255) ... -> llvm.sadd.with.overflow.i8
1372 CmpInst::Predicate Pred = Cmp.getPredicate();
1373 Value *Op0 = Cmp.getOperand(0), *Op1 = Cmp.getOperand(1);
1374 Value *A, *B;
1375 ConstantInt *CI, *CI2; // I = icmp ugt (add (add A, B), CI2), CI
1376 if (Pred == ICmpInst::ICMP_UGT && match(Op1, m_ConstantInt(CI)) &&
1377 match(Op0, m_Add(m_Add(m_Value(A), m_Value(B)), m_ConstantInt(CI2))))
1378 if (Instruction *Res = processUGT_ADDCST_ADD(Cmp, A, B, CI2, CI, *this))
1379 return Res;
1380
1381 // icmp(phi(C1, C2, ...), C) -> phi(icmp(C1, C), icmp(C2, C), ...).
1383 if (!C)
1384 return nullptr;
1385
1386 if (auto *Phi = dyn_cast<PHINode>(Op0))
1387 if (all_of(Phi->operands(), IsaPred<Constant>)) {
1389 for (Value *V : Phi->incoming_values()) {
1390 Constant *Res =
1392 if (!Res)
1393 return nullptr;
1394 Ops.push_back(Res);
1395 }
1396 Builder.SetInsertPoint(Phi);
1397 PHINode *NewPhi = Builder.CreatePHI(Cmp.getType(), Phi->getNumOperands());
1398 for (auto [V, Pred] : zip(Ops, Phi->blocks()))
1399 NewPhi->addIncoming(V, Pred);
1400 return replaceInstUsesWith(Cmp, NewPhi);
1401 }
1402
1404 return R;
1405
1406 return nullptr;
1407}
1408
1409/// Canonicalize icmp instructions based on dominating conditions.
1411 // We already checked simple implication in InstSimplify, only handle complex
1412 // cases here.
1413 Value *X = Cmp.getOperand(0), *Y = Cmp.getOperand(1);
1414 const APInt *C;
1415 if (!match(Y, m_APInt(C)))
1416 return nullptr;
1417
1418 CmpInst::Predicate Pred = Cmp.getPredicate();
1420
1421 auto handleDomCond = [&](ICmpInst::Predicate DomPred,
1422 const APInt *DomC) -> Instruction * {
1423 // We have 2 compares of a variable with constants. Calculate the constant
1424 // ranges of those compares to see if we can transform the 2nd compare:
1425 // DomBB:
1426 // DomCond = icmp DomPred X, DomC
1427 // br DomCond, CmpBB, FalseBB
1428 // CmpBB:
1429 // Cmp = icmp Pred X, C
1430 ConstantRange DominatingCR =
1431 ConstantRange::makeExactICmpRegion(DomPred, *DomC);
1432 ConstantRange Intersection = DominatingCR.intersectWith(CR);
1433 ConstantRange Difference = DominatingCR.difference(CR);
1434 if (Intersection.isEmptySet())
1435 return replaceInstUsesWith(Cmp, Builder.getFalse());
1436 if (Difference.isEmptySet())
1437 return replaceInstUsesWith(Cmp, Builder.getTrue());
1438
1439 // Canonicalizing a sign bit comparison that gets used in a branch,
1440 // pessimizes codegen by generating branch on zero instruction instead
1441 // of a test and branch. So we avoid canonicalizing in such situations
1442 // because test and branch instruction has better branch displacement
1443 // than compare and branch instruction.
1444 bool UnusedBit;
1445 bool IsSignBit = isSignBitCheck(Pred, *C, UnusedBit);
1446 if (Cmp.isEquality() || (IsSignBit && hasBranchUse(Cmp)))
1447 return nullptr;
1448
1449 // Avoid an infinite loop with min/max canonicalization.
1450 // TODO: This will be unnecessary if we canonicalize to min/max intrinsics.
1451 if (Cmp.hasOneUse() &&
1452 match(Cmp.user_back(), m_MaxOrMin(m_Value(), m_Value())))
1453 return nullptr;
1454
1455 if (const APInt *EqC = Intersection.getSingleElement())
1456 return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(*EqC));
1457 if (const APInt *NeC = Difference.getSingleElement())
1458 return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(*NeC));
1459 return nullptr;
1460 };
1461
1462 for (CondBrInst *BI : DC.conditionsFor(X)) {
1463 CmpPredicate DomPred;
1464 const APInt *DomC;
1465 if (!match(BI->getCondition(),
1466 m_ICmp(DomPred, m_Specific(X), m_APInt(DomC))))
1467 continue;
1468
1469 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(0));
1470 if (DT.dominates(Edge0, Cmp.getParent())) {
1471 if (auto *V = handleDomCond(DomPred, DomC))
1472 return V;
1473 } else {
1474 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(1));
1475 if (DT.dominates(Edge1, Cmp.getParent()))
1476 if (auto *V =
1477 handleDomCond(CmpInst::getInversePredicate(DomPred), DomC))
1478 return V;
1479 }
1480 }
1481
1482 return nullptr;
1483}
1484
1485/// Fold icmp (trunc X), C.
1487 TruncInst *Trunc,
1488 const APInt &C) {
1489 ICmpInst::Predicate Pred = Cmp.getPredicate();
1490 Value *X = Trunc->getOperand(0);
1491 Type *SrcTy = X->getType();
1492 unsigned DstBits = Trunc->getType()->getScalarSizeInBits(),
1493 SrcBits = SrcTy->getScalarSizeInBits();
1494
1495 // Match (icmp pred (trunc nuw/nsw X), C)
1496 // Which we can convert to (icmp pred X, (sext/zext C))
1497 if (shouldChangeType(Trunc->getType(), SrcTy)) {
1498 if (Trunc->hasNoSignedWrap())
1499 return new ICmpInst(Pred, X, ConstantInt::get(SrcTy, C.sext(SrcBits)));
1500 if (!Cmp.isSigned() && Trunc->hasNoUnsignedWrap())
1501 return new ICmpInst(Pred, X, ConstantInt::get(SrcTy, C.zext(SrcBits)));
1502 }
1503
1504 if (C.isOne() && C.getBitWidth() > 1) {
1505 // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1
1506 Value *V = nullptr;
1507 if (Pred == ICmpInst::ICMP_SLT && match(X, m_Signum(m_Value(V))))
1508 return new ICmpInst(ICmpInst::ICMP_SLT, V,
1509 ConstantInt::get(V->getType(), 1));
1510 }
1511
1512 // TODO: Handle non-equality predicates.
1513 Value *Y;
1514 const APInt *Pow2;
1515 if (Cmp.isEquality() && match(X, m_Shl(m_Power2(Pow2), m_Value(Y))) &&
1516 DstBits > Pow2->logBase2()) {
1517 // (trunc (Pow2 << Y) to iN) == 0 --> Y u>= N - log2(Pow2)
1518 // (trunc (Pow2 << Y) to iN) != 0 --> Y u< N - log2(Pow2)
1519 // iff N > log2(Pow2)
1520 if (C.isZero()) {
1521 auto NewPred = (Pred == Cmp.ICMP_EQ) ? Cmp.ICMP_UGE : Cmp.ICMP_ULT;
1522 return new ICmpInst(NewPred, Y,
1523 ConstantInt::get(SrcTy, DstBits - Pow2->logBase2()));
1524 }
1525 // (trunc (Pow2 << Y) to iN) == 2**C --> Y == C - log2(Pow2)
1526 // (trunc (Pow2 << Y) to iN) != 2**C --> Y != C - log2(Pow2)
1527 if (C.isPowerOf2())
1528 return new ICmpInst(
1529 Pred, Y, ConstantInt::get(SrcTy, C.logBase2() - Pow2->logBase2()));
1530 }
1531
1532 if (Cmp.isEquality() && (Trunc->hasOneUse() || Trunc->hasNoUnsignedWrap())) {
1533 // Canonicalize to a mask and wider compare if the wide type is suitable:
1534 // (trunc X to i8) == C --> (X & 0xff) == (zext C)
1535 if (!SrcTy->isVectorTy() && shouldChangeType(DstBits, SrcBits)) {
1536 Constant *Mask =
1537 ConstantInt::get(SrcTy, APInt::getLowBitsSet(SrcBits, DstBits));
1538 Value *And = Trunc->hasNoUnsignedWrap() ? X : Builder.CreateAnd(X, Mask);
1539 Constant *WideC = ConstantInt::get(SrcTy, C.zext(SrcBits));
1540 return new ICmpInst(Pred, And, WideC);
1541 }
1542
1543 // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all
1544 // of the high bits truncated out of x are known.
1546
1547 // If all the high bits are known, we can do this xform.
1548 if ((Known.Zero | Known.One).countl_one() >= SrcBits - DstBits) {
1549 // Pull in the high bits from known-ones set.
1550 APInt NewRHS = C.zext(SrcBits);
1551 NewRHS |= Known.One & APInt::getHighBitsSet(SrcBits, SrcBits - DstBits);
1552 return new ICmpInst(Pred, X, ConstantInt::get(SrcTy, NewRHS));
1553 }
1554 }
1555
1556 // Look through truncated right-shift of the sign-bit for a sign-bit check:
1557 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] < 0 --> ShOp < 0
1558 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] > -1 --> ShOp > -1
1559 Value *ShOp;
1560 uint64_t ShAmt;
1561 bool TrueIfSigned;
1562 if (isSignBitCheck(Pred, C, TrueIfSigned) &&
1563 match(X, m_Shr(m_Value(ShOp), m_ConstantInt(ShAmt))) &&
1564 DstBits == SrcBits - ShAmt) {
1565 return TrueIfSigned ? new ICmpInst(ICmpInst::ICMP_SLT, ShOp,
1567 : new ICmpInst(ICmpInst::ICMP_SGT, ShOp,
1569 }
1570
1571 return nullptr;
1572}
1573
1574/// Fold icmp (trunc nuw/nsw X), (trunc nuw/nsw Y).
1575/// Fold icmp (trunc nuw/nsw X), (zext/sext Y).
1578 const SimplifyQuery &Q) {
1579 Value *X, *Y;
1580 CmpPredicate Pred;
1581 bool YIsSExt = false;
1582 // Try to match icmp (trunc X), (trunc Y)
1583 if (match(&Cmp, m_ICmp(Pred, m_Trunc(m_Value(X)), m_Trunc(m_Value(Y))))) {
1584 unsigned NoWrapFlags = cast<TruncInst>(Cmp.getOperand(0))->getNoWrapKind() &
1585 cast<TruncInst>(Cmp.getOperand(1))->getNoWrapKind();
1586 if (Cmp.isSigned()) {
1587 // For signed comparisons, both truncs must be nsw.
1588 if (!(NoWrapFlags & TruncInst::NoSignedWrap))
1589 return nullptr;
1590 } else {
1591 // For unsigned and equality comparisons, either both must be nuw or
1592 // both must be nsw, we don't care which.
1593 if (!NoWrapFlags)
1594 return nullptr;
1595 }
1596
1597 if (X->getType() != Y->getType() &&
1598 (!Cmp.getOperand(0)->hasOneUse() || !Cmp.getOperand(1)->hasOneUse()))
1599 return nullptr;
1600 if (!isDesirableIntType(X->getType()->getScalarSizeInBits()) &&
1601 isDesirableIntType(Y->getType()->getScalarSizeInBits())) {
1602 std::swap(X, Y);
1603 Pred = Cmp.getSwappedPredicate(Pred);
1604 }
1605 YIsSExt = !(NoWrapFlags & TruncInst::NoUnsignedWrap);
1606 }
1607 // Try to match icmp (trunc nuw X), (zext Y)
1608 else if (!Cmp.isSigned() &&
1609 match(&Cmp, m_c_ICmp(Pred, m_NUWTrunc(m_Value(X)),
1610 m_OneUse(m_ZExt(m_Value(Y)))))) {
1611 // Can fold trunc nuw + zext for unsigned and equality predicates.
1612 }
1613 // Try to match icmp (trunc nsw X), (sext Y)
1614 else if (match(&Cmp, m_c_ICmp(Pred, m_NSWTrunc(m_Value(X)),
1616 // Can fold trunc nsw + zext/sext for all predicates.
1617 YIsSExt =
1618 isa<SExtInst>(Cmp.getOperand(0)) || isa<SExtInst>(Cmp.getOperand(1));
1619 } else
1620 return nullptr;
1621
1622 Type *TruncTy = Cmp.getOperand(0)->getType();
1623 unsigned TruncBits = TruncTy->getScalarSizeInBits();
1624
1625 // If this transform will end up changing from desirable types -> undesirable
1626 // types skip it.
1627 if (isDesirableIntType(TruncBits) &&
1628 !isDesirableIntType(X->getType()->getScalarSizeInBits()))
1629 return nullptr;
1630
1631 Value *NewY = Builder.CreateIntCast(Y, X->getType(), YIsSExt);
1632 return new ICmpInst(Pred, X, NewY);
1633}
1634
1635/// Fold icmp (xor X, Y), C.
1638 const APInt &C) {
1639 if (Instruction *I = foldICmpXorShiftConst(Cmp, Xor, C))
1640 return I;
1641
1642 Value *X = Xor->getOperand(0);
1643 Value *Y = Xor->getOperand(1);
1644 const APInt *XorC;
1645 if (!match(Y, m_APInt(XorC)))
1646 return nullptr;
1647
1648 // If this is a comparison that tests the signbit (X < 0) or (x > -1),
1649 // fold the xor.
1650 ICmpInst::Predicate Pred = Cmp.getPredicate();
1651 bool TrueIfSigned = false;
1652 if (isSignBitCheck(Cmp.getPredicate(), C, TrueIfSigned)) {
1653
1654 // If the sign bit of the XorCst is not set, there is no change to
1655 // the operation, just stop using the Xor.
1656 if (!XorC->isNegative())
1657 return replaceOperand(Cmp, 0, X);
1658
1659 // Emit the opposite comparison.
1660 if (TrueIfSigned)
1661 return new ICmpInst(ICmpInst::ICMP_SGT, X,
1662 ConstantInt::getAllOnesValue(X->getType()));
1663 else
1664 return new ICmpInst(ICmpInst::ICMP_SLT, X,
1665 ConstantInt::getNullValue(X->getType()));
1666 }
1667
1668 if (Xor->hasOneUse()) {
1669 // (icmp u/s (xor X SignMask), C) -> (icmp s/u X, (xor C SignMask))
1670 if (!Cmp.isEquality() && XorC->isSignMask()) {
1671 Pred = Cmp.getFlippedSignednessPredicate();
1672 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC));
1673 }
1674
1675 // (icmp u/s (xor X ~SignMask), C) -> (icmp s/u X, (xor C ~SignMask))
1676 if (!Cmp.isEquality() && XorC->isMaxSignedValue()) {
1677 Pred = Cmp.getFlippedSignednessPredicate();
1678 Pred = Cmp.getSwappedPredicate(Pred);
1679 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC));
1680 }
1681 }
1682
1683 // Mask constant magic can eliminate an 'xor' with unsigned compares.
1684 if (Pred == ICmpInst::ICMP_UGT) {
1685 // (xor X, ~C) >u C --> X <u ~C (when C+1 is a power of 2)
1686 if (*XorC == ~C && (C + 1).isPowerOf2())
1687 return new ICmpInst(ICmpInst::ICMP_ULT, X, Y);
1688 // (xor X, C) >u C --> X >u C (when C+1 is a power of 2)
1689 if (*XorC == C && (C + 1).isPowerOf2())
1690 return new ICmpInst(ICmpInst::ICMP_UGT, X, Y);
1691 }
1692 if (Pred == ICmpInst::ICMP_ULT) {
1693 // (xor X, -C) <u C --> X >u ~C (when C is a power of 2)
1694 if (*XorC == -C && C.isPowerOf2())
1695 return new ICmpInst(ICmpInst::ICMP_UGT, X,
1696 ConstantInt::get(X->getType(), ~C));
1697 // (xor X, C) <u C --> X >u ~C (when -C is a power of 2)
1698 if (*XorC == C && (-C).isPowerOf2())
1699 return new ICmpInst(ICmpInst::ICMP_UGT, X,
1700 ConstantInt::get(X->getType(), ~C));
1701 }
1702 return nullptr;
1703}
1704
1705/// For power-of-2 C:
1706/// ((X s>> ShiftC) ^ X) u< C --> (X + C) u< (C << 1)
1707/// ((X s>> ShiftC) ^ X) u> (C - 1) --> (X + C) u> ((C << 1) - 1)
1710 const APInt &C) {
1711 CmpInst::Predicate Pred = Cmp.getPredicate();
1712 APInt PowerOf2;
1713 if (Pred == ICmpInst::ICMP_ULT)
1714 PowerOf2 = C;
1715 else if (Pred == ICmpInst::ICMP_UGT && !C.isMaxValue())
1716 PowerOf2 = C + 1;
1717 else
1718 return nullptr;
1719 if (!PowerOf2.isPowerOf2())
1720 return nullptr;
1721 Value *X;
1722 const APInt *ShiftC;
1724 m_AShr(m_Deferred(X), m_APInt(ShiftC))))))
1725 return nullptr;
1726 uint64_t Shift = ShiftC->getLimitedValue();
1727 Type *XType = X->getType();
1728 if (Shift == 0 || PowerOf2.isMinSignedValue())
1729 return nullptr;
1730 Value *Add = Builder.CreateAdd(X, ConstantInt::get(XType, PowerOf2));
1731 APInt Bound =
1732 Pred == ICmpInst::ICMP_ULT ? PowerOf2 << 1 : ((PowerOf2 << 1) - 1);
1733 return new ICmpInst(Pred, Add, ConstantInt::get(XType, Bound));
1734}
1735
1736/// Fold icmp (and (sh X, Y), C2), C1.
1739 const APInt &C1,
1740 const APInt &C2) {
1741 BinaryOperator *Shift = dyn_cast<BinaryOperator>(And->getOperand(0));
1742 if (!Shift || !Shift->isShift())
1743 return nullptr;
1744
1745 // If this is: (X >> C3) & C2 != C1 (where any shift and any compare could
1746 // exist), turn it into (X & (C2 << C3)) != (C1 << C3). This happens a LOT in
1747 // code produced by the clang front-end, for bitfield access.
1748 // This seemingly simple opportunity to fold away a shift turns out to be
1749 // rather complicated. See PR17827 for details.
1750 unsigned ShiftOpcode = Shift->getOpcode();
1751 bool IsShl = ShiftOpcode == Instruction::Shl;
1752 const APInt *C3;
1753 if (match(Shift->getOperand(1), m_APInt(C3))) {
1754 APInt NewAndCst, NewCmpCst;
1755 bool AnyCmpCstBitsShiftedOut;
1756 if (ShiftOpcode == Instruction::Shl) {
1757 // For a left shift, we can fold if the comparison is not signed. We can
1758 // also fold a signed comparison if the mask value and comparison value
1759 // are not negative. These constraints may not be obvious, but we can
1760 // prove that they are correct using an SMT solver.
1761 if (Cmp.isSigned() && (C2.isNegative() || C1.isNegative()))
1762 return nullptr;
1763
1764 NewCmpCst = C1.lshr(*C3);
1765 NewAndCst = C2.lshr(*C3);
1766 AnyCmpCstBitsShiftedOut = NewCmpCst.shl(*C3) != C1;
1767 } else if (ShiftOpcode == Instruction::LShr) {
1768 // For a logical right shift, we can fold if the comparison is not signed.
1769 // We can also fold a signed comparison if the shifted mask value and the
1770 // shifted comparison value are not negative. These constraints may not be
1771 // obvious, but we can prove that they are correct using an SMT solver.
1772 NewCmpCst = C1.shl(*C3);
1773 NewAndCst = C2.shl(*C3);
1774 AnyCmpCstBitsShiftedOut = NewCmpCst.lshr(*C3) != C1;
1775 if (Cmp.isSigned() && (NewAndCst.isNegative() || NewCmpCst.isNegative()))
1776 return nullptr;
1777 } else {
1778 // For an arithmetic shift, check that both constants don't use (in a
1779 // signed sense) the top bits being shifted out.
1780 assert(ShiftOpcode == Instruction::AShr && "Unknown shift opcode");
1781 NewCmpCst = C1.shl(*C3);
1782 NewAndCst = C2.shl(*C3);
1783 AnyCmpCstBitsShiftedOut = NewCmpCst.ashr(*C3) != C1;
1784 if (NewAndCst.ashr(*C3) != C2)
1785 return nullptr;
1786 }
1787
1788 if (AnyCmpCstBitsShiftedOut) {
1789 // If we shifted bits out, the fold is not going to work out. As a
1790 // special case, check to see if this means that the result is always
1791 // true or false now.
1792 if (Cmp.getPredicate() == ICmpInst::ICMP_EQ)
1793 return replaceInstUsesWith(Cmp, ConstantInt::getFalse(Cmp.getType()));
1794 if (Cmp.getPredicate() == ICmpInst::ICMP_NE)
1795 return replaceInstUsesWith(Cmp, ConstantInt::getTrue(Cmp.getType()));
1796 } else {
1797 Value *NewAnd = Builder.CreateAnd(
1798 Shift->getOperand(0), ConstantInt::get(And->getType(), NewAndCst));
1799 return new ICmpInst(Cmp.getPredicate(), NewAnd,
1800 ConstantInt::get(And->getType(), NewCmpCst));
1801 }
1802 }
1803
1804 // Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is
1805 // preferable because it allows the C2 << Y expression to be hoisted out of a
1806 // loop if Y is invariant and X is not.
1807 if (Shift->hasOneUse() && C1.isZero() && Cmp.isEquality() &&
1808 !Shift->isArithmeticShift() &&
1809 ((!IsShl && C2.isOne()) || !isa<Constant>(Shift->getOperand(0)))) {
1810 // Compute C2 << Y.
1811 Value *NewShift =
1812 IsShl ? Builder.CreateLShr(And->getOperand(1), Shift->getOperand(1))
1813 : Builder.CreateShl(And->getOperand(1), Shift->getOperand(1));
1814
1815 // Compute X & (C2 << Y).
1816 Value *NewAnd = Builder.CreateAnd(Shift->getOperand(0), NewShift);
1817 return new ICmpInst(Cmp.getPredicate(), NewAnd, Cmp.getOperand(1));
1818 }
1819
1820 return nullptr;
1821}
1822
1823/// Fold icmp (and X, C2), C1.
1826 const APInt &C1) {
1827 bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE;
1828
1829 // icmp ne (and X, 1), 0 --> trunc X to i1
1830 if (isICMP_NE && C1.isZero() && match(And->getOperand(1), m_One()))
1831 return new TruncInst(And->getOperand(0), Cmp.getType());
1832
1833 const APInt *C2;
1834 Value *X;
1835 if (!match(And, m_And(m_Value(X), m_APInt(C2))))
1836 return nullptr;
1837
1838 // (and X, highmask) s> [0, ~highmask] --> X s> ~highmask
1839 if (Cmp.getPredicate() == ICmpInst::ICMP_SGT && C1.ule(~*C2) &&
1840 C2->isNegatedPowerOf2())
1841 return new ICmpInst(ICmpInst::ICMP_SGT, X,
1842 ConstantInt::get(X->getType(), ~*C2));
1843 // (and X, highmask) s< [1, -highmask] --> X s< -highmask
1844 if (Cmp.getPredicate() == ICmpInst::ICMP_SLT && !C1.isSignMask() &&
1845 (C1 - 1).ule(~*C2) && C2->isNegatedPowerOf2() && !C2->isSignMask())
1846 return new ICmpInst(ICmpInst::ICMP_SLT, X,
1847 ConstantInt::get(X->getType(), -*C2));
1848
1849 // Don't perform the following transforms if the AND has multiple uses
1850 if (!And->hasOneUse())
1851 return nullptr;
1852
1853 if (Cmp.isEquality() && C1.isZero()) {
1854 // Restrict this fold to single-use 'and' (PR10267).
1855 // Replace (and X, (1 << size(X)-1) != 0) with X s< 0
1856 if (C2->isSignMask()) {
1857 Constant *Zero = Constant::getNullValue(X->getType());
1858 auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE;
1859 return new ICmpInst(NewPred, X, Zero);
1860 }
1861
1862 APInt NewC2 = *C2;
1863 KnownBits Know = computeKnownBits(And->getOperand(0), And);
1864 // Set high zeros of C2 to allow matching negated power-of-2.
1865 NewC2 = *C2 | APInt::getHighBitsSet(C2->getBitWidth(),
1866 Know.countMinLeadingZeros());
1867
1868 // Restrict this fold only for single-use 'and' (PR10267).
1869 // ((%x & C) == 0) --> %x u< (-C) iff (-C) is power of two.
1870 if (NewC2.isNegatedPowerOf2()) {
1871 Constant *NegBOC = ConstantInt::get(And->getType(), -NewC2);
1872 auto NewPred = isICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT;
1873 return new ICmpInst(NewPred, X, NegBOC);
1874 }
1875 }
1876
1877 // If the LHS is an 'and' of a truncate and we can widen the and/compare to
1878 // the input width without changing the value produced, eliminate the cast:
1879 //
1880 // icmp (and (trunc W), C2), C1 -> icmp (and W, C2'), C1'
1881 //
1882 // We can do this transformation if the constants do not have their sign bits
1883 // set or if it is an equality comparison. Extending a relational comparison
1884 // when we're checking the sign bit would not work.
1885 Value *W;
1886 if (match(And->getOperand(0), m_OneUse(m_Trunc(m_Value(W)))) &&
1887 (Cmp.isEquality() || (!C1.isNegative() && !C2->isNegative()))) {
1888 // TODO: Is this a good transform for vectors? Wider types may reduce
1889 // throughput. Should this transform be limited (even for scalars) by using
1890 // shouldChangeType()?
1891 if (!Cmp.getType()->isVectorTy()) {
1892 Type *WideType = W->getType();
1893 unsigned WideScalarBits = WideType->getScalarSizeInBits();
1894 Constant *ZextC1 = ConstantInt::get(WideType, C1.zext(WideScalarBits));
1895 Constant *ZextC2 = ConstantInt::get(WideType, C2->zext(WideScalarBits));
1896 Value *NewAnd = Builder.CreateAnd(W, ZextC2, And->getName());
1897 return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1);
1898 }
1899 }
1900
1901 if (Instruction *I = foldICmpAndShift(Cmp, And, C1, *C2))
1902 return I;
1903
1904 // (icmp pred (and (or (lshr A, B), A), 1), 0) -->
1905 // (icmp pred (and A, (or (shl 1, B), 1), 0))
1906 //
1907 // iff pred isn't signed
1908 if (!Cmp.isSigned() && C1.isZero() && And->getOperand(0)->hasOneUse() &&
1909 match(And->getOperand(1), m_One())) {
1910 Constant *One = cast<Constant>(And->getOperand(1));
1911 Value *Or = And->getOperand(0);
1912 Value *A, *B, *LShr;
1913 if (match(Or, m_Or(m_Value(LShr), m_Value(A))) &&
1914 match(LShr, m_LShr(m_Specific(A), m_Value(B)))) {
1915 unsigned UsesRemoved = 0;
1916 if (And->hasOneUse())
1917 ++UsesRemoved;
1918 if (Or->hasOneUse())
1919 ++UsesRemoved;
1920 if (LShr->hasOneUse())
1921 ++UsesRemoved;
1922
1923 // Compute A & ((1 << B) | 1)
1924 unsigned RequireUsesRemoved = match(B, m_ImmConstant()) ? 1 : 3;
1925 if (UsesRemoved >= RequireUsesRemoved) {
1926 Value *NewOr =
1927 Builder.CreateOr(Builder.CreateShl(One, B, LShr->getName(),
1928 /*HasNUW=*/true),
1929 One, Or->getName());
1930 Value *NewAnd = Builder.CreateAnd(A, NewOr, And->getName());
1931 return new ICmpInst(Cmp.getPredicate(), NewAnd, Cmp.getOperand(1));
1932 }
1933 }
1934 }
1935
1936 // (icmp eq (and (bitcast X to int), ExponentMask), ExponentMask) -->
1937 // llvm.is.fpclass(X, fcInf|fcNan)
1938 // (icmp ne (and (bitcast X to int), ExponentMask), ExponentMask) -->
1939 // llvm.is.fpclass(X, ~(fcInf|fcNan))
1940 // (icmp eq (and (bitcast X to int), ExponentMask), 0) -->
1941 // llvm.is.fpclass(X, fcSubnormal|fcZero)
1942 // (icmp ne (and (bitcast X to int), ExponentMask), 0) -->
1943 // llvm.is.fpclass(X, ~(fcSubnormal|fcZero))
1944 Value *V;
1945 if (!Cmp.getParent()->getParent()->hasFnAttribute(
1946 Attribute::NoImplicitFloat) &&
1947 Cmp.isEquality() &&
1949 Type *FPType = V->getType()->getScalarType();
1950 if (FPType->isIEEELikeFPTy() && (C1.isZero() || C1 == *C2)) {
1951 APInt ExponentMask =
1952 APFloat::getInf(FPType->getFltSemantics()).bitcastToAPInt();
1953 if (*C2 == ExponentMask) {
1954 unsigned Mask = C1.isZero()
1957 if (isICMP_NE)
1958 Mask = ~Mask & fcAllFlags;
1959 return replaceInstUsesWith(Cmp, Builder.createIsFPClass(V, Mask));
1960 }
1961 }
1962 }
1963
1964 return nullptr;
1965}
1966
1967/// Fold icmp (and X, Y), C.
1970 const APInt &C) {
1971 if (Instruction *I = foldICmpAndConstConst(Cmp, And, C))
1972 return I;
1973
1974 const ICmpInst::Predicate Pred = Cmp.getPredicate();
1975 bool TrueIfNeg;
1976 if (isSignBitCheck(Pred, C, TrueIfNeg)) {
1977 // ((X - 1) & ~X) < 0 --> X == 0
1978 // ((X - 1) & ~X) >= 0 --> X != 0
1979 Value *X;
1980 if (match(And->getOperand(0), m_Add(m_Value(X), m_AllOnes())) &&
1981 match(And->getOperand(1), m_Not(m_Specific(X)))) {
1982 auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
1983 return new ICmpInst(NewPred, X, ConstantInt::getNullValue(X->getType()));
1984 }
1985 // (X & -X) < 0 --> X == MinSignedC
1986 // (X & -X) > -1 --> X != MinSignedC
1987 if (match(And, m_c_And(m_Neg(m_Value(X)), m_Deferred(X)))) {
1988 Constant *MinSignedC = ConstantInt::get(
1989 X->getType(),
1990 APInt::getSignedMinValue(X->getType()->getScalarSizeInBits()));
1991 auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
1992 return new ICmpInst(NewPred, X, MinSignedC);
1993 }
1994 }
1995
1996 // TODO: These all require that Y is constant too, so refactor with the above.
1997
1998 // Try to optimize things like "A[i] & 42 == 0" to index computations.
1999 Value *X = And->getOperand(0);
2000 Value *Y = And->getOperand(1);
2001 if (auto *C2 = dyn_cast<ConstantInt>(Y))
2002 if (auto *LI = dyn_cast<LoadInst>(X))
2004 LI, Cmp.getPredicate(), cast<Constant>(Cmp.getOperand(1)), Cmp,
2005 C2))
2006 return Res;
2007
2008 if (!Cmp.isEquality())
2009 return nullptr;
2010
2011 // (X & -X) == 0 --> X == 0
2012 // (X & -X) != 0 --> X != 0
2013 // (X & -X) == 1 --> trunc X to i1
2014 // (X & -X) != 1 --> !(trunc X to i1)
2015 // Cmp is == or != by the check above.
2016 Value *MatchedX;
2017 // Match X & -X in either operand order.
2018 if (C.getBitWidth() > 1 && (C.isZero() || C.isOne()) &&
2019 match(And, m_c_And(m_Neg(m_Value(MatchedX)), m_Deferred(MatchedX)))) {
2020 // Preserve the predicate: (X & -X) ==/!= 0 --> X ==/!= 0.
2021 if (C.isZero())
2022 return new ICmpInst(Pred, MatchedX, Cmp.getOperand(1));
2023
2024 // (X & -X) == 1 iff the low bit of X is set.
2025 if (Pred == CmpInst::ICMP_EQ)
2026 return new TruncInst(MatchedX, Cmp.getType());
2027
2028 // The remaining case needs a trunc and not. Require the original and
2029 // to become dead to avoid increasing the instruction count.
2030 if (And->hasOneUse()) {
2031 Value *Trunc = Builder.CreateTrunc(MatchedX, Cmp.getType());
2032 return BinaryOperator::CreateNot(Trunc);
2033 }
2034 }
2035
2036 // X & -C == -C -> X > u ~C
2037 // X & -C != -C -> X <= u ~C
2038 // iff C is a power of 2
2039 if (Cmp.getOperand(1) == Y && C.isNegatedPowerOf2()) {
2040 auto NewPred =
2042 return new ICmpInst(NewPred, X, SubOne(cast<Constant>(Cmp.getOperand(1))));
2043 }
2044
2045 // ((zext i1 X) & Y) == 0 --> !((trunc Y) & X)
2046 // ((zext i1 X) & Y) != 0 --> ((trunc Y) & X)
2047 // ((zext i1 X) & Y) == 1 --> ((trunc Y) & X)
2048 // ((zext i1 X) & Y) != 1 --> !((trunc Y) & X)
2050 X->getType()->isIntOrIntVectorTy(1) && (C.isZero() || C.isOne())) {
2051 Value *TruncY = Builder.CreateTrunc(Y, X->getType());
2052 if (C.isZero() ^ (Pred == CmpInst::ICMP_NE)) {
2053 Value *And = Builder.CreateAnd(TruncY, X);
2055 }
2056 return BinaryOperator::CreateAnd(TruncY, X);
2057 }
2058
2059 // (icmp eq/ne (and (shl -1, X), Y), 0)
2060 // -> (icmp eq/ne (lshr Y, X), 0)
2061 // We could technically handle any C == 0 or (C < 0 && isOdd(C)) but it seems
2062 // highly unlikely the non-zero case will ever show up in code.
2063 if (C.isZero() &&
2065 m_Value(Y))))) {
2066 Value *LShr = Builder.CreateLShr(Y, X);
2067 return new ICmpInst(Pred, LShr, Constant::getNullValue(LShr->getType()));
2068 }
2069
2070 // (icmp eq/ne (and (add A, Addend), Msk), C)
2071 // -> (icmp eq/ne (and A, Msk), (and (sub C, Addend), Msk))
2072 {
2073 Value *A;
2074 const APInt *Addend, *Msk;
2076 m_LowBitMask(Msk)))) &&
2077 C.ule(*Msk)) {
2078 APInt NewComperand = (C - *Addend) & *Msk;
2079 Value *MaskA = Builder.CreateAnd(A, ConstantInt::get(A->getType(), *Msk));
2080 return new ICmpInst(Pred, MaskA,
2081 ConstantInt::get(MaskA->getType(), NewComperand));
2082 }
2083 }
2084
2085 return nullptr;
2086}
2087
2088/// Fold icmp eq/ne (or (xor/sub (X1, X2), xor/sub (X3, X4))), 0.
2090 InstCombiner::BuilderTy &Builder) {
2091 // Are we using xors or subs to bitwise check for a pair or pairs of
2092 // (in)equalities? Convert to a shorter form that has more potential to be
2093 // folded even further.
2094 // ((X1 ^/- X2) || (X3 ^/- X4)) == 0 --> (X1 == X2) && (X3 == X4)
2095 // ((X1 ^/- X2) || (X3 ^/- X4)) != 0 --> (X1 != X2) || (X3 != X4)
2096 // ((X1 ^/- X2) || (X3 ^/- X4) || (X5 ^/- X6)) == 0 -->
2097 // (X1 == X2) && (X3 == X4) && (X5 == X6)
2098 // ((X1 ^/- X2) || (X3 ^/- X4) || (X5 ^/- X6)) != 0 -->
2099 // (X1 != X2) || (X3 != X4) || (X5 != X6)
2101 SmallVector<Value *, 16> WorkList(1, Or);
2102
2103 while (!WorkList.empty()) {
2104 auto MatchOrOperatorArgument = [&](Value *OrOperatorArgument) {
2105 Value *Lhs, *Rhs;
2106
2107 if (match(OrOperatorArgument,
2108 m_OneUse(m_Xor(m_Value(Lhs), m_Value(Rhs))))) {
2109 CmpValues.emplace_back(Lhs, Rhs);
2110 return;
2111 }
2112
2113 if (match(OrOperatorArgument,
2114 m_OneUse(m_Sub(m_Value(Lhs), m_Value(Rhs))))) {
2115 CmpValues.emplace_back(Lhs, Rhs);
2116 return;
2117 }
2118
2119 WorkList.push_back(OrOperatorArgument);
2120 };
2121
2122 Value *CurrentValue = WorkList.pop_back_val();
2123 Value *OrOperatorLhs, *OrOperatorRhs;
2124
2125 if (!match(CurrentValue,
2126 m_Or(m_Value(OrOperatorLhs), m_Value(OrOperatorRhs)))) {
2127 return nullptr;
2128 }
2129
2130 MatchOrOperatorArgument(OrOperatorRhs);
2131 MatchOrOperatorArgument(OrOperatorLhs);
2132 }
2133
2134 ICmpInst::Predicate Pred = Cmp.getPredicate();
2135 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2136 Value *LhsCmp = Builder.CreateICmp(Pred, CmpValues.rbegin()->first,
2137 CmpValues.rbegin()->second);
2138
2139 for (auto It = CmpValues.rbegin() + 1; It != CmpValues.rend(); ++It) {
2140 Value *RhsCmp = Builder.CreateICmp(Pred, It->first, It->second);
2141 LhsCmp = Builder.CreateBinOp(BOpc, LhsCmp, RhsCmp);
2142 }
2143
2144 return LhsCmp;
2145}
2146
2147/// Fold icmp (or X, Y), C.
2150 const APInt &C) {
2151 ICmpInst::Predicate Pred = Cmp.getPredicate();
2152 if (C.isOne()) {
2153 // icmp slt signum(V) 1 --> icmp slt V, 1
2154 Value *V = nullptr;
2155 if (Pred == ICmpInst::ICMP_SLT && match(Or, m_Signum(m_Value(V))))
2156 return new ICmpInst(ICmpInst::ICMP_SLT, V,
2157 ConstantInt::get(V->getType(), 1));
2158 }
2159
2160 Value *OrOp0 = Or->getOperand(0), *OrOp1 = Or->getOperand(1);
2161
2162 // (icmp eq/ne (or disjoint x, C0), C1)
2163 // -> (icmp eq/ne x, C0^C1)
2164 if (Cmp.isEquality() && match(OrOp1, m_ImmConstant()) &&
2165 cast<PossiblyDisjointInst>(Or)->isDisjoint()) {
2166 Value *NewC =
2167 Builder.CreateXor(OrOp1, ConstantInt::get(OrOp1->getType(), C));
2168 return new ICmpInst(Pred, OrOp0, NewC);
2169 }
2170
2171 const APInt *MaskC;
2172 if (match(OrOp1, m_APInt(MaskC)) && Cmp.isEquality()) {
2173 if (*MaskC == C && (C + 1).isPowerOf2()) {
2174 // X | C == C --> X <=u C
2175 // X | C != C --> X >u C
2176 // iff C+1 is a power of 2 (C is a bitmask of the low bits)
2178 return new ICmpInst(Pred, OrOp0, OrOp1);
2179 }
2180
2181 // More general: canonicalize 'equality with set bits mask' to
2182 // 'equality with clear bits mask'.
2183 // (X | MaskC) == C --> (X & ~MaskC) == C ^ MaskC
2184 // (X | MaskC) != C --> (X & ~MaskC) != C ^ MaskC
2185 if (Or->hasOneUse()) {
2186 Value *And = Builder.CreateAnd(OrOp0, ~(*MaskC));
2187 Constant *NewC = ConstantInt::get(Or->getType(), C ^ (*MaskC));
2188 return new ICmpInst(Pred, And, NewC);
2189 }
2190 }
2191
2192 // (X | (X-1)) s< 0 --> X s< 1
2193 // (X | (X-1)) s> -1 --> X s> 0
2194 Value *X;
2195 bool TrueIfSigned;
2196 if (isSignBitCheck(Pred, C, TrueIfSigned) &&
2198 auto NewPred = TrueIfSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGT;
2199 Constant *NewC = ConstantInt::get(X->getType(), TrueIfSigned ? 1 : 0);
2200 return new ICmpInst(NewPred, X, NewC);
2201 }
2202
2203 const APInt *OrC;
2204 // icmp(X | OrC, C) --> icmp(X, 0)
2205 if (C.isNonNegative() && match(Or, m_Or(m_Value(X), m_APInt(OrC)))) {
2206 switch (Pred) {
2207 // X | OrC s< C --> X s< 0 iff OrC s>= C s>= 0
2208 case ICmpInst::ICMP_SLT:
2209 // X | OrC s>= C --> X s>= 0 iff OrC s>= C s>= 0
2210 case ICmpInst::ICMP_SGE:
2211 if (OrC->sge(C))
2212 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
2213 break;
2214 // X | OrC s<= C --> X s< 0 iff OrC s> C s>= 0
2215 case ICmpInst::ICMP_SLE:
2216 // X | OrC s> C --> X s>= 0 iff OrC s> C s>= 0
2217 case ICmpInst::ICMP_SGT:
2218 if (OrC->sgt(C))
2220 ConstantInt::getNullValue(X->getType()));
2221 break;
2222 default:
2223 break;
2224 }
2225 }
2226
2227 if (!Cmp.isEquality() || !C.isZero() || !Or->hasOneUse())
2228 return nullptr;
2229
2230 Value *P, *Q;
2232 // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0
2233 // -> and (icmp eq P, null), (icmp eq Q, null).
2234 Value *CmpP =
2235 Builder.CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType()));
2236 Value *CmpQ =
2237 Builder.CreateICmp(Pred, Q, ConstantInt::getNullValue(Q->getType()));
2238 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2239 return BinaryOperator::Create(BOpc, CmpP, CmpQ);
2240 }
2241
2242 if (Value *V = foldICmpOrXorSubChain(Cmp, Or, Builder))
2243 return replaceInstUsesWith(Cmp, V);
2244
2245 return nullptr;
2246}
2247
2248/// Fold icmp (mul X, Y), C.
2251 const APInt &C) {
2252 ICmpInst::Predicate Pred = Cmp.getPredicate();
2253 Type *MulTy = Mul->getType();
2254 Value *X = Mul->getOperand(0);
2255
2256 // If comparing a square with a constant, try simplifying to comparing square
2257 // roots.
2258 if (X == Mul->getOperand(1) && !Cmp.isSigned()) {
2259 APInt R = C.sqrtFloor();
2260 bool IsSqr = C == R * R;
2261
2262 // X * X eq/ne C
2263 if (Cmp.isEquality() &&
2264 (Mul->hasNoUnsignedWrap() || (Mul->hasNoSignedWrap() && C.isZero()))) {
2265
2266 // If constant is not a square, eq/ne is false/true respectively
2267 if (!IsSqr)
2268 return replaceInstUsesWith(
2269 Cmp,
2270 ConstantInt::getBool(Cmp.getType(), Pred == ICmpInst::ICMP_NE));
2271
2272 return new ICmpInst(Pred, X, ConstantInt::get(MulTy, R));
2273 }
2274
2275 // If the multiply does not wrap
2276 // X * X pred C --> X pred R
2277 if (Mul->hasNoUnsignedWrap()) {
2278
2279 if (IsSqr)
2280 return new ICmpInst(Pred, X, ConstantInt::get(MulTy, R));
2281
2282 // If C is not a square, we use floor/ceil of sqrt(C).
2283 //
2284 // If LT or LE, we need R to be an overestimate of sqrt(C),
2285 // then use the strict predicate (LT->LT, LE->LT).
2286 //
2287 // If GT or GE, we need R to be an underestimate of sqrt(C),
2288 // then use the strict predicate (GT->GT, GE->GT).
2289 //
2290 // R is already an underestimate of sqrt(C) due to sqrtFloor.
2291 if (ICmpInst::isLT(Pred) || ICmpInst::isLE(Pred))
2292 ++R;
2293
2294 return new ICmpInst(Cmp.getStrictPredicate(), X,
2295 ConstantInt::get(MulTy, R));
2296 }
2297 }
2298
2299 const APInt *MulC;
2300 if (!match(Mul->getOperand(1), m_APInt(MulC)))
2301 return nullptr;
2302
2303 // If this is a test of the sign bit and the multiply is sign-preserving with
2304 // a constant operand, use the multiply LHS operand instead:
2305 // (X * +MulC) < 0 --> X < 0
2306 // (X * -MulC) < 0 --> X > 0
2307 if (isSignTest(Pred, C) && Mul->hasNoSignedWrap()) {
2308 if (MulC->isNegative())
2309 Pred = ICmpInst::getSwappedPredicate(Pred);
2310 return new ICmpInst(Pred, X, ConstantInt::getNullValue(MulTy));
2311 }
2312
2313 if (MulC->isZero())
2314 return nullptr;
2315
2316 // If the multiply does not wrap or the constant is odd, try to divide the
2317 // compare constant by the multiplication factor.
2318 if (Cmp.isEquality()) {
2319 // (mul nsw X, MulC) eq/ne C --> X eq/ne C /s MulC
2320 if (Mul->hasNoSignedWrap() && C.srem(*MulC).isZero()) {
2321 Constant *NewC = ConstantInt::get(MulTy, C.sdiv(*MulC));
2322 return new ICmpInst(Pred, X, NewC);
2323 }
2324
2325 // C % MulC == 0 is weaker than we could use if MulC is odd because it
2326 // correct to transform if MulC * N == C including overflow. I.e with i8
2327 // (icmp eq (mul X, 5), 101) -> (icmp eq X, 225) but since 101 % 5 != 0, we
2328 // miss that case.
2329 if (C.urem(*MulC).isZero()) {
2330 // (mul nuw X, MulC) eq/ne C --> X eq/ne C /u MulC
2331 // (mul X, OddC) eq/ne N * C --> X eq/ne N
2332 if ((*MulC & 1).isOne() || Mul->hasNoUnsignedWrap()) {
2333 Constant *NewC = ConstantInt::get(MulTy, C.udiv(*MulC));
2334 return new ICmpInst(Pred, X, NewC);
2335 }
2336 }
2337 }
2338
2339 // With a matching no-overflow guarantee, fold the constants:
2340 // (X * MulC) < C --> X < (C / MulC)
2341 // (X * MulC) > C --> X > (C / MulC)
2342 // TODO: Assert that Pred is not equal to SGE, SLE, UGE, ULE?
2343 Constant *NewC = nullptr;
2344 if (Mul->hasNoSignedWrap() && ICmpInst::isSigned(Pred)) {
2345 // MININT / -1 --> overflow.
2346 if (C.isMinSignedValue() && MulC->isAllOnes())
2347 return nullptr;
2348 if (MulC->isNegative())
2349 Pred = ICmpInst::getSwappedPredicate(Pred);
2350
2351 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) {
2352 NewC = ConstantInt::get(
2354 } else {
2355 assert((Pred == ICmpInst::ICMP_SLE || Pred == ICmpInst::ICMP_SGT) &&
2356 "Unexpected predicate");
2357 NewC = ConstantInt::get(
2359 }
2360 } else if (Mul->hasNoUnsignedWrap() && ICmpInst::isUnsigned(Pred)) {
2361 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) {
2362 NewC = ConstantInt::get(
2364 } else {
2365 assert((Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
2366 "Unexpected predicate");
2367 NewC = ConstantInt::get(
2369 }
2370 }
2371
2372 return NewC ? new ICmpInst(Pred, X, NewC) : nullptr;
2373}
2374
2375/// Fold icmp (shl nuw C2, Y), C.
2377 const APInt &C) {
2378 Value *Y;
2379 const APInt *C2;
2380 if (!match(Shl, m_NUWShl(m_APInt(C2), m_Value(Y))))
2381 return nullptr;
2382
2383 Type *ShiftType = Shl->getType();
2384 unsigned TypeBits = C.getBitWidth();
2385 ICmpInst::Predicate Pred = Cmp.getPredicate();
2386 if (Cmp.isUnsigned()) {
2387 if (C2->isZero() || C2->ugt(C))
2388 return nullptr;
2389 APInt Div, Rem;
2390 APInt::udivrem(C, *C2, Div, Rem);
2391 bool CIsPowerOf2 = Rem.isZero() && Div.isPowerOf2();
2392
2393 // (1 << Y) pred C -> Y pred Log2(C)
2394 if (!CIsPowerOf2) {
2395 // (1 << Y) < 30 -> Y <= 4
2396 // (1 << Y) <= 30 -> Y <= 4
2397 // (1 << Y) >= 30 -> Y > 4
2398 // (1 << Y) > 30 -> Y > 4
2399 if (Pred == ICmpInst::ICMP_ULT)
2401 else if (Pred == ICmpInst::ICMP_UGE)
2403 }
2404
2405 unsigned CLog2 = Div.logBase2();
2406 return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, CLog2));
2407 } else if (Cmp.isSigned() && C2->isOne()) {
2408 Constant *BitWidthMinusOne = ConstantInt::get(ShiftType, TypeBits - 1);
2409 // (1 << Y) > 0 -> Y != 31
2410 // (1 << Y) > C -> Y != 31 if C is negative.
2411 if (Pred == ICmpInst::ICMP_SGT && C.sle(0))
2412 return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne);
2413
2414 // (1 << Y) < 0 -> Y == 31
2415 // (1 << Y) < 1 -> Y == 31
2416 // (1 << Y) < C -> Y == 31 if C is negative and not signed min.
2417 // Exclude signed min by subtracting 1 and lower the upper bound to 0.
2418 if (Pred == ICmpInst::ICMP_SLT && (C - 1).sle(0))
2419 return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne);
2420 }
2421
2422 return nullptr;
2423}
2424
2425/// Fold icmp (shl X, Y), C.
2427 BinaryOperator *Shl,
2428 const APInt &C) {
2429 const APInt *ShiftVal;
2430 if (Cmp.isEquality() && match(Shl->getOperand(0), m_APInt(ShiftVal)))
2431 return foldICmpShlConstConst(Cmp, Shl->getOperand(1), C, *ShiftVal);
2432
2433 ICmpInst::Predicate Pred = Cmp.getPredicate();
2434 // (icmp pred (shl nuw&nsw X, Y), Csle0)
2435 // -> (icmp pred X, Csle0)
2436 //
2437 // The idea is the nuw/nsw essentially freeze the sign bit for the shift op
2438 // so X's must be what is used.
2439 if (C.sle(0) && Shl->hasNoUnsignedWrap() && Shl->hasNoSignedWrap())
2440 return new ICmpInst(Pred, Shl->getOperand(0), Cmp.getOperand(1));
2441
2442 // (icmp eq/ne (shl nuw|nsw X, Y), 0)
2443 // -> (icmp eq/ne X, 0)
2444 if (ICmpInst::isEquality(Pred) && C.isZero() &&
2445 (Shl->hasNoUnsignedWrap() || Shl->hasNoSignedWrap()))
2446 return new ICmpInst(Pred, Shl->getOperand(0), Cmp.getOperand(1));
2447
2448 // (icmp slt (shl nsw X, Y), 0/1)
2449 // -> (icmp slt X, 0/1)
2450 // (icmp sgt (shl nsw X, Y), 0/-1)
2451 // -> (icmp sgt X, 0/-1)
2452 //
2453 // NB: sge/sle with a constant will canonicalize to sgt/slt.
2454 if (Shl->hasNoSignedWrap() &&
2455 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT))
2456 if (C.isZero() || (Pred == ICmpInst::ICMP_SGT ? C.isAllOnes() : C.isOne()))
2457 return new ICmpInst(Pred, Shl->getOperand(0), Cmp.getOperand(1));
2458
2459 const APInt *ShiftAmt;
2460 if (!match(Shl->getOperand(1), m_APInt(ShiftAmt)))
2461 return foldICmpShlLHSC(Cmp, Shl, C);
2462
2463 // Check that the shift amount is in range. If not, don't perform undefined
2464 // shifts. When the shift is visited, it will be simplified.
2465 unsigned TypeBits = C.getBitWidth();
2466 if (ShiftAmt->uge(TypeBits))
2467 return nullptr;
2468
2469 Value *X = Shl->getOperand(0);
2470 Type *ShType = Shl->getType();
2471
2472 // NSW guarantees that we are only shifting out sign bits from the high bits,
2473 // so we can ASHR the compare constant without needing a mask and eliminate
2474 // the shift.
2475 if (Shl->hasNoSignedWrap()) {
2476 if (Pred == ICmpInst::ICMP_SGT) {
2477 // icmp Pred (shl nsw X, ShiftAmt), C --> icmp Pred X, (C >>s ShiftAmt)
2478 APInt ShiftedC = C.ashr(*ShiftAmt);
2479 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2480 }
2481 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2482 C.ashr(*ShiftAmt).shl(*ShiftAmt) == C) {
2483 APInt ShiftedC = C.ashr(*ShiftAmt);
2484 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2485 }
2486 if (Pred == ICmpInst::ICMP_SLT) {
2487 // SLE is the same as above, but SLE is canonicalized to SLT, so convert:
2488 // (X << S) <=s C is equiv to X <=s (C >> S) for all C
2489 // (X << S) <s (C + 1) is equiv to X <s (C >> S) + 1 if C <s SMAX
2490 // (X << S) <s C is equiv to X <s ((C - 1) >> S) + 1 if C >s SMIN
2491 assert(!C.isMinSignedValue() && "Unexpected icmp slt");
2492 APInt ShiftedC = (C - 1).ashr(*ShiftAmt) + 1;
2493 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2494 }
2495 }
2496
2497 // NUW guarantees that we are only shifting out zero bits from the high bits,
2498 // so we can LSHR the compare constant without needing a mask and eliminate
2499 // the shift.
2500 if (Shl->hasNoUnsignedWrap()) {
2501 if (Pred == ICmpInst::ICMP_UGT) {
2502 // icmp Pred (shl nuw X, ShiftAmt), C --> icmp Pred X, (C >>u ShiftAmt)
2503 APInt ShiftedC = C.lshr(*ShiftAmt);
2504 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2505 }
2506 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2507 C.lshr(*ShiftAmt).shl(*ShiftAmt) == C) {
2508 APInt ShiftedC = C.lshr(*ShiftAmt);
2509 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2510 }
2511 if (Pred == ICmpInst::ICMP_ULT) {
2512 // ULE is the same as above, but ULE is canonicalized to ULT, so convert:
2513 // (X << S) <=u C is equiv to X <=u (C >> S) for all C
2514 // (X << S) <u (C + 1) is equiv to X <u (C >> S) + 1 if C <u ~0u
2515 // (X << S) <u C is equiv to X <u ((C - 1) >> S) + 1 if C >u 0
2516 assert(C.ugt(0) && "ult 0 should have been eliminated");
2517 APInt ShiftedC = (C - 1).lshr(*ShiftAmt) + 1;
2518 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2519 }
2520 }
2521
2522 if (Cmp.isEquality() && Shl->hasOneUse()) {
2523 // Strength-reduce the shift into an 'and'.
2524 Constant *Mask = ConstantInt::get(
2525 ShType,
2526 APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt->getZExtValue()));
2527 Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask");
2528 Constant *LShrC = ConstantInt::get(ShType, C.lshr(*ShiftAmt));
2529 return new ICmpInst(Pred, And, LShrC);
2530 }
2531
2532 // Otherwise, if this is a comparison of the sign bit, simplify to and/test.
2533 bool TrueIfSigned = false;
2534 if (Shl->hasOneUse() && isSignBitCheck(Pred, C, TrueIfSigned)) {
2535 // (X << 31) <s 0 --> (X & 1) != 0
2536 Constant *Mask = ConstantInt::get(
2537 ShType,
2538 APInt::getOneBitSet(TypeBits, TypeBits - ShiftAmt->getZExtValue() - 1));
2539 Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask");
2540 return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ,
2541 And, Constant::getNullValue(ShType));
2542 }
2543
2544 // Simplify 'shl' inequality test into 'and' equality test.
2545 if (Cmp.isUnsigned() && Shl->hasOneUse()) {
2546 // (X l<< C2) u<=/u> C1 iff C1+1 is power of two -> X & (~C1 l>> C2) ==/!= 0
2547 if ((C + 1).isPowerOf2() &&
2548 (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT)) {
2549 Value *And = Builder.CreateAnd(X, (~C).lshr(ShiftAmt->getZExtValue()));
2550 return new ICmpInst(Pred == ICmpInst::ICMP_ULE ? ICmpInst::ICMP_EQ
2552 And, Constant::getNullValue(ShType));
2553 }
2554 // (X l<< C2) u</u>= C1 iff C1 is power of two -> X & (-C1 l>> C2) ==/!= 0
2555 if (C.isPowerOf2() &&
2556 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) {
2557 Value *And =
2558 Builder.CreateAnd(X, (~(C - 1)).lshr(ShiftAmt->getZExtValue()));
2559 return new ICmpInst(Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_EQ
2561 And, Constant::getNullValue(ShType));
2562 }
2563 }
2564
2565 // Transform (icmp pred iM (shl iM %v, N), C)
2566 // -> (icmp pred i(M-N) (trunc %v iM to i(M-N)), (trunc (C>>N))
2567 // Transform the shl to a trunc if (trunc (C>>N)) has no loss and M-N.
2568 // This enables us to get rid of the shift in favor of a trunc that may be
2569 // free on the target. It has the additional benefit of comparing to a
2570 // smaller constant that may be more target-friendly.
2571 unsigned Amt = ShiftAmt->getLimitedValue(TypeBits - 1);
2572 if (Shl->hasOneUse() && Amt != 0 &&
2573 shouldChangeType(ShType->getScalarSizeInBits(), TypeBits - Amt)) {
2574 ICmpInst::Predicate CmpPred = Pred;
2575 APInt RHSC = C;
2576
2577 if (RHSC.countr_zero() < Amt && ICmpInst::isStrictPredicate(CmpPred)) {
2578 // Try the flipped strictness predicate.
2579 // e.g.:
2580 // icmp ult i64 (shl X, 32), 8589934593 ->
2581 // icmp ule i64 (shl X, 32), 8589934592 ->
2582 // icmp ule i32 (trunc X, i32), 2 ->
2583 // icmp ult i32 (trunc X, i32), 3
2584 if (auto FlippedStrictness = getFlippedStrictnessPredicateAndConstant(
2585 Pred, ConstantInt::get(ShType->getContext(), C))) {
2586 CmpPred = FlippedStrictness->first;
2587 RHSC = cast<ConstantInt>(FlippedStrictness->second)->getValue();
2588 }
2589 }
2590
2591 if (RHSC.countr_zero() >= Amt) {
2592 Type *TruncTy = ShType->getWithNewBitWidth(TypeBits - Amt);
2593 Constant *NewC =
2594 ConstantInt::get(TruncTy, RHSC.ashr(*ShiftAmt).trunc(TypeBits - Amt));
2595 return new ICmpInst(CmpPred,
2596 Builder.CreateTrunc(X, TruncTy, "", /*IsNUW=*/false,
2597 Shl->hasNoSignedWrap()),
2598 NewC);
2599 }
2600 }
2601
2602 return nullptr;
2603}
2604
2605/// Fold icmp ({al}shr X, Y), C.
2607 BinaryOperator *Shr,
2608 const APInt &C) {
2609 // An exact shr only shifts out zero bits, so:
2610 // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0
2611 Value *X = Shr->getOperand(0);
2612 CmpInst::Predicate Pred = Cmp.getPredicate();
2613 if (Cmp.isEquality() && Shr->isExact() && C.isZero())
2614 return new ICmpInst(Pred, X, Cmp.getOperand(1));
2615
2616 bool IsAShr = Shr->getOpcode() == Instruction::AShr;
2617 const APInt *ShiftValC;
2618 if (match(X, m_APInt(ShiftValC))) {
2619 if (Cmp.isEquality())
2620 return foldICmpShrConstConst(Cmp, Shr->getOperand(1), C, *ShiftValC);
2621
2622 // (ShiftValC >> Y) >s -1 --> Y != 0 with ShiftValC < 0
2623 // (ShiftValC >> Y) <s 0 --> Y == 0 with ShiftValC < 0
2624 bool TrueIfSigned;
2625 if (!IsAShr && ShiftValC->isNegative() &&
2626 isSignBitCheck(Pred, C, TrueIfSigned))
2627 return new ICmpInst(TrueIfSigned ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE,
2628 Shr->getOperand(1),
2629 ConstantInt::getNullValue(X->getType()));
2630
2631 // If the shifted constant is a power-of-2, test the shift amount directly:
2632 // (ShiftValC >> Y) >u C --> X <u (LZ(C) - LZ(ShiftValC))
2633 // (ShiftValC >> Y) <u C --> X >=u (LZ(C-1) - LZ(ShiftValC))
2634 if (!IsAShr && ShiftValC->isPowerOf2() &&
2635 (Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_ULT)) {
2636 bool IsUGT = Pred == CmpInst::ICMP_UGT;
2637 assert(ShiftValC->uge(C) && "Expected simplify of compare");
2638 assert((IsUGT || !C.isZero()) && "Expected X u< 0 to simplify");
2639
2640 unsigned CmpLZ = IsUGT ? C.countl_zero() : (C - 1).countl_zero();
2641 unsigned ShiftLZ = ShiftValC->countl_zero();
2642 Constant *NewC = ConstantInt::get(Shr->getType(), CmpLZ - ShiftLZ);
2643 auto NewPred = IsUGT ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE;
2644 return new ICmpInst(NewPred, Shr->getOperand(1), NewC);
2645 }
2646 }
2647
2648 const APInt *ShiftAmtC;
2649 if (!match(Shr->getOperand(1), m_APInt(ShiftAmtC)))
2650 return nullptr;
2651
2652 // Check that the shift amount is in range. If not, don't perform undefined
2653 // shifts. When the shift is visited it will be simplified.
2654 unsigned TypeBits = C.getBitWidth();
2655 unsigned ShAmtVal = ShiftAmtC->getLimitedValue(TypeBits);
2656 if (ShAmtVal >= TypeBits || ShAmtVal == 0)
2657 return nullptr;
2658
2659 bool IsExact = Shr->isExact();
2660 Type *ShrTy = Shr->getType();
2661 // TODO: If we could guarantee that InstSimplify would handle all of the
2662 // constant-value-based preconditions in the folds below, then we could assert
2663 // those conditions rather than checking them. This is difficult because of
2664 // undef/poison (PR34838).
2665 if (IsAShr && Shr->hasOneUse()) {
2666 if (IsExact && (Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) &&
2667 (C - 1).isPowerOf2() && C.countLeadingZeros() > ShAmtVal) {
2668 // When C - 1 is a power of two and the transform can be legally
2669 // performed, prefer this form so the produced constant is close to a
2670 // power of two.
2671 // icmp slt/ult (ashr exact X, ShAmtC), C
2672 // --> icmp slt/ult X, (C - 1) << ShAmtC) + 1
2673 APInt ShiftedC = (C - 1).shl(ShAmtVal) + 1;
2674 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2675 }
2676 if (IsExact || Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) {
2677 // When ShAmtC can be shifted losslessly:
2678 // icmp PRED (ashr exact X, ShAmtC), C --> icmp PRED X, (C << ShAmtC)
2679 // icmp slt/ult (ashr X, ShAmtC), C --> icmp slt/ult X, (C << ShAmtC)
2680 APInt ShiftedC = C.shl(ShAmtVal);
2681 if (ShiftedC.ashr(ShAmtVal) == C)
2682 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2683 }
2684 if (Pred == CmpInst::ICMP_SGT) {
2685 // icmp sgt (ashr X, ShAmtC), C --> icmp sgt X, ((C + 1) << ShAmtC) - 1
2686 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2687 if (!C.isMaxSignedValue() && !(C + 1).shl(ShAmtVal).isMinSignedValue() &&
2688 (ShiftedC + 1).ashr(ShAmtVal) == (C + 1))
2689 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2690 }
2691 if (Pred == CmpInst::ICMP_UGT) {
2692 // icmp ugt (ashr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1
2693 // 'C + 1 << ShAmtC' can overflow as a signed number, so the 2nd
2694 // clause accounts for that pattern.
2695 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2696 if ((ShiftedC + 1).ashr(ShAmtVal) == (C + 1) ||
2697 (C + 1).shl(ShAmtVal).isMinSignedValue())
2698 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2699 }
2700
2701 // If the compare constant has significant bits above the lowest sign-bit,
2702 // then convert an unsigned cmp to a test of the sign-bit:
2703 // (ashr X, ShiftC) u> C --> X s< 0
2704 // (ashr X, ShiftC) u< C --> X s> -1
2705 if (C.getBitWidth() > 2 && C.getNumSignBits() <= ShAmtVal) {
2706 if (Pred == CmpInst::ICMP_UGT) {
2707 return new ICmpInst(CmpInst::ICMP_SLT, X,
2709 }
2710 if (Pred == CmpInst::ICMP_ULT) {
2711 return new ICmpInst(CmpInst::ICMP_SGT, X,
2713 }
2714 }
2715 } else if (!IsAShr) {
2716 if (Pred == CmpInst::ICMP_ULT || (Pred == CmpInst::ICMP_UGT && IsExact)) {
2717 // icmp ult (lshr X, ShAmtC), C --> icmp ult X, (C << ShAmtC)
2718 // icmp ugt (lshr exact X, ShAmtC), C --> icmp ugt X, (C << ShAmtC)
2719 APInt ShiftedC = C.shl(ShAmtVal);
2720 if (ShiftedC.lshr(ShAmtVal) == C)
2721 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2722 }
2723 if (Pred == CmpInst::ICMP_UGT) {
2724 // icmp ugt (lshr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1
2725 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2726 if ((ShiftedC + 1).lshr(ShAmtVal) == (C + 1))
2727 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2728 }
2729 }
2730
2731 if (!Cmp.isEquality())
2732 return nullptr;
2733
2734 // Handle equality comparisons of shift-by-constant.
2735
2736 // If the comparison constant changes with the shift, the comparison cannot
2737 // succeed (bits of the comparison constant cannot match the shifted value).
2738 // This should be known by InstSimplify and already be folded to true/false.
2739 assert(((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) ||
2740 (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) &&
2741 "Expected icmp+shr simplify did not occur.");
2742
2743 // If the bits shifted out are known zero, compare the unshifted value:
2744 // (X & 4) >> 1 == 2 --> (X & 4) == 4.
2745 if (Shr->isExact())
2746 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, C << ShAmtVal));
2747
2748 if (Shr->hasOneUse()) {
2749 // Canonicalize the shift into an 'and':
2750 // icmp eq/ne (shr X, ShAmt), C --> icmp eq/ne (and X, HiMask), (C << ShAmt)
2751 APInt Val(APInt::getHighBitsSet(TypeBits, TypeBits - ShAmtVal));
2752 Constant *Mask = ConstantInt::get(ShrTy, Val);
2753 Value *And = Builder.CreateAnd(X, Mask, Shr->getName() + ".mask");
2754 return new ICmpInst(Pred, And, ConstantInt::get(ShrTy, C << ShAmtVal));
2755 }
2756
2757 return nullptr;
2758}
2759
2761 BinaryOperator *SRem,
2762 const APInt &C) {
2763 const ICmpInst::Predicate Pred = Cmp.getPredicate();
2764 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULT) {
2765 // Canonicalize unsigned predicates to signed:
2766 // (X s% DivisorC) u> C -> (X s% DivisorC) s< 0
2767 // iff (C s< 0 ? ~C : C) u>= abs(DivisorC)-1
2768 // (X s% DivisorC) u< C+1 -> (X s% DivisorC) s> -1
2769 // iff (C+1 s< 0 ? ~C : C) u>= abs(DivisorC)-1
2770
2771 const APInt *DivisorC;
2772 if (!match(SRem->getOperand(1), m_APInt(DivisorC)))
2773 return nullptr;
2774 if (DivisorC->isZero())
2775 return nullptr;
2776
2777 APInt NormalizedC = C;
2778 if (Pred == ICmpInst::ICMP_ULT) {
2779 assert(!NormalizedC.isZero() &&
2780 "ult X, 0 should have been simplified already.");
2781 --NormalizedC;
2782 }
2783 if (C.isNegative())
2784 NormalizedC.flipAllBits();
2785 if (!NormalizedC.uge(DivisorC->abs() - 1))
2786 return nullptr;
2787
2788 Type *Ty = SRem->getType();
2789 if (Pred == ICmpInst::ICMP_UGT)
2790 return new ICmpInst(ICmpInst::ICMP_SLT, SRem,
2792 return new ICmpInst(ICmpInst::ICMP_SGT, SRem,
2794 }
2795 // Match an 'is positive' or 'is negative' comparison of remainder by a
2796 // constant power-of-2 value:
2797 // (X % pow2C) sgt/slt 0
2798 if (Pred != ICmpInst::ICMP_SGT && Pred != ICmpInst::ICMP_SLT &&
2799 Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE)
2800 return nullptr;
2801
2802 // TODO: The one-use check is standard because we do not typically want to
2803 // create longer instruction sequences, but this might be a special-case
2804 // because srem is not good for analysis or codegen.
2805 if (!SRem->hasOneUse())
2806 return nullptr;
2807
2808 const APInt *DivisorC;
2809 if (!match(SRem->getOperand(1), m_Power2(DivisorC)))
2810 return nullptr;
2811
2812 // For cmp_sgt/cmp_slt only zero valued C is handled.
2813 // For cmp_eq/cmp_ne only positive valued C is handled.
2814 if (((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT) &&
2815 !C.isZero()) ||
2816 ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2817 !C.isStrictlyPositive()))
2818 return nullptr;
2819
2820 // Mask off the sign bit and the modulo bits (low-bits).
2821 Type *Ty = SRem->getType();
2822 APInt SignMask = APInt::getSignMask(Ty->getScalarSizeInBits());
2823 Constant *MaskC = ConstantInt::get(Ty, SignMask | (*DivisorC - 1));
2824 Value *And = Builder.CreateAnd(SRem->getOperand(0), MaskC);
2825
2826 if (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE)
2827 return new ICmpInst(Pred, And, ConstantInt::get(Ty, C));
2828
2829 // For 'is positive?' check that the sign-bit is clear and at least 1 masked
2830 // bit is set. Example:
2831 // (i8 X % 32) s> 0 --> (X & 159) s> 0
2832 if (Pred == ICmpInst::ICMP_SGT)
2834
2835 // For 'is negative?' check that the sign-bit is set and at least 1 masked
2836 // bit is set. Example:
2837 // (i16 X % 4) s< 0 --> (X & 32771) u> 32768
2838 return new ICmpInst(ICmpInst::ICMP_UGT, And, ConstantInt::get(Ty, SignMask));
2839}
2840
2841/// Fold icmp (udiv X, Y), C.
2843 BinaryOperator *UDiv,
2844 const APInt &C) {
2845 ICmpInst::Predicate Pred = Cmp.getPredicate();
2846 Value *X = UDiv->getOperand(0);
2847 Value *Y = UDiv->getOperand(1);
2848 Type *Ty = UDiv->getType();
2849
2850 const APInt *C2;
2851 if (!match(X, m_APInt(C2)))
2852 return nullptr;
2853
2854 assert(*C2 != 0 && "udiv 0, X should have been simplified already.");
2855
2856 // (icmp ugt (udiv C2, Y), C) -> (icmp ule Y, C2/(C+1))
2857 if (Pred == ICmpInst::ICMP_UGT) {
2858 assert(!C.isMaxValue() &&
2859 "icmp ugt X, UINT_MAX should have been simplified already.");
2860 return new ICmpInst(ICmpInst::ICMP_ULE, Y,
2861 ConstantInt::get(Ty, C2->udiv(C + 1)));
2862 }
2863
2864 // (icmp ult (udiv C2, Y), C) -> (icmp ugt Y, C2/C)
2865 if (Pred == ICmpInst::ICMP_ULT) {
2866 assert(C != 0 && "icmp ult X, 0 should have been simplified already.");
2867 return new ICmpInst(ICmpInst::ICMP_UGT, Y,
2868 ConstantInt::get(Ty, C2->udiv(C)));
2869 }
2870
2871 return nullptr;
2872}
2873
2874/// Fold icmp ({su}div X, Y), C.
2876 BinaryOperator *Div,
2877 const APInt &C) {
2878 ICmpInst::Predicate Pred = Cmp.getPredicate();
2879 Value *X = Div->getOperand(0);
2880 Value *Y = Div->getOperand(1);
2881 Type *Ty = Div->getType();
2882 bool DivIsSigned = Div->getOpcode() == Instruction::SDiv;
2883
2884 // If unsigned division and the compare constant is bigger than
2885 // UMAX/2 (negative), there's only one pair of values that satisfies an
2886 // equality check, so eliminate the division:
2887 // (X u/ Y) == C --> (X == C) && (Y == 1)
2888 // (X u/ Y) != C --> (X != C) || (Y != 1)
2889 // Similarly, if signed division and the compare constant is exactly SMIN:
2890 // (X s/ Y) == SMIN --> (X == SMIN) && (Y == 1)
2891 // (X s/ Y) != SMIN --> (X != SMIN) || (Y != 1)
2892 if (Cmp.isEquality() && Div->hasOneUse() && C.isSignBitSet() &&
2893 (!DivIsSigned || C.isMinSignedValue())) {
2894 Value *XBig = Builder.CreateICmp(Pred, X, ConstantInt::get(Ty, C));
2895 Value *YOne = Builder.CreateICmp(Pred, Y, ConstantInt::get(Ty, 1));
2896 auto Logic = Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2897 return BinaryOperator::Create(Logic, XBig, YOne);
2898 }
2899
2900 // Fold: icmp pred ([us]div X, C2), C -> range test
2901 // Fold this div into the comparison, producing a range check.
2902 // Determine, based on the divide type, what the range is being
2903 // checked. If there is an overflow on the low or high side, remember
2904 // it, otherwise compute the range [low, hi) bounding the new value.
2905 // See: InsertRangeTest above for the kinds of replacements possible.
2906 const APInt *C2;
2907 if (!match(Y, m_APInt(C2)))
2908 return nullptr;
2909
2910 // FIXME: If the operand types don't match the type of the divide
2911 // then don't attempt this transform. The code below doesn't have the
2912 // logic to deal with a signed divide and an unsigned compare (and
2913 // vice versa). This is because (x /s C2) <s C produces different
2914 // results than (x /s C2) <u C or (x /u C2) <s C or even
2915 // (x /u C2) <u C. Simply casting the operands and result won't
2916 // work. :( The if statement below tests that condition and bails
2917 // if it finds it.
2918 // However, when the divisor is a positive constant and the dividend is
2919 // known non-negative, sdiv is equivalent to udiv, so we can lower
2920 // DivIsSigned and proceed through the unsigned path.
2921 if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned()) {
2922 if (!DivIsSigned || !C2->isStrictlyPositive() ||
2923 !isKnownNonNegative(X, SQ.getWithInstruction(&Cmp)))
2924 return nullptr;
2925 DivIsSigned = false;
2926 }
2927
2928 // The ProdOV computation fails on divide by 0 and divide by -1. Cases with
2929 // INT_MIN will also fail if the divisor is 1. Although folds of all these
2930 // division-by-constant cases should be present, we can not assert that they
2931 // have happened before we reach this icmp instruction.
2932 if (C2->isZero() || C2->isOne() || (DivIsSigned && C2->isAllOnes()))
2933 return nullptr;
2934
2935 // Compute Prod = C * C2. We are essentially solving an equation of
2936 // form X / C2 = C. We solve for X by multiplying C2 and C.
2937 // By solving for X, we can turn this into a range check instead of computing
2938 // a divide.
2939 APInt Prod = C * *C2;
2940
2941 // Determine if the product overflows by seeing if the product is not equal to
2942 // the divide. Make sure we do the same kind of divide as in the LHS
2943 // instruction that we're folding.
2944 bool ProdOV = (DivIsSigned ? Prod.sdiv(*C2) : Prod.udiv(*C2)) != C;
2945
2946 // If the division is known to be exact, then there is no remainder from the
2947 // divide, so the covered range size is unit, otherwise it is the divisor.
2948 APInt RangeSize = Div->isExact() ? APInt(C2->getBitWidth(), 1) : *C2;
2949
2950 // Figure out the interval that is being checked. For example, a comparison
2951 // like "X /u 5 == 0" is really checking that X is in the interval [0, 5).
2952 // Compute this interval based on the constants involved and the signedness of
2953 // the compare/divide. This computes a half-open interval, keeping track of
2954 // whether either value in the interval overflows. After analysis each
2955 // overflow variable is set to 0 if it's corresponding bound variable is valid
2956 // -1 if overflowed off the bottom end, or +1 if overflowed off the top end.
2957 int LoOverflow = 0, HiOverflow = 0;
2958 APInt LoBound, HiBound;
2959
2960 if (!DivIsSigned) { // udiv
2961 // e.g. X/5 op 3 --> [15, 20)
2962 LoBound = Prod;
2963 HiOverflow = LoOverflow = ProdOV;
2964 if (!HiOverflow) {
2965 // If this is not an exact divide, then many values in the range collapse
2966 // to the same result value.
2967 HiOverflow = addWithOverflow(HiBound, LoBound, RangeSize, false);
2968 }
2969 } else if (C2->isStrictlyPositive()) { // Divisor is > 0.
2970 if (C.isZero()) { // (X / pos) op 0
2971 // Can't overflow. e.g. X/2 op 0 --> [-1, 2)
2972 LoBound = -(RangeSize - 1);
2973 HiBound = RangeSize;
2974 } else if (C.isStrictlyPositive()) { // (X / pos) op pos
2975 LoBound = Prod; // e.g. X/5 op 3 --> [15, 20)
2976 HiOverflow = LoOverflow = ProdOV;
2977 if (!HiOverflow)
2978 HiOverflow = addWithOverflow(HiBound, Prod, RangeSize, true);
2979 } else { // (X / pos) op neg
2980 // e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14)
2981 HiBound = Prod + 1;
2982 LoOverflow = HiOverflow = ProdOV ? -1 : 0;
2983 if (!LoOverflow) {
2984 APInt DivNeg = -RangeSize;
2985 LoOverflow = addWithOverflow(LoBound, HiBound, DivNeg, true) ? -1 : 0;
2986 }
2987 }
2988 } else if (C2->isNegative()) { // Divisor is < 0.
2989 if (Div->isExact())
2990 RangeSize.negate();
2991 if (C.isZero()) { // (X / neg) op 0
2992 // e.g. X/-5 op 0 --> [-4, 5)
2993 LoBound = RangeSize + 1;
2994 HiBound = -RangeSize;
2995 if (HiBound == *C2) { // -INTMIN = INTMIN
2996 HiOverflow = 1; // [INTMIN+1, overflow)
2997 HiBound = APInt(); // e.g. X/INTMIN = 0 --> X > INTMIN
2998 }
2999 } else if (C.isStrictlyPositive()) { // (X / neg) op pos
3000 // e.g. X/-5 op 3 --> [-19, -14)
3001 HiBound = Prod + 1;
3002 HiOverflow = LoOverflow = ProdOV ? -1 : 0;
3003 if (!LoOverflow)
3004 LoOverflow =
3005 addWithOverflow(LoBound, HiBound, RangeSize, true) ? -1 : 0;
3006 } else { // (X / neg) op neg
3007 LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20)
3008 LoOverflow = HiOverflow = ProdOV;
3009 if (!HiOverflow)
3010 HiOverflow = subWithOverflow(HiBound, Prod, RangeSize, true);
3011 }
3012
3013 // Dividing by a negative swaps the condition. LT <-> GT
3014 Pred = ICmpInst::getSwappedPredicate(Pred);
3015 }
3016
3017 switch (Pred) {
3018 default:
3019 llvm_unreachable("Unhandled icmp predicate!");
3020 case ICmpInst::ICMP_EQ:
3021 if (LoOverflow && HiOverflow)
3022 return replaceInstUsesWith(Cmp, Builder.getFalse());
3023 if (HiOverflow)
3024 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE,
3025 X, ConstantInt::get(Ty, LoBound));
3026 if (LoOverflow)
3027 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
3028 X, ConstantInt::get(Ty, HiBound));
3029 return replaceInstUsesWith(
3030 Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, true));
3031 case ICmpInst::ICMP_NE:
3032 if (LoOverflow && HiOverflow)
3033 return replaceInstUsesWith(Cmp, Builder.getTrue());
3034 if (HiOverflow)
3035 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
3036 X, ConstantInt::get(Ty, LoBound));
3037 if (LoOverflow)
3038 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE,
3039 X, ConstantInt::get(Ty, HiBound));
3040 return replaceInstUsesWith(
3041 Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, false));
3042 case ICmpInst::ICMP_ULT:
3043 case ICmpInst::ICMP_SLT:
3044 if (LoOverflow == +1) // Low bound is greater than input range.
3045 return replaceInstUsesWith(Cmp, Builder.getTrue());
3046 if (LoOverflow == -1) // Low bound is less than input range.
3047 return replaceInstUsesWith(Cmp, Builder.getFalse());
3048 return new ICmpInst(Pred, X, ConstantInt::get(Ty, LoBound));
3049 case ICmpInst::ICMP_UGT:
3050 case ICmpInst::ICMP_SGT:
3051 if (HiOverflow == +1) // High bound greater than input range.
3052 return replaceInstUsesWith(Cmp, Builder.getFalse());
3053 if (HiOverflow == -1) // High bound less than input range.
3054 return replaceInstUsesWith(Cmp, Builder.getTrue());
3055 if (Pred == ICmpInst::ICMP_UGT)
3056 return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, HiBound));
3057 return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, HiBound));
3058 }
3059
3060 return nullptr;
3061}
3062
3063/// Fold icmp (sub X, Y), C.
3066 const APInt &C) {
3067 Value *X = Sub->getOperand(0), *Y = Sub->getOperand(1);
3068 ICmpInst::Predicate Pred = Cmp.getPredicate();
3069 Type *Ty = Sub->getType();
3070
3071 // (X - (X urem D)) is D*(X/D), a multiple of D, so it is u> C exactly when
3072 // X u>= D (for C u< D), and u< C exactly when X u< D (for 0 u< C u<= D):
3073 // icmp ugt (sub X, (urem X, D)), C --> icmp ugt X, D-1
3074 // icmp ult (sub X, (urem X, D)), C --> icmp ult X, D
3075 const APInt *D;
3076 if (match(Y, m_URem(m_Specific(X), m_APInt(D))) && !D->isZero()) {
3077 if (Pred == ICmpInst::ICMP_UGT && C.ult(*D))
3078 return new ICmpInst(ICmpInst::ICMP_UGT, X, ConstantInt::get(Ty, *D - 1));
3079 if (Pred == ICmpInst::ICMP_ULT && !C.isZero() && C.ule(*D))
3080 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, *D));
3081 }
3082
3083 // (SubC - Y) == C) --> Y == (SubC - C)
3084 // (SubC - Y) != C) --> Y != (SubC - C)
3085 Constant *SubC;
3086 if (Cmp.isEquality() && match(X, m_ImmConstant(SubC))) {
3087 return new ICmpInst(Pred, Y,
3088 ConstantExpr::getSub(SubC, ConstantInt::get(Ty, C)));
3089 }
3090
3091 // (icmp P (sub nuw|nsw C2, Y), C) -> (icmp swap(P) Y, C2-C)
3092 const APInt *C2;
3093 APInt SubResult;
3094 ICmpInst::Predicate SwappedPred = Cmp.getSwappedPredicate();
3095 bool HasNSW = Sub->hasNoSignedWrap();
3096 bool HasNUW = Sub->hasNoUnsignedWrap();
3097 if (match(X, m_APInt(C2)) &&
3098 ((Cmp.isUnsigned() && HasNUW) || (Cmp.isSigned() && HasNSW)) &&
3099 !subWithOverflow(SubResult, *C2, C, Cmp.isSigned()))
3100 return new ICmpInst(SwappedPred, Y, ConstantInt::get(Ty, SubResult));
3101
3102 // X - Y == 0 --> X == Y.
3103 // X - Y != 0 --> X != Y.
3104 // TODO: We allow this with multiple uses as long as the other uses are not
3105 // in phis. The phi use check is guarding against a codegen regression
3106 // for a loop test. If the backend could undo this (and possibly
3107 // subsequent transforms), we would not need this hack.
3108 if (Cmp.isEquality() && C.isZero() &&
3109 none_of((Sub->users()), [](const User *U) { return isa<PHINode>(U); }))
3110 return new ICmpInst(Pred, X, Y);
3111
3112 // The following transforms are only worth it if the only user of the subtract
3113 // is the icmp.
3114 // TODO: This is an artificial restriction for all of the transforms below
3115 // that only need a single replacement icmp. Can these use the phi test
3116 // like the transform above here?
3117 if (!Sub->hasOneUse())
3118 return nullptr;
3119
3120 if (Sub->hasNoSignedWrap()) {
3121 // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y)
3122 if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes())
3123 return new ICmpInst(ICmpInst::ICMP_SGE, X, Y);
3124
3125 // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y)
3126 if (Pred == ICmpInst::ICMP_SGT && C.isZero())
3127 return new ICmpInst(ICmpInst::ICMP_SGT, X, Y);
3128
3129 // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y)
3130 if (Pred == ICmpInst::ICMP_SLT && C.isZero())
3131 return new ICmpInst(ICmpInst::ICMP_SLT, X, Y);
3132
3133 // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y)
3134 if (Pred == ICmpInst::ICMP_SLT && C.isOne())
3135 return new ICmpInst(ICmpInst::ICMP_SLE, X, Y);
3136 }
3137
3138 if (!match(X, m_APInt(C2)))
3139 return nullptr;
3140
3141 // C2 - Y <u C -> (Y | (C - 1)) == C2
3142 // iff (C2 & (C - 1)) == C - 1 and C is a power of 2
3143 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() &&
3144 (*C2 & (C - 1)) == (C - 1))
3145 return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateOr(Y, C - 1), X);
3146
3147 // C2 - Y >u C -> (Y | C) != C2
3148 // iff C2 & C == C and C + 1 is a power of 2
3149 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == C)
3150 return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateOr(Y, C), X);
3151
3152 // We have handled special cases that reduce.
3153 // Canonicalize any remaining sub to add as:
3154 // (C2 - Y) > C --> (Y + ~C2) < ~C
3155 Value *Add = Builder.CreateAdd(Y, ConstantInt::get(Ty, ~(*C2)), "notsub",
3156 HasNUW, HasNSW);
3157 return new ICmpInst(SwappedPred, Add, ConstantInt::get(Ty, ~C));
3158}
3159
3160static Value *createLogicFromTable(const std::bitset<4> &Table, Value *Op0,
3161 Value *Op1, IRBuilderBase &Builder,
3162 bool HasOneUse) {
3163 auto FoldConstant = [&](bool Val) {
3164 Constant *Res = Val ? Builder.getTrue() : Builder.getFalse();
3165 if (Op0->getType()->isVectorTy())
3167 cast<VectorType>(Op0->getType())->getElementCount(), Res);
3168 return Res;
3169 };
3170
3171 switch (Table.to_ulong()) {
3172 case 0: // 0 0 0 0
3173 return FoldConstant(false);
3174 case 1: // 0 0 0 1
3175 return HasOneUse ? Builder.CreateNot(Builder.CreateOr(Op0, Op1)) : nullptr;
3176 case 2: // 0 0 1 0
3177 return HasOneUse ? Builder.CreateAnd(Builder.CreateNot(Op0), Op1) : nullptr;
3178 case 3: // 0 0 1 1
3179 return Builder.CreateNot(Op0);
3180 case 4: // 0 1 0 0
3181 return HasOneUse ? Builder.CreateAnd(Op0, Builder.CreateNot(Op1)) : nullptr;
3182 case 5: // 0 1 0 1
3183 return Builder.CreateNot(Op1);
3184 case 6: // 0 1 1 0
3185 return Builder.CreateXor(Op0, Op1);
3186 case 7: // 0 1 1 1
3187 return HasOneUse ? Builder.CreateNot(Builder.CreateAnd(Op0, Op1)) : nullptr;
3188 case 8: // 1 0 0 0
3189 return Builder.CreateAnd(Op0, Op1);
3190 case 9: // 1 0 0 1
3191 return HasOneUse ? Builder.CreateNot(Builder.CreateXor(Op0, Op1)) : nullptr;
3192 case 10: // 1 0 1 0
3193 return Op1;
3194 case 11: // 1 0 1 1
3195 return HasOneUse ? Builder.CreateOr(Builder.CreateNot(Op0), Op1) : nullptr;
3196 case 12: // 1 1 0 0
3197 return Op0;
3198 case 13: // 1 1 0 1
3199 return HasOneUse ? Builder.CreateOr(Op0, Builder.CreateNot(Op1)) : nullptr;
3200 case 14: // 1 1 1 0
3201 return Builder.CreateOr(Op0, Op1);
3202 case 15: // 1 1 1 1
3203 return FoldConstant(true);
3204 default:
3205 llvm_unreachable("Invalid Operation");
3206 }
3207 return nullptr;
3208}
3209
3211 ICmpInst &Cmp, BinaryOperator *BO, const APInt &C) {
3212 Value *A, *B;
3213 Constant *C1, *C2, *C3, *C4;
3214 if (!match(BO->getOperand(0),
3216 !match(BO->getOperand(1),
3218 Cmp.getType() != A->getType() || Cmp.getType() != B->getType())
3219 return nullptr;
3220
3221 std::bitset<4> Table;
3222 auto ComputeTable = [&](bool First, bool Second) -> std::optional<bool> {
3223 Constant *L = First ? C1 : C2;
3224 Constant *R = Second ? C3 : C4;
3225 if (auto *Res = ConstantFoldBinaryOpOperands(BO->getOpcode(), L, R, DL)) {
3226 auto *Val = Res->getType()->isVectorTy() ? Res->getSplatValue() : Res;
3227 if (auto *CI = dyn_cast_or_null<ConstantInt>(Val))
3228 return ICmpInst::compare(CI->getValue(), C, Cmp.getPredicate());
3229 }
3230 return std::nullopt;
3231 };
3232
3233 for (unsigned I = 0; I < 4; ++I) {
3234 bool First = (I >> 1) & 1;
3235 bool Second = I & 1;
3236 if (auto Res = ComputeTable(First, Second))
3237 Table[I] = *Res;
3238 else
3239 return nullptr;
3240 }
3241
3242 // Synthesize optimal logic.
3243 if (auto *Cond = createLogicFromTable(Table, A, B, Builder, BO->hasOneUse()))
3244 return replaceInstUsesWith(Cmp, Cond);
3245 return nullptr;
3246}
3247
3248/// Fold icmp (add X, Y), C.
3251 const APInt &C) {
3252 Value *Y = Add->getOperand(1);
3253 Value *X = Add->getOperand(0);
3254 const CmpPredicate Pred = Cmp.getCmpPredicate();
3255
3256 // icmp ult (add nuw A, (lshr A, ShAmtC)), C --> icmp ult A, C
3257 // when C <= (1 << ShAmtC).
3258 const APInt *ShAmtC;
3259 Value *A;
3260 unsigned BitWidth = C.getBitWidth();
3261 if (Pred == ICmpInst::ICMP_ULT &&
3262 match(Add,
3263 m_c_NUWAdd(m_Value(A), m_LShr(m_Deferred(A), m_APInt(ShAmtC)))) &&
3264 ShAmtC->ult(BitWidth) &&
3265 C.ule(APInt::getOneBitSet(BitWidth, ShAmtC->getZExtValue())))
3266 return new ICmpInst(Pred, A, ConstantInt::get(A->getType(), C));
3267
3268 const APInt *C2;
3269 if (Cmp.isEquality() || !match(Y, m_APInt(C2)))
3270 return nullptr;
3271
3272 // Fold icmp pred (add X, C2), C.
3273 Type *Ty = Add->getType();
3274
3275 // If the add does not wrap, we can always adjust the compare by subtracting
3276 // the constants. Equality comparisons are handled elsewhere. SGE/SLE/UGE/ULE
3277 // have been canonicalized to SGT/SLT/UGT/ULT.
3278 if (Add->hasNoUnsignedWrap() &&
3279 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULT)) {
3280 bool Overflow;
3281 APInt NewC = C.usub_ov(*C2, Overflow);
3282 // If there is overflow, the result must be true or false.
3283 if (!Overflow)
3284 // icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2)
3285 return new ICmpInst(Pred, X, ConstantInt::get(Ty, NewC));
3286 }
3287
3288 CmpInst::Predicate ChosenPred = Pred.getPreferredSignedPredicate();
3289
3290 if (Add->hasNoSignedWrap() &&
3291 (ChosenPred == ICmpInst::ICMP_SGT || ChosenPred == ICmpInst::ICMP_SLT)) {
3292 bool Overflow;
3293 APInt NewC = C.ssub_ov(*C2, Overflow);
3294 if (!Overflow)
3295 // icmp samesign ugt/ult (add nsw X, C2), C
3296 // -> icmp sgt/slt X, (C - C2)
3297 return new ICmpInst(ChosenPred, X, ConstantInt::get(Ty, NewC));
3298 }
3299
3300 if (ICmpInst::isUnsigned(Pred) && Add->hasNoSignedWrap() &&
3301 C.isNonNegative() && (C - *C2).isNonNegative() &&
3302 computeConstantRange(X, /*ForSigned=*/true, SQ.getWithInstruction(&Cmp))
3303 .add(*C2)
3304 .isAllNonNegative())
3305 return new ICmpInst(ICmpInst::getSignedPredicate(Pred), X,
3306 ConstantInt::get(Ty, C - *C2));
3307
3308 auto CR = ConstantRange::makeExactICmpRegion(Pred, C).subtract(*C2);
3309 const APInt &Upper = CR.getUpper();
3310 const APInt &Lower = CR.getLower();
3311 if (Cmp.isSigned()) {
3312 if (Lower.isSignMask())
3313 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, Upper));
3314 if (Upper.isSignMask())
3315 return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, Lower));
3316 } else {
3317 if (Lower.isMinValue())
3318 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, Upper));
3319 if (Upper.isMinValue())
3320 return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, Lower));
3321 }
3322
3323 // This set of folds is intentionally placed after folds that use no-wrapping
3324 // flags because those folds are likely better for later analysis/codegen.
3325 const APInt SMax = APInt::getSignedMaxValue(Ty->getScalarSizeInBits());
3326 const APInt SMin = APInt::getSignedMinValue(Ty->getScalarSizeInBits());
3327
3328 // Fold compare with offset to opposite sign compare if it eliminates offset:
3329 // (X + C2) >u C --> X <s -C2 (if C == C2 + SMAX)
3330 if (Pred == CmpInst::ICMP_UGT && C == *C2 + SMax)
3331 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, -(*C2)));
3332
3333 // (X + C2) <u C --> X >s ~C2 (if C == C2 + SMIN)
3334 if (Pred == CmpInst::ICMP_ULT && C == *C2 + SMin)
3335 return new ICmpInst(ICmpInst::ICMP_SGT, X, ConstantInt::get(Ty, ~(*C2)));
3336
3337 // (X + C2) >s C --> X <u (SMAX - C) (if C == C2 - 1)
3338 if (Pred == CmpInst::ICMP_SGT && C == *C2 - 1)
3339 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, SMax - C));
3340
3341 // (X + C2) <s C --> X >u (C ^ SMAX) (if C == C2)
3342 if (Pred == CmpInst::ICMP_SLT && C == *C2)
3343 return new ICmpInst(ICmpInst::ICMP_UGT, X, ConstantInt::get(Ty, C ^ SMax));
3344
3345 // (X + -1) <u C --> X <=u C (if X is never null)
3346 if (Pred == CmpInst::ICMP_ULT && C2->isAllOnes()) {
3347 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
3348 if (llvm::isKnownNonZero(X, Q))
3349 return new ICmpInst(ICmpInst::ICMP_ULE, X, ConstantInt::get(Ty, C));
3350 }
3351
3352 if (!Add->hasOneUse())
3353 return nullptr;
3354
3355 // X+C <u C2 -> (X & -C2) == C
3356 // iff C & (C2-1) == 0
3357 // C2 is a power of 2
3358 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && (*C2 & (C - 1)) == 0)
3359 return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateAnd(X, -C),
3361
3362 // X+C2 <u C -> (X & C) == 2C
3363 // iff C == -(C2)
3364 // C2 is a power of 2
3365 if (Pred == ICmpInst::ICMP_ULT && C2->isPowerOf2() && C == -*C2)
3366 return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateAnd(X, C),
3367 ConstantInt::get(Ty, C * 2));
3368
3369 // X+C >u C2 -> (X & ~C2) != C
3370 // iff C & C2 == 0
3371 // C2+1 is a power of 2
3372 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == 0)
3373 return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateAnd(X, ~C),
3375
3376 // The range test idiom can use either ult or ugt. Arbitrarily canonicalize
3377 // to the ult form.
3378 // X+C2 >u C -> X+(C2-C-1) <u ~C
3379 if (Pred == ICmpInst::ICMP_UGT)
3380 return new ICmpInst(ICmpInst::ICMP_ULT,
3381 Builder.CreateAdd(X, ConstantInt::get(Ty, *C2 - C - 1)),
3382 ConstantInt::get(Ty, ~C));
3383
3384 // zext(V) + C2 pred C -> V + C3 pred' C4
3385 Value *V;
3386 if (match(X, m_ZExt(m_Value(V)))) {
3387 Type *NewCmpTy = V->getType();
3388 unsigned NewCmpBW = NewCmpTy->getScalarSizeInBits();
3389 if (shouldChangeType(Ty, NewCmpTy)) {
3390 ConstantRange SrcCR = CR.truncate(NewCmpBW, TruncInst::NoUnsignedWrap);
3391 CmpInst::Predicate EquivPred;
3392 APInt EquivInt;
3393 APInt EquivOffset;
3394
3395 SrcCR.getEquivalentICmp(EquivPred, EquivInt, EquivOffset);
3396 return new ICmpInst(
3397 EquivPred,
3398 EquivOffset.isZero()
3399 ? V
3400 : Builder.CreateAdd(V, ConstantInt::get(NewCmpTy, EquivOffset)),
3401 ConstantInt::get(NewCmpTy, EquivInt));
3402 }
3403 }
3404
3405 return nullptr;
3406}
3407
3409 Value *&RHS, ConstantInt *&Less,
3410 ConstantInt *&Equal,
3411 ConstantInt *&Greater) {
3412 // TODO: Generalize this to work with other comparison idioms or ensure
3413 // they get canonicalized into this form.
3414
3415 // select i1 (a == b),
3416 // i32 Equal,
3417 // i32 (select i1 (a < b), i32 Less, i32 Greater)
3418 // where Equal, Less and Greater are placeholders for any three constants.
3419 CmpPredicate PredA;
3420 if (!match(SI->getCondition(), m_ICmp(PredA, m_Value(LHS), m_Value(RHS))) ||
3421 !ICmpInst::isEquality(PredA))
3422 return false;
3423 Value *EqualVal = SI->getTrueValue();
3424 Value *UnequalVal = SI->getFalseValue();
3425 // We still can get non-canonical predicate here, so canonicalize.
3426 if (PredA == ICmpInst::ICMP_NE)
3427 std::swap(EqualVal, UnequalVal);
3428 if (!match(EqualVal, m_ConstantInt(Equal)))
3429 return false;
3430 CmpPredicate PredB;
3431 Value *LHS2, *RHS2;
3432 if (!match(UnequalVal, m_Select(m_ICmp(PredB, m_Value(LHS2), m_Value(RHS2)),
3433 m_ConstantInt(Less), m_ConstantInt(Greater))))
3434 return false;
3435 // We can get predicate mismatch here, so canonicalize if possible:
3436 // First, ensure that 'LHS' match.
3437 if (LHS2 != LHS) {
3438 // x sgt y <--> y slt x
3439 std::swap(LHS2, RHS2);
3440 PredB = ICmpInst::getSwappedPredicate(PredB);
3441 }
3442 if (LHS2 != LHS)
3443 return false;
3444 // We also need to canonicalize 'RHS'.
3445 if (PredB == ICmpInst::ICMP_SGT && isa<Constant>(RHS2)) {
3446 // x sgt C-1 <--> x sge C <--> not(x slt C)
3447 auto FlippedStrictness =
3449 if (!FlippedStrictness)
3450 return false;
3451 assert(FlippedStrictness->first == ICmpInst::ICMP_SGE &&
3452 "basic correctness failure");
3453 RHS2 = FlippedStrictness->second;
3454 // And kind-of perform the result swap.
3455 std::swap(Less, Greater);
3456 PredB = ICmpInst::ICMP_SLT;
3457 }
3458 return PredB == ICmpInst::ICMP_SLT && RHS == RHS2;
3459}
3460
3463 ConstantInt *C) {
3464
3465 assert(C && "Cmp RHS should be a constant int!");
3466 // If we're testing a constant value against the result of a three way
3467 // comparison, the result can be expressed directly in terms of the
3468 // original values being compared. Note: We could possibly be more
3469 // aggressive here and remove the hasOneUse test. The original select is
3470 // really likely to simplify or sink when we remove a test of the result.
3471 Value *OrigLHS, *OrigRHS;
3472 ConstantInt *C1LessThan, *C2Equal, *C3GreaterThan;
3473 if (Cmp.hasOneUse() &&
3474 matchThreeWayIntCompare(Select, OrigLHS, OrigRHS, C1LessThan, C2Equal,
3475 C3GreaterThan)) {
3476 assert(C1LessThan && C2Equal && C3GreaterThan);
3477
3478 bool TrueWhenLessThan = ICmpInst::compare(
3479 C1LessThan->getValue(), C->getValue(), Cmp.getPredicate());
3480 bool TrueWhenEqual = ICmpInst::compare(C2Equal->getValue(), C->getValue(),
3481 Cmp.getPredicate());
3482 bool TrueWhenGreaterThan = ICmpInst::compare(
3483 C3GreaterThan->getValue(), C->getValue(), Cmp.getPredicate());
3484
3485 // This generates the new instruction that will replace the original Cmp
3486 // Instruction. Instead of enumerating the various combinations when
3487 // TrueWhenLessThan, TrueWhenEqual and TrueWhenGreaterThan are true versus
3488 // false, we rely on chaining of ORs and future passes of InstCombine to
3489 // simplify the OR further (i.e. a s< b || a == b becomes a s<= b).
3490
3491 // When none of the three constants satisfy the predicate for the RHS (C),
3492 // the entire original Cmp can be simplified to a false.
3493 Value *Cond = Builder.getFalse();
3494 if (TrueWhenLessThan)
3495 Cond = Builder.CreateOr(
3496 Cond, Builder.CreateICmp(ICmpInst::ICMP_SLT, OrigLHS, OrigRHS));
3497 if (TrueWhenEqual)
3498 Cond = Builder.CreateOr(
3499 Cond, Builder.CreateICmp(ICmpInst::ICMP_EQ, OrigLHS, OrigRHS));
3500 if (TrueWhenGreaterThan)
3501 Cond = Builder.CreateOr(
3502 Cond, Builder.CreateICmp(ICmpInst::ICMP_SGT, OrigLHS, OrigRHS));
3503
3504 return replaceInstUsesWith(Cmp, Cond);
3505 }
3506 return nullptr;
3507}
3508
3510 auto *Bitcast = dyn_cast<BitCastInst>(Cmp.getOperand(0));
3511 if (!Bitcast)
3512 return nullptr;
3513
3514 ICmpInst::Predicate Pred = Cmp.getPredicate();
3515 Value *Op1 = Cmp.getOperand(1);
3516 Value *BCSrcOp = Bitcast->getOperand(0);
3517 Type *SrcType = Bitcast->getSrcTy();
3518 Type *DstType = Bitcast->getType();
3519
3520 // Make sure the bitcast doesn't change between scalar and vector and
3521 // doesn't change the number of vector elements.
3522 if (SrcType->isVectorTy() == DstType->isVectorTy() &&
3523 SrcType->getScalarSizeInBits() == DstType->getScalarSizeInBits()) {
3524 // Zero-equality and sign-bit checks are preserved through sitofp + bitcast.
3525 Value *X;
3526 if (match(BCSrcOp, m_SIToFP(m_Value(X)))) {
3527 // icmp eq (bitcast (sitofp X)), 0 --> icmp eq X, 0
3528 // icmp ne (bitcast (sitofp X)), 0 --> icmp ne X, 0
3529 // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0
3530 // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0
3531 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT ||
3532 Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) &&
3533 match(Op1, m_Zero()))
3534 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
3535
3536 // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1
3537 if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_One()))
3538 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), 1));
3539
3540 // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1
3541 if (Pred == ICmpInst::ICMP_SGT && match(Op1, m_AllOnes()))
3542 return new ICmpInst(Pred, X,
3543 ConstantInt::getAllOnesValue(X->getType()));
3544 }
3545
3546 // Zero-equality checks are preserved through unsigned floating-point casts:
3547 // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0
3548 // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0
3549 if (match(BCSrcOp, m_UIToFP(m_Value(X))))
3550 if (Cmp.isEquality() && match(Op1, m_Zero()))
3551 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
3552
3553 const APInt *C;
3554 bool TrueIfSigned;
3555 if (match(Op1, m_APInt(C)) && Bitcast->hasOneUse()) {
3556 // If this is a sign-bit test of a bitcast of a casted FP value, eliminate
3557 // the FP extend/truncate because that cast does not change the sign-bit.
3558 // This is true for all standard IEEE-754 types and the X86 80-bit type.
3559 // The sign-bit is always the most significant bit in those types.
3560 if (isSignBitCheck(Pred, *C, TrueIfSigned) &&
3561 (match(BCSrcOp, m_FPExt(m_Value(X))) ||
3562 match(BCSrcOp, m_FPTrunc(m_Value(X))))) {
3563 // (bitcast (fpext/fptrunc X)) to iX) < 0 --> (bitcast X to iY) < 0
3564 // (bitcast (fpext/fptrunc X)) to iX) > -1 --> (bitcast X to iY) > -1
3565 Type *XType = X->getType();
3566
3567 // We can't currently handle Power style floating point operations here.
3568 if (!(XType->isPPC_FP128Ty() || SrcType->isPPC_FP128Ty())) {
3569 Type *NewType = Builder.getIntNTy(XType->getScalarSizeInBits());
3570 if (auto *XVTy = dyn_cast<VectorType>(XType))
3571 NewType = VectorType::get(NewType, XVTy->getElementCount());
3572 Value *NewBitcast = Builder.CreateBitCast(X, NewType);
3573 if (TrueIfSigned)
3574 return new ICmpInst(ICmpInst::ICMP_SLT, NewBitcast,
3575 ConstantInt::getNullValue(NewType));
3576 else
3577 return new ICmpInst(ICmpInst::ICMP_SGT, NewBitcast,
3579 }
3580 }
3581
3582 // icmp eq/ne (bitcast X to int), special fp -> llvm.is.fpclass(X, class)
3583 Type *FPType = SrcType->getScalarType();
3584 if (!Cmp.getParent()->getParent()->hasFnAttribute(
3585 Attribute::NoImplicitFloat) &&
3586 Cmp.isEquality() && FPType->isIEEELikeFPTy()) {
3587 FPClassTest Mask = APFloat(FPType->getFltSemantics(), *C).classify();
3588 if (Mask & (fcInf | fcZero)) {
3589 if (Pred == ICmpInst::ICMP_NE)
3590 Mask = ~Mask;
3591 return replaceInstUsesWith(Cmp,
3592 Builder.createIsFPClass(BCSrcOp, Mask));
3593 }
3594 }
3595 }
3596 }
3597
3598 // Fold the canonicalized form of vector_reduce_or if the arg is
3599 // get_active_lane mask.
3600 // icmp ne (bitcast <N x i1> to iN (get_active_lane_mask(l, h))), 0 ->
3601 // icmp ult l, h
3602 // icmp eq (bitcast <N x i1> to iN (get_active_lane_mask(l, h))), 0 ->
3603 // icmp uge l, h
3604 Value *Upper, *Lower;
3606 m_Value(Lower), m_Value(Upper))) &&
3607 match(Op1, m_Zero()) && DstType->isIntegerTy()) {
3608 if (Pred == ICmpInst::ICMP_NE)
3609 return new ICmpInst(ICmpInst::ICMP_ULT, Lower, Upper);
3610 if (Pred == ICmpInst::ICMP_EQ)
3611 return new ICmpInst(ICmpInst::ICMP_UGE, Lower, Upper);
3612 }
3613
3614 const APInt *C;
3615 if (!match(Cmp.getOperand(1), m_APInt(C)) || !DstType->isIntegerTy() ||
3616 !SrcType->isIntOrIntVectorTy())
3617 return nullptr;
3618
3619 // If this is checking if all elements of a vector compare are set or not,
3620 // invert the casted vector equality compare and test if all compare
3621 // elements are clear or not. Compare against zero is generally easier for
3622 // analysis and codegen.
3623 // icmp eq/ne (bitcast (not X) to iN), -1 --> icmp eq/ne (bitcast X to iN), 0
3624 // Example: are all elements equal? --> are zero elements not equal?
3625 // TODO: Try harder to reduce compare of 2 freely invertible operands?
3626 if (Cmp.isEquality() && C->isAllOnes() && Bitcast->hasOneUse()) {
3627 if (Value *NotBCSrcOp =
3628 getFreelyInverted(BCSrcOp, BCSrcOp->hasOneUse(), &Builder)) {
3629 Value *Cast = Builder.CreateBitCast(NotBCSrcOp, DstType);
3630 return new ICmpInst(Pred, Cast, ConstantInt::getNullValue(DstType));
3631 }
3632 }
3633
3634 // If this is checking if all elements of an extended vector are clear or not,
3635 // compare in a narrow type to eliminate the extend:
3636 // icmp eq/ne (bitcast (ext X) to iN), 0 --> icmp eq/ne (bitcast X to iM), 0
3637 Value *X;
3638 if (Cmp.isEquality() && C->isZero() && Bitcast->hasOneUse() &&
3639 match(BCSrcOp, m_ZExtOrSExt(m_Value(X)))) {
3640 if (auto *VecTy = dyn_cast<FixedVectorType>(X->getType())) {
3641 Type *NewType = Builder.getIntNTy(VecTy->getPrimitiveSizeInBits());
3642 Value *NewCast = Builder.CreateBitCast(X, NewType);
3643 return new ICmpInst(Pred, NewCast, ConstantInt::getNullValue(NewType));
3644 }
3645 }
3646
3647 // Folding: icmp <pred> iN X, C
3648 // where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN
3649 // and C is a splat of a K-bit pattern
3650 // and SC is a constant vector = <C', C', C', ..., C'>
3651 // Into:
3652 // %E = extractelement <M x iK> %vec, i32 C'
3653 // icmp <pred> iK %E, trunc(C)
3654 Value *Vec;
3655 ArrayRef<int> Mask;
3656 if (match(BCSrcOp, m_Shuffle(m_Value(Vec), m_Undef(), m_Mask(Mask)))) {
3657 // Check whether every element of Mask is the same constant
3658 if (all_equal(Mask)) {
3659 auto *VecTy = cast<VectorType>(SrcType);
3660 auto *EltTy = cast<IntegerType>(VecTy->getElementType());
3661 if (C->isSplat(EltTy->getBitWidth())) {
3662 // Fold the icmp based on the value of C
3663 // If C is M copies of an iK sized bit pattern,
3664 // then:
3665 // => %E = extractelement <N x iK> %vec, i64 Elem
3666 // icmp <pred> iK %SplatVal, <pattern>
3667 Value *Extract = Builder.CreateExtractElement(Vec, Mask[0]);
3668 Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth()));
3669 return new ICmpInst(Pred, Extract, NewC);
3670 }
3671 }
3672 }
3673 return nullptr;
3674}
3675
3676/// Try to fold integer comparisons with a constant operand: icmp Pred X, C
3677/// where X is some kind of instruction.
3679 const APInt *C;
3680
3681 if (match(Cmp.getOperand(1), m_APInt(C))) {
3682 if (auto *BO = dyn_cast<BinaryOperator>(Cmp.getOperand(0)))
3683 if (Instruction *I = foldICmpBinOpWithConstant(Cmp, BO, *C))
3684 return I;
3685
3686 if (auto *SI = dyn_cast<SelectInst>(Cmp.getOperand(0)))
3687 // For now, we only support constant integers while folding the
3688 // ICMP(SELECT)) pattern. We can extend this to support vector of integers
3689 // similar to the cases handled by binary ops above.
3690 if (auto *ConstRHS = dyn_cast<ConstantInt>(Cmp.getOperand(1)))
3691 if (Instruction *I = foldICmpSelectConstant(Cmp, SI, ConstRHS))
3692 return I;
3693
3694 if (auto *TI = dyn_cast<TruncInst>(Cmp.getOperand(0)))
3695 if (Instruction *I = foldICmpTruncConstant(Cmp, TI, *C))
3696 return I;
3697
3698 if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0)))
3700 return I;
3701
3702 {
3703 // icmp slt/sgt (extractvalue (frexp X), 1), C -->
3704 // fcmp olt/oge (fabs X), 2^ExpVal
3705 // slt -> olt, ExpVal = C-1; sgt -> oge, ExpVal = C.
3706 Value *X;
3707 if (match(Cmp.getOperand(0),
3710 ICmpInst::Predicate Pred = Cmp.getPredicate();
3711 APInt Exp;
3712 FCmpInst::Predicate NewPred;
3713 bool ValidPred = true;
3714
3715 switch (Pred) {
3716 case ICmpInst::ICMP_SLT:
3717 NewPred = FCmpInst::FCMP_OLT;
3718 Exp = *C - 1;
3719 break;
3720 case ICmpInst::ICMP_SGT:
3721 NewPred = FCmpInst::FCMP_OGE;
3722 Exp = *C;
3723 break;
3724 default:
3725 ValidPred = false;
3726 break;
3727 }
3728
3729 if (ValidPred) {
3730 const fltSemantics &Sem =
3731 X->getType()->getScalarType()->getFltSemantics();
3732 int MaxExp = APFloat::semanticsMaxExponent(Sem);
3733
3734 if (!Exp.isNegative() && Exp.sle(MaxExp + 1) &&
3735 isKnownNeverInfOrNaN(X, SQ.getWithInstruction(&Cmp))) {
3736 int ExpVal = static_cast<int>(Exp.getSExtValue());
3737 APFloat CmpConst = scalbn(APFloat::getOne(Sem), ExpVal,
3739 Value *Fabs = Builder.CreateFAbs(X);
3740 return new FCmpInst(NewPred, Fabs,
3741 ConstantFP::get(X->getType(), CmpConst));
3742 }
3743 }
3744 }
3745 }
3746
3747 // (extractval ([s/u]subo X, Y), 0) == 0 --> X == Y
3748 // (extractval ([s/u]subo X, Y), 0) != 0 --> X != Y
3749 // TODO: This checks one-use, but that is not strictly necessary.
3750 Value *Cmp0 = Cmp.getOperand(0);
3751 Value *X, *Y;
3752 if (C->isZero() && Cmp.isEquality() && Cmp0->hasOneUse() &&
3753 (match(Cmp0,
3755 m_Value(X), m_Value(Y)))) ||
3756 match(Cmp0,
3758 m_Value(X), m_Value(Y))))))
3759 return new ICmpInst(Cmp.getPredicate(), X, Y);
3760 }
3761
3762 if (match(Cmp.getOperand(1), m_APIntAllowPoison(C)))
3764
3765 return nullptr;
3766}
3767
3768/// Fold an icmp equality instruction with binary operator LHS and constant RHS:
3769/// icmp eq/ne BO, C.
3771 ICmpInst &Cmp, BinaryOperator *BO, const APInt &C) {
3772 // TODO: Some of these folds could work with arbitrary constants, but this
3773 // function is limited to scalar and vector splat constants.
3774 if (!Cmp.isEquality())
3775 return nullptr;
3776
3777 ICmpInst::Predicate Pred = Cmp.getPredicate();
3778 bool isICMP_NE = Pred == ICmpInst::ICMP_NE;
3779 Constant *RHS = cast<Constant>(Cmp.getOperand(1));
3780 Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1);
3781
3782 switch (BO->getOpcode()) {
3783 case Instruction::SRem:
3784 // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one.
3785 if (C.isZero() && BO->hasOneUse()) {
3786 const APInt *BOC;
3787 if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) {
3788 Value *NewRem = Builder.CreateURem(BOp0, BOp1, BO->getName());
3789 return new ICmpInst(Pred, NewRem,
3791 }
3792 }
3793 break;
3794 case Instruction::Add: {
3795 // (A + C2) == C --> A == (C - C2)
3796 // (A + C2) != C --> A != (C - C2)
3797 // TODO: Remove the one-use limitation? See discussion in D58633.
3798 if (Constant *C2 = dyn_cast<Constant>(BOp1)) {
3799 if (BO->hasOneUse())
3800 return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(RHS, C2));
3801 } else if (C.isZero()) {
3802 // Replace ((add A, B) != 0) with (A != -B) if A or B is
3803 // efficiently invertible, or if the add has just this one use.
3804 if (Value *NegVal = dyn_castNegVal(BOp1))
3805 return new ICmpInst(Pred, BOp0, NegVal);
3806 if (Value *NegVal = dyn_castNegVal(BOp0))
3807 return new ICmpInst(Pred, NegVal, BOp1);
3808 if (BO->hasOneUse()) {
3809 // (add nuw A, B) != 0 -> (or A, B) != 0
3810 if (match(BO, m_NUWAdd(m_Value(), m_Value()))) {
3811 Value *Or = Builder.CreateOr(BOp0, BOp1);
3812 return new ICmpInst(Pred, Or, Constant::getNullValue(BO->getType()));
3813 }
3814 Value *Neg = Builder.CreateNeg(BOp1);
3815 Neg->takeName(BO);
3816 return new ICmpInst(Pred, BOp0, Neg);
3817 }
3818 }
3819 break;
3820 }
3821 case Instruction::Xor:
3822 if (Constant *BOC = dyn_cast<Constant>(BOp1)) {
3823 // For the xor case, we can xor two constants together, eliminating
3824 // the explicit xor.
3825 return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC));
3826 } else if (C.isZero()) {
3827 // Replace ((xor A, B) != 0) with (A != B)
3828 return new ICmpInst(Pred, BOp0, BOp1);
3829 }
3830 break;
3831 case Instruction::Or: {
3832 const APInt *BOC;
3833 if (match(BOp1, m_APInt(BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) {
3834 // Comparing if all bits outside of a constant mask are set?
3835 // Replace (X | C) == -1 with (X & ~C) == ~C.
3836 // This removes the -1 constant.
3838 Value *And = Builder.CreateAnd(BOp0, NotBOC);
3839 return new ICmpInst(Pred, And, NotBOC);
3840 }
3841 // (icmp eq (or (select cond, 0, NonZero), Other), 0)
3842 // -> (and cond, (icmp eq Other, 0))
3843 // (icmp ne (or (select cond, NonZero, 0), Other), 0)
3844 // -> (or cond, (icmp ne Other, 0))
3845 Value *Cond, *TV, *FV, *Other, *Sel;
3846 if (C.isZero() &&
3847 match(BO,
3850 m_Value(FV))),
3851 m_Value(Other)))) &&
3852 Cond->getType() == Cmp.getType()) {
3853 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
3854 // Easy case is if eq/ne matches whether 0 is trueval/falseval.
3855 if (Pred == ICmpInst::ICMP_EQ
3856 ? (match(TV, m_Zero()) && isKnownNonZero(FV, Q))
3857 : (match(FV, m_Zero()) && isKnownNonZero(TV, Q))) {
3858 Value *Cmp = Builder.CreateICmp(
3859 Pred, Other, Constant::getNullValue(Other->getType()));
3861 Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or, Cmp,
3862 Cond);
3863 }
3864 // Harder case is if eq/ne matches whether 0 is falseval/trueval. In this
3865 // case we need to invert the select condition so we need to be careful to
3866 // avoid creating extra instructions.
3867 // (icmp ne (or (select cond, 0, NonZero), Other), 0)
3868 // -> (or (not cond), (icmp ne Other, 0))
3869 // (icmp eq (or (select cond, NonZero, 0), Other), 0)
3870 // -> (and (not cond), (icmp eq Other, 0))
3871 //
3872 // Only do this if the inner select has one use, in which case we are
3873 // replacing `select` with `(not cond)`. Otherwise, we will create more
3874 // uses. NB: Trying to freely invert cond doesn't make sense here, as if
3875 // cond was freely invertable, the select arms would have been inverted.
3876 if (Sel->hasOneUse() &&
3877 (Pred == ICmpInst::ICMP_EQ
3878 ? (match(FV, m_Zero()) && isKnownNonZero(TV, Q))
3879 : (match(TV, m_Zero()) && isKnownNonZero(FV, Q)))) {
3880 Value *NotCond = Builder.CreateNot(Cond);
3881 Value *Cmp = Builder.CreateICmp(
3882 Pred, Other, Constant::getNullValue(Other->getType()));
3884 Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or, Cmp,
3885 NotCond);
3886 }
3887 }
3888 break;
3889 }
3890 case Instruction::UDiv:
3891 case Instruction::SDiv:
3892 if (BO->isExact()) {
3893 // div exact X, Y eq/ne 0 -> X eq/ne 0
3894 // div exact X, Y eq/ne 1 -> X eq/ne Y
3895 // div exact X, Y eq/ne C ->
3896 // if Y * C never-overflow && OneUse:
3897 // -> Y * C eq/ne X
3898 if (C.isZero())
3899 return new ICmpInst(Pred, BOp0, Constant::getNullValue(BO->getType()));
3900 else if (C.isOne())
3901 return new ICmpInst(Pred, BOp0, BOp1);
3902 else if (BO->hasOneUse()) {
3904 Instruction::Mul, BO->getOpcode() == Instruction::SDiv, BOp1,
3905 Cmp.getOperand(1), BO);
3907 Value *YC =
3908 Builder.CreateMul(BOp1, ConstantInt::get(BO->getType(), C));
3909 return new ICmpInst(Pred, YC, BOp0);
3910 }
3911 }
3912 }
3913 if (BO->getOpcode() == Instruction::UDiv && C.isZero()) {
3914 // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A)
3915 auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT;
3916 return new ICmpInst(NewPred, BOp1, BOp0);
3917 }
3918 break;
3919 default:
3920 break;
3921 }
3922 return nullptr;
3923}
3924
3926 const APInt &CRhs,
3927 InstCombiner::BuilderTy &Builder,
3928 const SimplifyQuery &Q) {
3929 assert(CtpopLhs->getIntrinsicID() == Intrinsic::ctpop &&
3930 "Non-ctpop intrin in ctpop fold");
3931 if (!CtpopLhs->hasOneUse())
3932 return nullptr;
3933
3934 // Power of 2 test:
3935 // isPow2OrZero : ctpop(X) u< 2
3936 // isPow2 : ctpop(X) == 1
3937 // NotPow2OrZero: ctpop(X) u> 1
3938 // NotPow2 : ctpop(X) != 1
3939 // If we know any bit of X can be folded to:
3940 // IsPow2 : X & (~Bit) == 0
3941 // NotPow2 : X & (~Bit) != 0
3942 const ICmpInst::Predicate Pred = I.getPredicate();
3943 if (((I.isEquality() || Pred == ICmpInst::ICMP_UGT) && CRhs == 1) ||
3944 (Pred == ICmpInst::ICMP_ULT && CRhs == 2)) {
3945 Value *Op = CtpopLhs->getArgOperand(0);
3946 KnownBits OpKnown = computeKnownBits(Op, Q.DL, Q.AC, Q.CtxI, Q.DT);
3947 // No need to check for count > 1, that should be already constant folded.
3948 if (OpKnown.countMinPopulation() == 1) {
3949 Value *And = Builder.CreateAnd(
3950 Op, Constant::getIntegerValue(Op->getType(), ~(OpKnown.One)));
3951 return new ICmpInst(
3955 And, Constant::getNullValue(Op->getType()));
3956 }
3957 }
3958
3959 return nullptr;
3960}
3961
3962/// Fold an equality icmp with LLVM intrinsic and constant operand.
3964 ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) {
3965 Type *Ty = II->getType();
3966 unsigned BitWidth = C.getBitWidth();
3967 const ICmpInst::Predicate Pred = Cmp.getPredicate();
3968
3969 switch (II->getIntrinsicID()) {
3970 case Intrinsic::abs:
3971 // abs(A) == 0 -> A == 0
3972 // abs(A) == INT_MIN -> A == INT_MIN
3973 if (C.isZero() || C.isMinSignedValue())
3974 return new ICmpInst(Pred, II->getArgOperand(0), ConstantInt::get(Ty, C));
3975 break;
3976
3977 case Intrinsic::bswap:
3978 // bswap(A) == C -> A == bswap(C)
3979 return new ICmpInst(Pred, II->getArgOperand(0),
3980 ConstantInt::get(Ty, C.byteSwap()));
3981
3982 case Intrinsic::bitreverse:
3983 // bitreverse(A) == C -> A == bitreverse(C)
3984 return new ICmpInst(Pred, II->getArgOperand(0),
3985 ConstantInt::get(Ty, C.reverseBits()));
3986
3987 case Intrinsic::ctlz:
3988 case Intrinsic::cttz: {
3989 // ctz(A) == bitwidth(A) -> A == 0 and likewise for !=
3990 if (C == BitWidth)
3991 return new ICmpInst(Pred, II->getArgOperand(0),
3993
3994 // ctz(A) == C -> A & Mask1 == Mask2, where Mask2 only has bit C set
3995 // and Mask1 has bits 0..C+1 set. Similar for ctl, but for high bits.
3996 // Limit to one use to ensure we don't increase instruction count.
3997 unsigned Num = C.getLimitedValue(BitWidth);
3998 if (Num != BitWidth && II->hasOneUse()) {
3999 bool IsTrailing = II->getIntrinsicID() == Intrinsic::cttz;
4000 APInt Mask1 = IsTrailing ? APInt::getLowBitsSet(BitWidth, Num + 1)
4001 : APInt::getHighBitsSet(BitWidth, Num + 1);
4002 APInt Mask2 = IsTrailing
4005 return new ICmpInst(Pred, Builder.CreateAnd(II->getArgOperand(0), Mask1),
4006 ConstantInt::get(Ty, Mask2));
4007 }
4008 break;
4009 }
4010
4011 case Intrinsic::ctpop: {
4012 // popcount(A) == 0 -> A == 0 and likewise for !=
4013 // popcount(A) == bitwidth(A) -> A == -1 and likewise for !=
4014 bool IsZero = C.isZero();
4015 if (IsZero || C == BitWidth)
4016 return new ICmpInst(Pred, II->getArgOperand(0),
4017 IsZero ? Constant::getNullValue(Ty)
4019
4020 break;
4021 }
4022
4023 case Intrinsic::fshl:
4024 case Intrinsic::fshr:
4025 if (II->getArgOperand(0) == II->getArgOperand(1)) {
4026 const APInt *RotAmtC;
4027 // ror(X, RotAmtC) == C --> X == rol(C, RotAmtC)
4028 // rol(X, RotAmtC) == C --> X == ror(C, RotAmtC)
4029 if (match(II->getArgOperand(2), m_APInt(RotAmtC)))
4030 return new ICmpInst(Pred, II->getArgOperand(0),
4031 II->getIntrinsicID() == Intrinsic::fshl
4032 ? ConstantInt::get(Ty, C.rotr(*RotAmtC))
4033 : ConstantInt::get(Ty, C.rotl(*RotAmtC)));
4034 }
4035 break;
4036
4037 case Intrinsic::umax:
4038 case Intrinsic::uadd_sat: {
4039 // uadd.sat(a, b) == 0 -> (a | b) == 0
4040 // umax(a, b) == 0 -> (a | b) == 0
4041 if (C.isZero() && II->hasOneUse()) {
4042 Value *Or = Builder.CreateOr(II->getArgOperand(0), II->getArgOperand(1));
4043 return new ICmpInst(Pred, Or, Constant::getNullValue(Ty));
4044 }
4045 break;
4046 }
4047
4048 case Intrinsic::ssub_sat:
4049 // ssub.sat(a, b) == 0 -> a == b
4050 //
4051 // Note this doesn't work for ssub.sat.i1 because ssub.sat.i1 0, -1 = 0
4052 // (because 1 saturates to 0). Just skip the optimization for i1.
4053 if (C.isZero() && II->getType()->getScalarSizeInBits() > 1)
4054 return new ICmpInst(Pred, II->getArgOperand(0), II->getArgOperand(1));
4055 break;
4056 case Intrinsic::usub_sat: {
4057 // usub.sat(a, b) == 0 -> a <= b
4058 if (C.isZero()) {
4059 ICmpInst::Predicate NewPred =
4061 return new ICmpInst(NewPred, II->getArgOperand(0), II->getArgOperand(1));
4062 }
4063 break;
4064 }
4065 default:
4066 break;
4067 }
4068
4069 return nullptr;
4070}
4071
4072/// Fold an icmp with LLVM intrinsics
4073static Instruction *
4075 InstCombiner::BuilderTy &Builder) {
4076 assert(Cmp.isEquality());
4077
4078 ICmpInst::Predicate Pred = Cmp.getPredicate();
4079 Value *Op0 = Cmp.getOperand(0);
4080 Value *Op1 = Cmp.getOperand(1);
4081 const auto *IIOp0 = dyn_cast<IntrinsicInst>(Op0);
4082 const auto *IIOp1 = dyn_cast<IntrinsicInst>(Op1);
4083 if (!IIOp0 || !IIOp1 || IIOp0->getIntrinsicID() != IIOp1->getIntrinsicID())
4084 return nullptr;
4085
4086 switch (IIOp0->getIntrinsicID()) {
4087 case Intrinsic::bswap:
4088 case Intrinsic::bitreverse:
4089 // If both operands are byte-swapped or bit-reversed, just compare the
4090 // original values.
4091 return new ICmpInst(Pred, IIOp0->getOperand(0), IIOp1->getOperand(0));
4092 case Intrinsic::fshl:
4093 case Intrinsic::fshr: {
4094 // If both operands are rotated by same amount, just compare the
4095 // original values.
4096 if (IIOp0->getOperand(0) != IIOp0->getOperand(1))
4097 break;
4098 if (IIOp1->getOperand(0) != IIOp1->getOperand(1))
4099 break;
4100 if (IIOp0->getOperand(2) == IIOp1->getOperand(2))
4101 return new ICmpInst(Pred, IIOp0->getOperand(0), IIOp1->getOperand(0));
4102
4103 // rotate(X, AmtX) == rotate(Y, AmtY)
4104 // -> rotate(X, AmtX - AmtY) == Y
4105 // Do this if either both rotates have one use or if only one has one use
4106 // and AmtX/AmtY are constants.
4107 const unsigned BW = IIOp0->getType()->getScalarSizeInBits();
4108 unsigned OneUses = IIOp0->hasOneUse() + IIOp1->hasOneUse();
4109 if (OneUses == 2 ||
4110 (OneUses == 1 && match(IIOp0->getOperand(2), m_ImmConstant()) &&
4111 match(IIOp1->getOperand(2), m_ImmConstant()))) {
4112
4113 // Only valid assuming (2**BW) % BW == 0, which only holds for powers
4114 // of two.
4115 if (isPowerOf2_32(BW)) {
4116 Value *SubAmt =
4117 Builder.CreateSub(IIOp0->getOperand(2), IIOp1->getOperand(2));
4118 Value *CombinedRotate = Builder.CreateIntrinsic(
4119 Op0->getType(), IIOp0->getIntrinsicID(),
4120 {IIOp0->getOperand(0), IIOp0->getOperand(0), SubAmt});
4121 return new ICmpInst(Pred, IIOp1->getOperand(0), CombinedRotate);
4122 }
4123 }
4124 } break;
4125 default:
4126 break;
4127 }
4128
4129 return nullptr;
4130}
4131
4132/// Try to fold integer comparisons with a constant operand: icmp Pred X, C
4133/// where X is some kind of instruction and C is AllowPoison.
4134/// TODO: Move more folds which allow poison to this function.
4137 const APInt &C) {
4138 const ICmpInst::Predicate Pred = Cmp.getPredicate();
4139 if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0))) {
4140 switch (II->getIntrinsicID()) {
4141 default:
4142 break;
4143 case Intrinsic::fshl:
4144 case Intrinsic::fshr:
4145 if (Cmp.isEquality() && II->getArgOperand(0) == II->getArgOperand(1)) {
4146 // (rot X, ?) == 0/-1 --> X == 0/-1
4147 if (C.isZero() || C.isAllOnes())
4148 return new ICmpInst(Pred, II->getArgOperand(0), Cmp.getOperand(1));
4149 }
4150 break;
4151 }
4152 }
4153
4154 return nullptr;
4155}
4156
4157/// Fold an icmp with BinaryOp and constant operand: icmp Pred BO, C.
4159 BinaryOperator *BO,
4160 const APInt &C) {
4161 switch (BO->getOpcode()) {
4162 case Instruction::Xor:
4163 if (Instruction *I = foldICmpXorConstant(Cmp, BO, C))
4164 return I;
4165 break;
4166 case Instruction::And:
4167 if (Instruction *I = foldICmpAndConstant(Cmp, BO, C))
4168 return I;
4169 break;
4170 case Instruction::Or:
4171 if (Instruction *I = foldICmpOrConstant(Cmp, BO, C))
4172 return I;
4173 break;
4174 case Instruction::Mul:
4175 if (Instruction *I = foldICmpMulConstant(Cmp, BO, C))
4176 return I;
4177 break;
4178 case Instruction::Shl:
4179 if (Instruction *I = foldICmpShlConstant(Cmp, BO, C))
4180 return I;
4181 break;
4182 case Instruction::LShr:
4183 case Instruction::AShr:
4184 if (Instruction *I = foldICmpShrConstant(Cmp, BO, C))
4185 return I;
4186 break;
4187 case Instruction::SRem:
4188 if (Instruction *I = foldICmpSRemConstant(Cmp, BO, C))
4189 return I;
4190 break;
4191 case Instruction::UDiv:
4192 if (Instruction *I = foldICmpUDivConstant(Cmp, BO, C))
4193 return I;
4194 [[fallthrough]];
4195 case Instruction::SDiv:
4196 if (Instruction *I = foldICmpDivConstant(Cmp, BO, C))
4197 return I;
4198 break;
4199 case Instruction::Sub:
4200 if (Instruction *I = foldICmpSubConstant(Cmp, BO, C))
4201 return I;
4202 break;
4203 case Instruction::Add:
4204 if (Instruction *I = foldICmpAddConstant(Cmp, BO, C))
4205 return I;
4206 break;
4207 default:
4208 break;
4209 }
4210
4211 // TODO: These folds could be refactored to be part of the above calls.
4213 return I;
4214
4215 // Fall back to handling `icmp pred (select A ? C1 : C2) binop (select B ? C3
4216 // : C4), C5` pattern, by computing a truth table of the four constant
4217 // variants.
4219}
4220
4221static Instruction *
4223 const APInt &C,
4224 InstCombiner::BuilderTy &Builder) {
4225 // This transform may end up producing more than one instruction for the
4226 // intrinsic, so limit it to one user of the intrinsic.
4227 if (!II->hasOneUse())
4228 return nullptr;
4229
4230 // Let Y = [add/sub]_sat(X, C) pred C2
4231 // SatVal = The saturating value for the operation
4232 // WillWrap = Whether or not the operation will underflow / overflow
4233 // => Y = (WillWrap ? SatVal : (X binop C)) pred C2
4234 // => Y = WillWrap ? (SatVal pred C2) : ((X binop C) pred C2)
4235 //
4236 // When (SatVal pred C2) is true, then
4237 // Y = WillWrap ? true : ((X binop C) pred C2)
4238 // => Y = WillWrap || ((X binop C) pred C2)
4239 // else
4240 // Y = WillWrap ? false : ((X binop C) pred C2)
4241 // => Y = !WillWrap ? ((X binop C) pred C2) : false
4242 // => Y = !WillWrap && ((X binop C) pred C2)
4243 Value *Op0 = II->getOperand(0);
4244 Value *Op1 = II->getOperand(1);
4245
4246 const APInt *COp1;
4247 // This transform only works when the intrinsic has an integral constant or
4248 // splat vector as the second operand.
4249 if (!match(Op1, m_APInt(COp1)))
4250 return nullptr;
4251
4252 APInt SatVal;
4253 switch (II->getIntrinsicID()) {
4254 default:
4256 "This function only works with usub_sat and uadd_sat for now!");
4257 case Intrinsic::uadd_sat:
4258 SatVal = APInt::getAllOnes(C.getBitWidth());
4259 break;
4260 case Intrinsic::usub_sat:
4261 SatVal = APInt::getZero(C.getBitWidth());
4262 break;
4263 }
4264
4265 // Check (SatVal pred C2)
4266 bool SatValCheck = ICmpInst::compare(SatVal, C, Pred);
4267
4268 // !WillWrap.
4270 II->getBinaryOp(), *COp1, II->getNoWrapKind());
4271
4272 // WillWrap.
4273 if (SatValCheck)
4274 C1 = C1.inverse();
4275
4277 if (II->getBinaryOp() == Instruction::Add)
4278 C2 = C2.sub(*COp1);
4279 else
4280 C2 = C2.add(*COp1);
4281
4282 Instruction::BinaryOps CombiningOp =
4283 SatValCheck ? Instruction::BinaryOps::Or : Instruction::BinaryOps::And;
4284
4285 std::optional<ConstantRange> Combination;
4286 if (CombiningOp == Instruction::BinaryOps::Or)
4287 Combination = C1.exactUnionWith(C2);
4288 else /* CombiningOp == Instruction::BinaryOps::And */
4289 Combination = C1.exactIntersectWith(C2);
4290
4291 if (!Combination)
4292 return nullptr;
4293
4294 CmpInst::Predicate EquivPred;
4295 APInt EquivInt;
4296 APInt EquivOffset;
4297
4298 Combination->getEquivalentICmp(EquivPred, EquivInt, EquivOffset);
4299
4300 return new ICmpInst(
4301 EquivPred,
4302 Builder.CreateAdd(Op0, ConstantInt::get(Op1->getType(), EquivOffset)),
4303 ConstantInt::get(Op1->getType(), EquivInt));
4304}
4305
4306static Instruction *
4308 const APInt &C,
4309 InstCombiner::BuilderTy &Builder) {
4310 std::optional<ICmpInst::Predicate> NewPredicate = std::nullopt;
4311 switch (Pred) {
4312 case ICmpInst::ICMP_EQ:
4313 case ICmpInst::ICMP_NE:
4314 if (C.isZero())
4315 NewPredicate = Pred;
4316 else if (C.isOne())
4317 NewPredicate =
4319 else if (C.isAllOnes())
4320 NewPredicate =
4322 break;
4323
4324 case ICmpInst::ICMP_SGT:
4325 if (C.isAllOnes())
4326 NewPredicate = ICmpInst::ICMP_UGE;
4327 else if (C.isZero())
4328 NewPredicate = ICmpInst::ICMP_UGT;
4329 break;
4330
4331 case ICmpInst::ICMP_SLT:
4332 if (C.isZero())
4333 NewPredicate = ICmpInst::ICMP_ULT;
4334 else if (C.isOne())
4335 NewPredicate = ICmpInst::ICMP_ULE;
4336 break;
4337
4338 case ICmpInst::ICMP_ULT:
4339 if (C.ugt(1))
4340 NewPredicate = ICmpInst::ICMP_UGE;
4341 break;
4342
4343 case ICmpInst::ICMP_UGT:
4344 if (!C.isZero() && !C.isAllOnes())
4345 NewPredicate = ICmpInst::ICMP_ULT;
4346 break;
4347
4348 default:
4349 break;
4350 }
4351
4352 if (!NewPredicate)
4353 return nullptr;
4354
4355 if (I->getIntrinsicID() == Intrinsic::scmp)
4356 NewPredicate = ICmpInst::getSignedPredicate(*NewPredicate);
4357 Value *LHS = I->getOperand(0);
4358 Value *RHS = I->getOperand(1);
4359 return new ICmpInst(*NewPredicate, LHS, RHS);
4360}
4361
4362/// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C.
4365 const APInt &C) {
4366 ICmpInst::Predicate Pred = Cmp.getPredicate();
4367
4368 // Handle folds that apply for any kind of icmp.
4369 switch (II->getIntrinsicID()) {
4370 default:
4371 break;
4372 case Intrinsic::uadd_sat:
4373 case Intrinsic::usub_sat:
4375 Pred, cast<SaturatingInst>(II), C, Builder))
4376 return Folded;
4377 break;
4378 case Intrinsic::ctpop: {
4379 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
4380 if (Instruction *R = foldCtpopPow2Test(Cmp, II, C, Builder, Q))
4381 return R;
4382 } break;
4383 case Intrinsic::scmp:
4384 case Intrinsic::ucmp:
4386 return Folded;
4387 break;
4388 }
4389
4390 if (Cmp.isEquality())
4391 return foldICmpEqIntrinsicWithConstant(Cmp, II, C);
4392
4393 Type *Ty = II->getType();
4394 unsigned BitWidth = C.getBitWidth();
4395 switch (II->getIntrinsicID()) {
4396 case Intrinsic::ctpop: {
4397 // (ctpop X > BitWidth - 1) --> X == -1
4398 Value *X = II->getArgOperand(0);
4399 if (C == BitWidth - 1 && Pred == ICmpInst::ICMP_UGT)
4400 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, X,
4402 // (ctpop X < BitWidth) --> X != -1
4403 if (C == BitWidth && Pred == ICmpInst::ICMP_ULT)
4404 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, X,
4406 break;
4407 }
4408 case Intrinsic::ctlz: {
4409 // ctlz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX < 0b00010000
4410 if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) {
4411 unsigned Num = C.getLimitedValue();
4412 APInt Limit = APInt::getOneBitSet(BitWidth, BitWidth - Num - 1);
4413 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT,
4414 II->getArgOperand(0), ConstantInt::get(Ty, Limit));
4415 }
4416
4417 // ctlz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX > 0b00011111
4418 if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) {
4419 unsigned Num = C.getLimitedValue();
4421 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT,
4422 II->getArgOperand(0), ConstantInt::get(Ty, Limit));
4423 }
4424 break;
4425 }
4426 case Intrinsic::cttz: {
4427 // Limit to one use to ensure we don't increase instruction count.
4428 if (!II->hasOneUse())
4429 return nullptr;
4430
4431 // cttz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX & 0b00001111 == 0
4432 if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) {
4433 APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue() + 1);
4434 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ,
4435 Builder.CreateAnd(II->getArgOperand(0), Mask),
4437 }
4438
4439 // cttz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX & 0b00000111 != 0
4440 if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) {
4441 APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue());
4442 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE,
4443 Builder.CreateAnd(II->getArgOperand(0), Mask),
4445 }
4446 break;
4447 }
4448 case Intrinsic::ssub_sat:
4449 // ssub.sat(a, b) spred 0 -> a spred b
4450 //
4451 // Note this doesn't work for ssub.sat.i1 because ssub.sat.i1 0, -1 = 0
4452 // (because 1 saturates to 0). Just skip the optimization for i1.
4453 if (ICmpInst::isSigned(Pred) && C.getBitWidth() > 1) {
4454 if (C.isZero())
4455 return new ICmpInst(Pred, II->getArgOperand(0), II->getArgOperand(1));
4456 // X s<= 0 is cannonicalized to X s< 1
4457 if (Pred == ICmpInst::ICMP_SLT && C.isOne())
4458 return new ICmpInst(ICmpInst::ICMP_SLE, II->getArgOperand(0),
4459 II->getArgOperand(1));
4460 // X s>= 0 is cannonicalized to X s> -1
4461 if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes())
4462 return new ICmpInst(ICmpInst::ICMP_SGE, II->getArgOperand(0),
4463 II->getArgOperand(1));
4464 }
4465 break;
4466 case Intrinsic::abs: {
4467 if (!II->hasOneUse())
4468 return nullptr;
4469
4470 Value *X = II->getArgOperand(0);
4471
4472 // If C >= 0:
4473 // abs(X) u> C --> X + C u> 2 * C
4474 if (Pred == CmpInst::ICMP_UGT && C.isNonNegative()) {
4475 return new ICmpInst(ICmpInst::ICMP_UGT,
4476 Builder.CreateAdd(X, ConstantInt::get(Ty, C)),
4477 ConstantInt::get(Ty, 2 * C));
4478 }
4479
4480 // If C >= 1:
4481 // abs(X) u< C --> X + (C - 1) u<= 2 * (C - 1)
4482 if (Pred == CmpInst::ICMP_ULT && C.sge(1))
4483 return new ICmpInst(ICmpInst::ICMP_ULE,
4484 Builder.CreateAdd(X, ConstantInt::get(Ty, C - 1)),
4485 ConstantInt::get(Ty, 2 * (C - 1)));
4486
4487 break;
4488 }
4489 default:
4490 break;
4491 }
4492
4493 return nullptr;
4494}
4495
4496/// Handle icmp with constant (but not simple integer constant) RHS.
4498 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
4499 Constant *RHSC = dyn_cast<Constant>(Op1);
4501 if (!RHSC || !LHSI)
4502 return nullptr;
4503
4504 switch (LHSI->getOpcode()) {
4505 // umax(X, Y) == null -> (X == null) && (Y == null).
4506 // umax(X, Y) != null -> (X != null) || (Y != null).
4507 case Instruction::Select: {
4508 CmpInst::Predicate Pred = I.getPredicate();
4509 if (!ICmpInst::isEquality(Pred) || !RHSC->getType()->isPtrOrPtrVectorTy())
4510 break;
4511
4512 // Require zero address bits, the minimum unsigned pointer value.
4513 Constant *RHSIndexBits = ConstantFoldCastOperand(
4514 Instruction::PtrToAddr, RHSC, DL.getIndexType(RHSC->getType()), DL);
4515 if (!RHSIndexBits || !RHSIndexBits->isNullValue())
4516 break;
4517
4518 auto *SI = cast<SelectInst>(LHSI);
4519 // Both the select and its comparison must die to avoid adding instructions.
4520 if (!SI->hasOneUse() || !SI->getCondition()->hasOneUse())
4521 break;
4522
4523 // Require the condition to compare the select operands directly.
4524 Value *A = SI->getTrueValue();
4525 Value *B = SI->getFalseValue();
4526 CmpPredicate SelectPred;
4527 if (!match(SI->getCondition(),
4528 m_c_ICmp(SelectPred, m_Specific(A), m_Specific(B))) ||
4529 (SelectPred != ICmpInst::ICMP_UGT && SelectPred != ICmpInst::ICMP_UGE))
4530 break;
4531
4532 Value *ACmp = Builder.CreateICmp(Pred, A, RHSC);
4533 Value *BCmp = Builder.CreateICmp(Pred, B, RHSC);
4534 // The original icmp propagates poison from both operands; and/or is safe.
4535 return BinaryOperator::Create(Pred == ICmpInst::ICMP_EQ ? Instruction::And
4536 : Instruction::Or,
4537 ACmp, BCmp);
4538 }
4539
4540 case Instruction::IntToPtr:
4541 // icmp pred inttoptr(X), null -> icmp pred X, null pointer value
4542 if (isa<ConstantPointerNull>(RHSC)) {
4543 Type *IntPtrTy = DL.getIntPtrType(RHSC->getType());
4544 if (IntPtrTy == LHSI->getOperand(0)->getType()) {
4545 APInt NullPtrValue =
4546 DL.getNullPtrValue(RHSC->getType()->getPointerAddressSpace());
4547 return new ICmpInst(I.getPredicate(), LHSI->getOperand(0),
4548 Constant::getIntegerValue(IntPtrTy, NullPtrValue));
4549 }
4550 }
4551 break;
4552
4553 case Instruction::Load:
4554 // Try to optimize things like "A[i] > 4" to index computations.
4556 cast<LoadInst>(LHSI), I.getPredicate(), RHSC, I))
4557 return Res;
4558 break;
4559 }
4560
4561 return nullptr;
4562}
4563
4565 Value *RHS, const ICmpInst &I) {
4566 // Try to fold the comparison into the select arms, which will cause the
4567 // select to be converted into a logical and/or.
4568 auto SimplifyOp = [&](Value *Op, bool SelectCondIsTrue) -> Value * {
4569 if (Value *Res = simplifyICmpInst(Pred, Op, RHS, SQ))
4570 return Res;
4571 if (std::optional<bool> Impl = isImpliedCondition(
4572 SI->getCondition(), Pred, Op, RHS, DL, SelectCondIsTrue))
4573 return ConstantInt::get(I.getType(), *Impl);
4574 return nullptr;
4575 };
4576
4577 ConstantInt *CI = nullptr;
4578 Value *Op1 = SimplifyOp(SI->getOperand(1), true);
4579 if (Op1)
4580 CI = dyn_cast<ConstantInt>(Op1);
4581
4582 Value *Op2 = SimplifyOp(SI->getOperand(2), false);
4583 if (Op2)
4584 CI = dyn_cast<ConstantInt>(Op2);
4585
4586 auto Simplifies = [&](Value *Op, unsigned Idx) {
4587 // A comparison of ucmp/scmp with a constant will fold into an icmp.
4588 const APInt *Dummy;
4589 return Op ||
4590 (isa<CmpIntrinsic>(SI->getOperand(Idx)) &&
4591 SI->getOperand(Idx)->hasOneUse() && match(RHS, m_APInt(Dummy)));
4592 };
4593
4594 // We only want to perform this transformation if it will not lead to
4595 // additional code. This is true if either both sides of the select
4596 // fold to a constant (in which case the icmp is replaced with a select
4597 // which will usually simplify) or this is the only user of the
4598 // select (in which case we are trading a select+icmp for a simpler
4599 // select+icmp) or all uses of the select can be replaced based on
4600 // dominance information ("Global cases").
4601 bool Transform = false;
4602 if (Op1 && Op2)
4603 Transform = true;
4604 else if (Simplifies(Op1, 1) || Simplifies(Op2, 2)) {
4605 // Local case
4606 if (SI->hasOneUse())
4607 Transform = true;
4608 // Global cases
4609 else if (CI && !CI->isZero())
4610 // When Op1 is constant try replacing select with second operand.
4611 // Otherwise Op2 is constant and try replacing select with first
4612 // operand.
4613 Transform = replacedSelectWithOperand(SI, &I, Op1 ? 2 : 1);
4614 }
4615 if (Transform) {
4616 if (!Op1)
4617 Op1 = Builder.CreateICmp(Pred, SI->getOperand(1), RHS, I.getName());
4618 if (!Op2)
4619 Op2 = Builder.CreateICmp(Pred, SI->getOperand(2), RHS, I.getName());
4620 return SelectInst::Create(SI->getOperand(0), Op1, Op2, "", nullptr, SI);
4621 }
4622
4623 // Fold icmp eq/ne X, select(icmp pred X, P, C1, C2)
4624 // When the select condition compares X with a constant P and the select
4625 // arms are constants C1/C2, we can fold to a set membership test.
4626 // Example: X == select(X >s 0, 2, 0) -> (X == 2) | (X == 0)
4627 // This is valid when C1 satisfies the condition (C1 >s 0) and C2 does not.
4628 if (ICmpInst::isEquality(Pred)) {
4629 CmpPredicate CondPred;
4630 const APInt *C1, *C2, *P;
4631 if (match(SI,
4632 m_OneUse(m_Select(m_ICmp(CondPred, m_Specific(RHS), m_APInt(P)),
4633 m_APInt(C1), m_APInt(C2))))) {
4634 bool C1SatisfiesCond = ICmpInst::compare(*C1, *P, CondPred);
4635 bool C2SatisfiesCond = ICmpInst::compare(*C2, *P, CondPred);
4636
4637 if (C1SatisfiesCond && !C2SatisfiesCond) {
4638 // X == select(cond, C1, C2) -> (X == C1) | (X == C2)
4639 // X != select(cond, C1, C2) -> (X != C1) & (X != C2)
4640 Value *Cmp1 = Builder.CreateICmp(Pred, RHS, SI->getTrueValue());
4641 Value *Cmp2 = Builder.CreateICmp(Pred, RHS, SI->getFalseValue());
4642 if (Pred == ICmpInst::ICMP_EQ)
4643 return BinaryOperator::CreateOr(Cmp1, Cmp2);
4644 return BinaryOperator::CreateAnd(Cmp1, Cmp2);
4645 }
4646 }
4647 }
4648
4649 return nullptr;
4650}
4651
4652// Returns whether V is a Mask ((X + 1) & X == 0) or ~Mask (-Pow2OrZero)
4653static bool isMaskOrZero(const Value *V, bool Not, const SimplifyQuery &Q,
4654 unsigned Depth = 0) {
4656 return true;
4657 if (V->getType()->getScalarSizeInBits() == 1)
4658 return true;
4660 return false;
4661 Value *X;
4663 if (!I)
4664 return false;
4665 switch (I->getOpcode()) {
4666 case Instruction::ZExt:
4667 // ZExt(Mask) is a Mask.
4668 return !Not && isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4669 case Instruction::SExt:
4670 // SExt(Mask) is a Mask.
4671 // SExt(~Mask) is a ~Mask.
4672 return isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4673 case Instruction::And:
4674 case Instruction::Or:
4675 // Mask0 | Mask1 is a Mask.
4676 // Mask0 & Mask1 is a Mask.
4677 // ~Mask0 | ~Mask1 is a ~Mask.
4678 // ~Mask0 & ~Mask1 is a ~Mask.
4679 return isMaskOrZero(I->getOperand(1), Not, Q, Depth) &&
4680 isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4681 case Instruction::Xor:
4682 if (match(V, m_Not(m_Value(X))))
4683 return isMaskOrZero(X, !Not, Q, Depth);
4684
4685 // (X ^ -X) is a ~Mask
4686 if (Not)
4687 return match(V, m_c_Xor(m_Value(X), m_Neg(m_Deferred(X))));
4688 // (X ^ (X - 1)) is a Mask
4689 else
4690 return match(V, m_c_Xor(m_Value(X), m_Add(m_Deferred(X), m_AllOnes())));
4691 case Instruction::Select:
4692 // c ? Mask0 : Mask1 is a Mask.
4693 return isMaskOrZero(I->getOperand(1), Not, Q, Depth) &&
4694 isMaskOrZero(I->getOperand(2), Not, Q, Depth);
4695 case Instruction::Shl:
4696 // (~Mask) << X is a ~Mask.
4697 return Not && isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4698 case Instruction::LShr:
4699 // Mask >> X is a Mask.
4700 return !Not && isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4701 case Instruction::AShr:
4702 // Mask s>> X is a Mask.
4703 // ~Mask s>> X is a ~Mask.
4704 return isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4705 case Instruction::Add:
4706 // Pow2 - 1 is a Mask.
4707 if (!Not && match(I->getOperand(1), m_AllOnes()))
4708 return isKnownToBeAPowerOfTwo(I->getOperand(0), Q.DL, /*OrZero*/ true,
4709 Q.AC, Q.CtxI, Q.DT, Depth);
4710 break;
4711 case Instruction::Sub:
4712 // -Pow2 is a ~Mask.
4713 if (Not && match(I->getOperand(0), m_Zero()))
4714 return isKnownToBeAPowerOfTwo(I->getOperand(1), Q.DL, /*OrZero*/ true,
4715 Q.AC, Q.CtxI, Q.DT, Depth);
4716 break;
4717 case Instruction::Call: {
4718 if (auto *II = dyn_cast<IntrinsicInst>(I)) {
4719 switch (II->getIntrinsicID()) {
4720 // min/max(Mask0, Mask1) is a Mask.
4721 // min/max(~Mask0, ~Mask1) is a ~Mask.
4722 case Intrinsic::umax:
4723 case Intrinsic::smax:
4724 case Intrinsic::umin:
4725 case Intrinsic::smin:
4726 return isMaskOrZero(II->getArgOperand(1), Not, Q, Depth) &&
4727 isMaskOrZero(II->getArgOperand(0), Not, Q, Depth);
4728
4729 // In the context of masks, bitreverse(Mask) == ~Mask
4730 case Intrinsic::bitreverse:
4731 return isMaskOrZero(II->getArgOperand(0), !Not, Q, Depth);
4732 default:
4733 break;
4734 }
4735 }
4736 break;
4737 }
4738 default:
4739 break;
4740 }
4741 return false;
4742}
4743
4744/// Some comparisons can be simplified.
4745/// In this case, we are looking for comparisons that look like
4746/// a check for a lossy truncation.
4747/// Folds:
4748/// icmp SrcPred (x & Mask), x to icmp DstPred x, Mask
4749/// icmp SrcPred (x & ~Mask), ~Mask to icmp DstPred x, ~Mask
4750/// icmp eq/ne (x & ~Mask), 0 to icmp DstPred x, Mask
4751/// icmp eq/ne (~x | Mask), -1 to icmp DstPred x, Mask
4752/// Where Mask is some pattern that produces all-ones in low bits:
4753/// (-1 >> y)
4754/// ((-1 << y) >> y) <- non-canonical, has extra uses
4755/// ~(-1 << y)
4756/// ((1 << y) + (-1)) <- non-canonical, has extra uses
4757/// The Mask can be a constant, too.
4758/// For some predicates, the operands are commutative.
4759/// For others, x can only be on a specific side.
4761 Value *Op1, const SimplifyQuery &Q,
4762 InstCombiner &IC) {
4763
4764 ICmpInst::Predicate DstPred;
4765 switch (Pred) {
4767 // x & Mask == x
4768 // x & ~Mask == 0
4769 // ~x | Mask == -1
4770 // -> x u<= Mask
4771 // x & ~Mask == ~Mask
4772 // -> ~Mask u<= x
4774 break;
4776 // x & Mask != x
4777 // x & ~Mask != 0
4778 // ~x | Mask != -1
4779 // -> x u> Mask
4780 // x & ~Mask != ~Mask
4781 // -> ~Mask u> x
4783 break;
4785 // x & Mask u< x
4786 // -> x u> Mask
4787 // x & ~Mask u< ~Mask
4788 // -> ~Mask u> x
4790 break;
4792 // x & Mask u>= x
4793 // -> x u<= Mask
4794 // x & ~Mask u>= ~Mask
4795 // -> ~Mask u<= x
4797 break;
4799 // x & Mask s< x [iff Mask s>= 0]
4800 // -> x s> Mask
4801 // x & ~Mask s< ~Mask [iff ~Mask != 0]
4802 // -> ~Mask s> x
4804 break;
4806 // x & Mask s>= x [iff Mask s>= 0]
4807 // -> x s<= Mask
4808 // x & ~Mask s>= ~Mask [iff ~Mask != 0]
4809 // -> ~Mask s<= x
4811 break;
4812 default:
4813 // We don't support sgt,sle
4814 // ult/ugt are simplified to true/false respectively.
4815 return nullptr;
4816 }
4817
4818 Value *X, *M;
4819 // Put search code in lambda for early positive returns.
4820 auto IsLowBitMask = [&]() {
4821 if (match(Op0, m_c_And(m_Specific(Op1), m_Value(M)))) {
4822 X = Op1;
4823 // Look for: x & Mask pred x
4824 if (isMaskOrZero(M, /*Not=*/false, Q)) {
4825 return !ICmpInst::isSigned(Pred) ||
4826 (match(M, m_NonNegative()) || isKnownNonNegative(M, Q));
4827 }
4828
4829 // Look for: x & ~Mask pred ~Mask
4830 if (isMaskOrZero(X, /*Not=*/true, Q)) {
4831 return !ICmpInst::isSigned(Pred) || isKnownNonZero(X, Q);
4832 }
4833 return false;
4834 }
4835 if (ICmpInst::isEquality(Pred) && match(Op1, m_AllOnes()) &&
4836 match(Op0, m_OneUse(m_Or(m_Value(X), m_Value(M))))) {
4837
4838 auto Check = [&]() {
4839 // Look for: ~x | Mask == -1
4840 if (isMaskOrZero(M, /*Not=*/false, Q)) {
4841 if (Value *NotX =
4842 IC.getFreelyInverted(X, X->hasOneUse(), &IC.Builder)) {
4843 X = NotX;
4844 return true;
4845 }
4846 }
4847 return false;
4848 };
4849 if (Check())
4850 return true;
4851 std::swap(X, M);
4852 return Check();
4853 }
4854 if (ICmpInst::isEquality(Pred) && match(Op1, m_Zero()) &&
4855 match(Op0, m_OneUse(m_And(m_Value(X), m_Value(M))))) {
4856 auto Check = [&]() {
4857 // Look for: x & ~Mask == 0
4858 if (isMaskOrZero(M, /*Not=*/true, Q)) {
4859 if (Value *NotM =
4860 IC.getFreelyInverted(M, M->hasOneUse(), &IC.Builder)) {
4861 M = NotM;
4862 return true;
4863 }
4864 }
4865 return false;
4866 };
4867 if (Check())
4868 return true;
4869 std::swap(X, M);
4870 return Check();
4871 }
4872 return false;
4873 };
4874
4875 if (!IsLowBitMask())
4876 return nullptr;
4877
4878 return IC.Builder.CreateICmp(DstPred, X, M);
4879}
4880
4881/// Some comparisons can be simplified.
4882/// In this case, we are looking for comparisons that look like
4883/// a check for a lossy signed truncation.
4884/// Folds: (MaskedBits is a constant.)
4885/// ((%x << MaskedBits) a>> MaskedBits) SrcPred %x
4886/// Into:
4887/// (add %x, (1 << (KeptBits-1))) DstPred (1 << KeptBits)
4888/// Where KeptBits = bitwidth(%x) - MaskedBits
4889static Value *
4891 InstCombiner::BuilderTy &Builder) {
4892 CmpPredicate SrcPred;
4893 Value *X;
4894 const APInt *C0, *C1; // FIXME: non-splats, potentially with undef.
4895 // We are ok with 'shl' having multiple uses, but 'ashr' must be one-use.
4896 if (!match(&I, m_c_ICmp(SrcPred,
4898 m_APInt(C1))),
4899 m_Deferred(X))))
4900 return nullptr;
4901
4902 // Potential handling of non-splats: for each element:
4903 // * if both are undef, replace with constant 0.
4904 // Because (1<<0) is OK and is 1, and ((1<<0)>>1) is also OK and is 0.
4905 // * if both are not undef, and are different, bailout.
4906 // * else, only one is undef, then pick the non-undef one.
4907
4908 // The shift amount must be equal.
4909 if (*C0 != *C1)
4910 return nullptr;
4911 const APInt &MaskedBits = *C0;
4912 assert(MaskedBits != 0 && "shift by zero should be folded away already.");
4913
4914 ICmpInst::Predicate DstPred;
4915 switch (SrcPred) {
4917 // ((%x << MaskedBits) a>> MaskedBits) == %x
4918 // =>
4919 // (add %x, (1 << (KeptBits-1))) u< (1 << KeptBits)
4921 break;
4923 // ((%x << MaskedBits) a>> MaskedBits) != %x
4924 // =>
4925 // (add %x, (1 << (KeptBits-1))) u>= (1 << KeptBits)
4927 break;
4928 // FIXME: are more folds possible?
4929 default:
4930 return nullptr;
4931 }
4932
4933 auto *XType = X->getType();
4934 const unsigned XBitWidth = XType->getScalarSizeInBits();
4935 const APInt BitWidth = APInt(XBitWidth, XBitWidth);
4936 assert(BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched");
4937
4938 // KeptBits = bitwidth(%x) - MaskedBits
4939 const APInt KeptBits = BitWidth - MaskedBits;
4940 assert(KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable");
4941 // ICmpCst = (1 << KeptBits)
4942 const APInt ICmpCst = APInt(XBitWidth, 1).shl(KeptBits);
4943 assert(ICmpCst.isPowerOf2());
4944 // AddCst = (1 << (KeptBits-1))
4945 const APInt AddCst = ICmpCst.lshr(1);
4946 assert(AddCst.ult(ICmpCst) && AddCst.isPowerOf2());
4947
4948 // T0 = add %x, AddCst
4949 Value *T0 = Builder.CreateAdd(X, ConstantInt::get(XType, AddCst));
4950 // T1 = T0 DstPred ICmpCst
4951 Value *T1 = Builder.CreateICmp(DstPred, T0, ConstantInt::get(XType, ICmpCst));
4952
4953 return T1;
4954}
4955
4956// Given pattern:
4957// icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
4958// we should move shifts to the same hand of 'and', i.e. rewrite as
4959// icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x)
4960// We are only interested in opposite logical shifts here.
4961// One of the shifts can be truncated.
4962// If we can, we want to end up creating 'lshr' shift.
4963static Value *
4965 InstCombiner::BuilderTy &Builder) {
4966 if (!I.isEquality() || !match(I.getOperand(1), m_Zero()) ||
4967 !I.getOperand(0)->hasOneUse())
4968 return nullptr;
4969
4970 auto m_AnyLogicalShift = m_LogicalShift(m_Value(), m_Value());
4971
4972 // Look for an 'and' of two logical shifts, one of which may be truncated.
4973 // We use m_TruncOrSelf() on the RHS to correctly handle commutative case.
4974 Instruction *XShift, *MaybeTruncation, *YShift;
4975 if (!match(
4976 I.getOperand(0),
4977 m_c_And(m_CombineAnd(m_AnyLogicalShift, m_Instruction(XShift)),
4979 m_AnyLogicalShift, m_Instruction(YShift))),
4980 m_Instruction(MaybeTruncation)))))
4981 return nullptr;
4982
4983 // We potentially looked past 'trunc', but only when matching YShift,
4984 // therefore YShift must have the widest type.
4985 Instruction *WidestShift = YShift;
4986 // Therefore XShift must have the shallowest type.
4987 // Or they both have identical types if there was no truncation.
4988 Instruction *NarrowestShift = XShift;
4989
4990 Type *WidestTy = WidestShift->getType();
4991 Type *NarrowestTy = NarrowestShift->getType();
4992 assert(NarrowestTy == I.getOperand(0)->getType() &&
4993 "We did not look past any shifts while matching XShift though.");
4994 bool HadTrunc = WidestTy != I.getOperand(0)->getType();
4995
4996 // If YShift is a 'lshr', swap the shifts around.
4997 if (match(YShift, m_LShr(m_Value(), m_Value())))
4998 std::swap(XShift, YShift);
4999
5000 // The shifts must be in opposite directions.
5001 auto XShiftOpcode = XShift->getOpcode();
5002 if (XShiftOpcode == YShift->getOpcode())
5003 return nullptr; // Do not care about same-direction shifts here.
5004
5005 Value *X, *XShAmt, *Y, *YShAmt;
5006 match(XShift, m_BinOp(m_Value(X), m_ZExtOrSelf(m_Value(XShAmt))));
5007 match(YShift, m_BinOp(m_Value(Y), m_ZExtOrSelf(m_Value(YShAmt))));
5008
5009 // If one of the values being shifted is a constant, then we will end with
5010 // and+icmp, and [zext+]shift instrs will be constant-folded. If they are not,
5011 // however, we will need to ensure that we won't increase instruction count.
5012 if (!isa<Constant>(X) && !isa<Constant>(Y)) {
5013 // At least one of the hands of the 'and' should be one-use shift.
5014 if (!match(I.getOperand(0),
5015 m_c_And(m_OneUse(m_AnyLogicalShift), m_Value())))
5016 return nullptr;
5017 if (HadTrunc) {
5018 // Due to the 'trunc', we will need to widen X. For that either the old
5019 // 'trunc' or the shift amt in the non-truncated shift should be one-use.
5020 if (!MaybeTruncation->hasOneUse() &&
5021 !NarrowestShift->getOperand(1)->hasOneUse())
5022 return nullptr;
5023 }
5024 }
5025
5026 // We have two shift amounts from two different shifts. The types of those
5027 // shift amounts may not match. If that's the case let's bailout now.
5028 if (XShAmt->getType() != YShAmt->getType())
5029 return nullptr;
5030
5031 // As input, we have the following pattern:
5032 // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
5033 // We want to rewrite that as:
5034 // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x)
5035 // While we know that originally (Q+K) would not overflow
5036 // (because 2 * (N-1) u<= iN -1), we have looked past extensions of
5037 // shift amounts. so it may now overflow in smaller bitwidth.
5038 // To ensure that does not happen, we need to ensure that the total maximal
5039 // shift amount is still representable in that smaller bit width.
5040 unsigned MaximalPossibleTotalShiftAmount =
5041 (WidestTy->getScalarSizeInBits() - 1) +
5042 (NarrowestTy->getScalarSizeInBits() - 1);
5043 APInt MaximalRepresentableShiftAmount =
5045 if (MaximalRepresentableShiftAmount.ult(MaximalPossibleTotalShiftAmount))
5046 return nullptr;
5047
5048 // Can we fold (XShAmt+YShAmt) ?
5049 auto *NewShAmt = dyn_cast_or_null<Constant>(
5050 simplifyAddInst(XShAmt, YShAmt, /*isNSW=*/false,
5051 /*isNUW=*/false, SQ.getWithInstruction(&I)));
5052 if (!NewShAmt)
5053 return nullptr;
5054 if (NewShAmt->getType() != WidestTy) {
5055 NewShAmt =
5056 ConstantFoldCastOperand(Instruction::ZExt, NewShAmt, WidestTy, SQ.DL);
5057 if (!NewShAmt)
5058 return nullptr;
5059 }
5060 unsigned WidestBitWidth = WidestTy->getScalarSizeInBits();
5061
5062 // Is the new shift amount smaller than the bit width?
5063 // FIXME: could also rely on ConstantRange.
5064 if (!match(NewShAmt,
5066 APInt(WidestBitWidth, WidestBitWidth))))
5067 return nullptr;
5068
5069 // An extra legality check is needed if we had trunc-of-lshr.
5070 if (HadTrunc && match(WidestShift, m_LShr(m_Value(), m_Value()))) {
5071 auto CanFold = [NewShAmt, WidestBitWidth, NarrowestShift, SQ,
5072 WidestShift]() {
5073 // It isn't obvious whether it's worth it to analyze non-constants here.
5074 // Also, let's basically give up on non-splat cases, pessimizing vectors.
5075 // If *any* of these preconditions matches we can perform the fold.
5076 Constant *NewShAmtSplat = NewShAmt->getType()->isVectorTy()
5077 ? NewShAmt->getSplatValue()
5078 : NewShAmt;
5079 // If it's edge-case shift (by 0 or by WidestBitWidth-1) we can fold.
5080 if (NewShAmtSplat &&
5081 (NewShAmtSplat->isNullValue() ||
5082 NewShAmtSplat->getUniqueInteger() == WidestBitWidth - 1))
5083 return true;
5084 // We consider *min* leading zeros so a single outlier
5085 // blocks the transform as opposed to allowing it.
5086 if (auto *C = dyn_cast<Constant>(NarrowestShift->getOperand(0))) {
5088 unsigned MinLeadZero = Known.countMinLeadingZeros();
5089 // If the value being shifted has at most lowest bit set we can fold.
5090 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero;
5091 if (MaxActiveBits <= 1)
5092 return true;
5093 // Precondition: NewShAmt u<= countLeadingZeros(C)
5094 if (NewShAmtSplat && NewShAmtSplat->getUniqueInteger().ule(MinLeadZero))
5095 return true;
5096 }
5097 if (auto *C = dyn_cast<Constant>(WidestShift->getOperand(0))) {
5099 unsigned MinLeadZero = Known.countMinLeadingZeros();
5100 // If the value being shifted has at most lowest bit set we can fold.
5101 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero;
5102 if (MaxActiveBits <= 1)
5103 return true;
5104 // Precondition: ((WidestBitWidth-1)-NewShAmt) u<= countLeadingZeros(C)
5105 if (NewShAmtSplat) {
5106 APInt AdjNewShAmt =
5107 (WidestBitWidth - 1) - NewShAmtSplat->getUniqueInteger();
5108 if (AdjNewShAmt.ule(MinLeadZero))
5109 return true;
5110 }
5111 }
5112 return false; // Can't tell if it's ok.
5113 };
5114 if (!CanFold())
5115 return nullptr;
5116 }
5117
5118 // All good, we can do this fold.
5119 X = Builder.CreateZExt(X, WidestTy);
5120 Y = Builder.CreateZExt(Y, WidestTy);
5121 // The shift is the same that was for X.
5122 Value *T0 = XShiftOpcode == Instruction::BinaryOps::LShr
5123 ? Builder.CreateLShr(X, NewShAmt)
5124 : Builder.CreateShl(X, NewShAmt);
5125 Value *T1 = Builder.CreateAnd(T0, Y);
5126 return Builder.CreateICmp(I.getPredicate(), T1,
5127 Constant::getNullValue(WidestTy));
5128}
5129
5130/// Fold
5131/// (-1 u/ x) u< y
5132/// ((x * y) ?/ x) != y
5133/// to
5134/// @llvm.?mul.with.overflow(x, y) plus extraction of overflow bit
5135/// Note that the comparison is commutative, while inverted (u>=, ==) predicate
5136/// will mean that we are looking for the opposite answer.
5138 CmpPredicate Pred;
5139 Value *X, *Y;
5141 Instruction *Div;
5142 bool NeedNegation;
5143 // Look for: (-1 u/ x) u</u>= y
5144 if (!I.isEquality() &&
5145 match(&I, m_c_ICmp(Pred,
5147 m_Instruction(Div)),
5148 m_Value(Y)))) {
5149 Mul = nullptr;
5150
5151 // Are we checking that overflow does not happen, or does happen?
5152 switch (Pred) {
5154 NeedNegation = false;
5155 break; // OK
5157 NeedNegation = true;
5158 break; // OK
5159 default:
5160 return nullptr; // Wrong predicate.
5161 }
5162 } else // Look for: ((x * y) / x) !=/== y
5163 if (I.isEquality() &&
5164 match(&I, m_c_ICmp(Pred, m_Value(Y),
5167 m_Value(X)),
5169 m_Deferred(X))),
5170 m_Instruction(Div))))) {
5171 NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ;
5172 } else
5173 return nullptr;
5174
5176 // If the pattern included (x * y), we'll want to insert new instructions
5177 // right before that original multiplication so that we can replace it.
5178 bool MulHadOtherUses = Mul && !Mul->hasOneUse();
5179 if (MulHadOtherUses)
5180 Builder.SetInsertPoint(Mul);
5181
5182 Value *Call = Builder.CreateIntrinsic(
5183 Div->getOpcode() == Instruction::UDiv ? Intrinsic::umul_with_overflow
5184 : Intrinsic::smul_with_overflow,
5185 X->getType(), {X, Y}, /*FMFSource=*/nullptr, "mul");
5186
5187 // If the multiplication was used elsewhere, to ensure that we don't leave
5188 // "duplicate" instructions, replace uses of that original multiplication
5189 // with the multiplication result from the with.overflow intrinsic.
5190 if (MulHadOtherUses)
5191 replaceInstUsesWith(*Mul, Builder.CreateExtractValue(Call, 0, "mul.val"));
5192
5193 Value *Res = Builder.CreateExtractValue(Call, 1, "mul.ov");
5194 if (NeedNegation) // This technically increases instruction count.
5195 Res = Builder.CreateNot(Res, "mul.not.ov");
5196
5197 // If we replaced the mul, erase it. Do this after all uses of Builder,
5198 // as the mul is used as insertion point.
5199 if (MulHadOtherUses)
5201
5202 return Res;
5203}
5204
5206 InstCombiner::BuilderTy &Builder) {
5208 Value *X;
5209 if (match(&I, m_c_ICmp(Pred, m_NSWNeg(m_Value(X)), m_Deferred(X)))) {
5210
5213 else if (ICmpInst::isUnsigned(Pred))
5215 // else for equality-comparisons just keep the predicate.
5216
5217 return ICmpInst::Create(Instruction::ICmp, Pred, X,
5218 Constant::getNullValue(X->getType()), I.getName());
5219 }
5220
5221 // A value is not equal to its negation unless that value is 0 or
5222 // MinSignedValue, ie: a != -a --> (a & MaxSignedVal) != 0
5225 Type *Ty = X->getType();
5226 uint32_t BitWidth = Ty->getScalarSizeInBits();
5227 Constant *MaxSignedVal =
5228 ConstantInt::get(Ty, APInt::getSignedMaxValue(BitWidth));
5229 Value *And = Builder.CreateAnd(X, MaxSignedVal);
5231 return CmpInst::Create(Instruction::ICmp, Pred, And, Zero);
5232 }
5233
5234 return nullptr;
5235}
5236
5238 InstCombinerImpl &IC) {
5239 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1), *A;
5240 // Normalize and operand as operand 0.
5241 CmpInst::Predicate Pred = I.getPredicate();
5242 if (match(Op1, m_c_And(m_Specific(Op0), m_Value()))) {
5243 std::swap(Op0, Op1);
5245 }
5246
5247 if (!match(Op0, m_c_And(m_Specific(Op1), m_Value(A))))
5248 return nullptr;
5249
5250 // (icmp (X & Y) u< X --> (X & Y) != X
5251 if (Pred == ICmpInst::ICMP_ULT)
5252 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
5253
5254 // (icmp (X & Y) u>= X --> (X & Y) == X
5255 if (Pred == ICmpInst::ICMP_UGE)
5256 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
5257
5258 if (ICmpInst::isEquality(Pred) && Op0->hasOneUse()) {
5259 // icmp (X & Y) eq/ne Y --> (X | ~Y) eq/ne -1 if Y is freely invertible and
5260 // Y is non-constant. If Y is constant the `X & C == C` form is preferable
5261 // so don't do this fold.
5262 if (!match(Op1, m_ImmConstant()))
5263 if (auto *NotOp1 =
5264 IC.getFreelyInverted(Op1, !Op1->hasNUsesOrMore(3), &IC.Builder))
5265 return new ICmpInst(Pred, IC.Builder.CreateOr(A, NotOp1),
5266 Constant::getAllOnesValue(Op1->getType()));
5267 // icmp (X & Y) eq/ne Y --> (~X & Y) eq/ne 0 if X is freely invertible.
5268 if (auto *NotA = IC.getFreelyInverted(A, A->hasOneUse(), &IC.Builder))
5269 return new ICmpInst(Pred, IC.Builder.CreateAnd(Op1, NotA),
5270 Constant::getNullValue(Op1->getType()));
5271 }
5272
5274 return nullptr;
5275
5276 KnownBits KnownY = IC.computeKnownBits(A, &I);
5277 // (X & NegY) spred X --> (X & NegY) upred X
5278 if (KnownY.isNegative())
5279 return new ICmpInst(ICmpInst::getUnsignedPredicate(Pred), Op0, Op1);
5280
5282 return nullptr;
5283
5284 if (KnownY.isNonNegative())
5285 // (X & PosY) s<= X --> X s>= 0
5286 // (X & PosY) s> X --> X s< 0
5288 Constant::getNullValue(Op1->getType()));
5289
5291 // (NegX & Y) s<= NegX --> Y s< 0
5292 // (NegX & Y) s> NegX --> Y s>= 0
5294 Constant::getNullValue(A->getType()));
5295
5296 return nullptr;
5297}
5298
5300 InstCombinerImpl &IC) {
5301 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1), *A;
5302
5303 // Normalize or operand as operand 0.
5304 CmpInst::Predicate Pred = I.getPredicate();
5305 if (match(Op1, m_c_Or(m_Specific(Op0), m_Value(A)))) {
5306 std::swap(Op0, Op1);
5308 } else if (!match(Op0, m_c_Or(m_Specific(Op1), m_Value(A)))) {
5309 return nullptr;
5310 }
5311
5312 // icmp (X | Y) u<= X --> (X | Y) == X
5313 if (Pred == ICmpInst::ICMP_ULE)
5314 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
5315
5316 // icmp (X | Y) u> X --> (X | Y) != X
5317 if (Pred == ICmpInst::ICMP_UGT)
5318 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
5319
5320 if (ICmpInst::isEquality(Pred) && Op0->hasOneUse()) {
5321 // icmp (X | Y) eq/ne Y --> (X & ~Y) eq/ne 0 if Y is freely invertible
5322 if (Value *NotOp1 = IC.getFreelyInverted(
5323 Op1, !isa<Constant>(Op1) && !Op1->hasNUsesOrMore(3), &IC.Builder))
5324 return new ICmpInst(Pred, IC.Builder.CreateAnd(A, NotOp1),
5325 Constant::getNullValue(Op1->getType()));
5326 // icmp (X | Y) eq/ne Y --> (~X | Y) eq/ne -1 if X is freely invertible.
5327 if (Value *NotA = IC.getFreelyInverted(A, A->hasOneUse(), &IC.Builder))
5328 return new ICmpInst(Pred, IC.Builder.CreateOr(Op1, NotA),
5329 Constant::getAllOnesValue(Op1->getType()));
5330 }
5331 return nullptr;
5332}
5333
5335 InstCombinerImpl &IC) {
5336 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1), *A;
5337 // Normalize xor operand as operand 0.
5338 CmpInst::Predicate Pred = I.getPredicate();
5339 if (match(Op1, m_c_Xor(m_Specific(Op0), m_Value()))) {
5340 std::swap(Op0, Op1);
5342 }
5343 if (!match(Op0, m_c_Xor(m_Specific(Op1), m_Value(A))))
5344 return nullptr;
5345
5346 // icmp (X ^ Y_NonZero) u>= X --> icmp (X ^ Y_NonZero) u> X
5347 // icmp (X ^ Y_NonZero) u<= X --> icmp (X ^ Y_NonZero) u< X
5348 // icmp (X ^ Y_NonZero) s>= X --> icmp (X ^ Y_NonZero) s> X
5349 // icmp (X ^ Y_NonZero) s<= X --> icmp (X ^ Y_NonZero) s< X
5351 if (PredOut != Pred && isKnownNonZero(A, Q))
5352 return new ICmpInst(PredOut, Op0, Op1);
5353
5354 // These transform work when A is negative.
5355 // X s< X^A, X s<= X^A, X u> X^A, X u>= X^A --> X s< 0
5356 // X s> X^A, X s>= X^A, X u< X^A, X u<= X^A --> X s>= 0
5357 if (match(A, m_Negative())) {
5358 CmpInst::Predicate NewPred;
5360 default:
5361 return nullptr;
5362 case ICmpInst::ICMP_SLT:
5363 case ICmpInst::ICMP_UGT:
5364 NewPred = ICmpInst::ICMP_SLT;
5365 break;
5366 case ICmpInst::ICMP_SGT:
5367 case ICmpInst::ICMP_ULT:
5368 NewPred = ICmpInst::ICMP_SGE;
5369 break;
5370 }
5371 Constant *Const = Constant::getNullValue(Op0->getType());
5372 return new ICmpInst(NewPred, Op0, Const);
5373 }
5374
5375 return nullptr;
5376}
5377
5378/// Return true if X is a multiple of C.
5379/// TODO: Handle non-power-of-2 factors.
5380static bool isMultipleOf(Value *X, const APInt &C, const SimplifyQuery &Q) {
5381 if (C.isOne())
5382 return true;
5383
5384 if (!C.isPowerOf2())
5385 return false;
5386
5387 return MaskedValueIsZero(X, C - 1, Q);
5388}
5389
5391 bool &HasNSW, bool &HasNUW) {
5393 HasNUW = BO.hasNoUnsignedWrap();
5394 HasNSW = BO.hasNoSignedWrap();
5395 return ICmpInst::isEquality(Pred) ||
5396 (CmpInst::isUnsigned(Pred) && HasNUW) ||
5397 (CmpInst::isSigned(Pred) && HasNSW);
5398 }
5399 if (BO.getOpcode() == Instruction::Or) {
5400 // The callers only use this result for an or matched by m_AddLike, which
5401 // must be an or disjoint and is equivalent to an add nuw nsw.
5402 HasNUW = true;
5403 HasNSW = true;
5404 return true;
5405 }
5406 return false;
5407}
5408
5410 CmpInst::Predicate Pred,
5411 Value *&LHS, Value *&RHS,
5412 const SimplifyQuery &Q) {
5413 auto *BO0 = dyn_cast<BinaryOperator>(Op0);
5414 auto *BO1 = dyn_cast<BinaryOperator>(Op1);
5415 if (!BO0 || !BO1)
5416 return false;
5417
5418 bool IsEquality = ICmpInst::isEquality(Pred);
5419 bool IsSigned = ICmpInst::isSigned(Pred);
5420
5421 bool Op0HasNSW = false, Op0HasNUW = false;
5422 bool Op1HasNSW = false, Op1HasNUW = false;
5423 bool NoOp0WrapProblem = hasNoWrapProblem(*BO0, Pred, Op0HasNSW, Op0HasNUW);
5424 bool NoOp1WrapProblem = hasNoWrapProblem(*BO1, Pred, Op1HasNSW, Op1HasNUW);
5425
5426 Value *A, *B, *C, *D;
5427
5428 // Handle all add-like operations.
5429 if (match(BO0, m_AddLike(m_Value(A), m_Value(B))) &&
5430 match(BO1, m_AddLike(m_Value(C), m_Value(D)))) {
5431 if (!NoOp0WrapProblem || !NoOp1WrapProblem)
5432 return false;
5433
5434 if (A == C) {
5435 // (A + B) cmp (A + D) -> B cmp D
5436 LHS = B;
5437 RHS = D;
5438 return true;
5439 }
5440 if (A == D) {
5441 // (A + B) cmp (C + A) -> B cmp C
5442 LHS = B;
5443 RHS = C;
5444 return true;
5445 }
5446 if (B == C) {
5447 // (A + B) cmp (B + D) -> A cmp D
5448 LHS = A;
5449 RHS = D;
5450 return true;
5451 }
5452 if (B == D) {
5453 // (A + B) cmp (C + B) -> A cmp C
5454 LHS = A;
5455 RHS = C;
5456 return true;
5457 }
5458 return false;
5459 }
5460
5461 if (BO0->getOpcode() != BO1->getOpcode())
5462 return false;
5463
5464 A = BO0->getOperand(0);
5465 B = BO0->getOperand(1);
5466 C = BO1->getOperand(0);
5467 D = BO1->getOperand(1);
5468
5469 switch (BO0->getOpcode()) {
5470 case Instruction::Sub:
5471 if (!NoOp0WrapProblem || !NoOp1WrapProblem)
5472 return false;
5473
5474 // (A - B) cmp (C - B) -> A cmp C
5475 if (B == D) {
5476 LHS = A;
5477 RHS = C;
5478 return true;
5479 }
5480 // (A - B) cmp (A - D) -> D cmp B
5481 if (A == C) {
5482 LHS = D;
5483 RHS = B;
5484 return true;
5485 }
5486 return false;
5487
5488 case Instruction::Mul: {
5489 // Equality comparisons have additional modular-arithmetic folds and are
5490 // handled by the existing multiplication logic in foldICmpBinOp().
5491 if (IsEquality || !NoOp0WrapProblem || !NoOp1WrapProblem)
5492 return false;
5493
5494 Value *Z;
5495
5496 if (A == C) {
5497 // (A * B) cmp (A * D) -> B cmp D
5498 Z = A;
5499 LHS = B;
5500 RHS = D;
5501 } else if (A == D) {
5502 // (A * B) cmp (C * A) -> B cmp C
5503 Z = A;
5504 LHS = B;
5505 RHS = C;
5506 } else if (B == C) {
5507 // (A * B) cmp (B * D) -> A cmp D
5508 Z = B;
5509 LHS = A;
5510 RHS = D;
5511 } else if (B == D) {
5512 // (A * B) cmp (C * B) -> A cmp C
5513 Z = B;
5514 LHS = A;
5515 RHS = C;
5516 } else {
5517 return false;
5518 }
5519
5520 if (!IsSigned)
5521 return isKnownNonZero(Z, Q);
5522
5523 WithCache<const Value *> ZCache(Z);
5524 if (isKnownPositive(ZCache, Q))
5525 return true;
5526
5527 if (isKnownNegative(ZCache, Q)) {
5528 std::swap(LHS, RHS);
5529 return true;
5530 }
5531
5532 return false;
5533 }
5534 default:
5535 return false;
5536 }
5537}
5538
5539/// Try to fold icmp (binop), X or icmp X, (binop).
5540/// TODO: A large part of this logic is duplicated in InstSimplify's
5541/// simplifyICmpWithBinOp(). We should be able to share that and avoid the code
5542/// duplication.
5544 const SimplifyQuery &SQ) {
5545 const SimplifyQuery Q = SQ.getWithInstruction(&I);
5546 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
5547
5548 // Special logic for binary operators.
5551 if (!BO0 && !BO1)
5552 return nullptr;
5553
5554 if (Instruction *NewICmp = foldICmpXNegX(I, Builder))
5555 return NewICmp;
5556
5557 const CmpInst::Predicate Pred = I.getPredicate();
5558
5559 // (X urem Y) == X --> X u< Y
5560 // (X urem Y) != X --> X u>= Y
5561 Value *Dividend, *Divisor;
5562 if (I.isEquality() &&
5563 match(&I, m_c_ICmp(m_URem(m_Value(Dividend), m_Value(Divisor)),
5564 m_Deferred(Dividend)))) {
5565 CmpInst::Predicate NewPred =
5567 return new ICmpInst(NewPred, Dividend, Divisor);
5568 }
5569
5570 Value *X;
5571
5572 // Convert add-with-unsigned-overflow comparisons into a 'not' with compare.
5573 // (Op1 + X) u</u>= Op1 --> ~Op1 u</u>= X
5574 if (match(Op0, m_OneUse(m_c_Add(m_Specific(Op1), m_Value(X)))) &&
5575 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))
5576 return new ICmpInst(Pred, Builder.CreateNot(Op1), X);
5577 // Op0 u>/u<= (Op0 + X) --> X u>/u<= ~Op0
5578 if (match(Op1, m_OneUse(m_c_Add(m_Specific(Op0), m_Value(X)))) &&
5579 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE))
5580 return new ICmpInst(Pred, X, Builder.CreateNot(Op0));
5581
5582 {
5583 // (Op1 + X) + C u</u>= Op1 --> ~C - X u</u>= Op1
5584 Constant *C;
5585 if (match(Op0, m_OneUse(m_Add(m_c_Add(m_Specific(Op1), m_Value(X)),
5586 m_ImmConstant(C)))) &&
5587 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) {
5589 return new ICmpInst(Pred, Builder.CreateSub(C2, X), Op1);
5590 }
5591 // Op0 u>/u<= (Op0 + X) + C --> Op0 u>/u<= ~C - X
5592 if (match(Op1, m_OneUse(m_Add(m_c_Add(m_Specific(Op0), m_Value(X)),
5593 m_ImmConstant(C)))) &&
5594 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) {
5596 return new ICmpInst(Pred, Op0, Builder.CreateSub(C2, X));
5597 }
5598 }
5599
5600 // (icmp eq/ne (X, -P2), INT_MIN)
5601 // -> (icmp slt/sge X, INT_MIN + P2)
5602 if (ICmpInst::isEquality(Pred) && BO0 &&
5603 match(I.getOperand(1), m_SignMask()) &&
5605 // Will Constant fold.
5606 Value *NewC = Builder.CreateSub(I.getOperand(1), BO0->getOperand(1));
5607 return new ICmpInst(Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_SLT
5609 BO0->getOperand(0), NewC);
5610 }
5611
5612 {
5613 // Similar to above: an unsigned overflow comparison may use offset + mask:
5614 // ((Op1 + C) & C) u< Op1 --> Op1 != 0
5615 // ((Op1 + C) & C) u>= Op1 --> Op1 == 0
5616 // Op0 u> ((Op0 + C) & C) --> Op0 != 0
5617 // Op0 u<= ((Op0 + C) & C) --> Op0 == 0
5618 BinaryOperator *BO;
5619 const APInt *C;
5620 if ((Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) &&
5621 match(Op0, m_And(m_BinOp(BO), m_LowBitMask(C))) &&
5623 CmpInst::Predicate NewPred =
5625 Constant *Zero = ConstantInt::getNullValue(Op1->getType());
5626 return new ICmpInst(NewPred, Op1, Zero);
5627 }
5628
5629 if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) &&
5630 match(Op1, m_And(m_BinOp(BO), m_LowBitMask(C))) &&
5632 CmpInst::Predicate NewPred =
5634 Constant *Zero = ConstantInt::getNullValue(Op1->getType());
5635 return new ICmpInst(NewPred, Op0, Zero);
5636 }
5637 }
5638
5639 bool NoOp0WrapProblem = false, NoOp1WrapProblem = false;
5640 bool Op0HasNUW = false, Op1HasNUW = false;
5641 bool Op0HasNSW = false, Op1HasNSW = false;
5642 // Analyze the case when either Op0 or Op1 is an add instruction.
5643 // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null).
5644 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
5645
5646 if (BO0) {
5647 match(BO0, m_AddLike(m_Value(A), m_Value(B)));
5648 NoOp0WrapProblem = hasNoWrapProblem(*BO0, Pred, Op0HasNSW, Op0HasNUW);
5649 }
5650 if (BO1) {
5651 match(BO1, m_AddLike(m_Value(C), m_Value(D)));
5652 NoOp1WrapProblem = hasNoWrapProblem(*BO1, Pred, Op1HasNSW, Op1HasNUW);
5653 }
5654
5655 // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow.
5656 // icmp (A+B), B -> icmp A, 0 for equalities or if there is no overflow.
5657 if ((A == Op1 || B == Op1) && NoOp0WrapProblem)
5658 return new ICmpInst(Pred, A == Op1 ? B : A,
5659 Constant::getNullValue(Op1->getType()));
5660
5661 // icmp C, (C+D) -> icmp 0, D for equalities or if there is no overflow.
5662 // icmp D, (C+D) -> icmp 0, C for equalities or if there is no overflow.
5663 if ((C == Op0 || D == Op0) && NoOp1WrapProblem)
5664 return new ICmpInst(Pred, Constant::getNullValue(Op0->getType()),
5665 C == Op0 ? D : C);
5666
5667 // Fold comparisons of binops with a removable common operand.
5668 Value *LHS;
5669 Value *RHS;
5670 if (matchCommonBinOpOperands(Op0, Op1, Pred, LHS, RHS, Q))
5671 return new ICmpInst(Pred, LHS, RHS);
5672
5673 if (ICmpInst::isRelational(Pred)) {
5674 // Return if both X and Y is divisible by Z/-Z.
5675 // TODO: Generalize to check if (X - Y) is divisible by Z/-Z.
5676 auto ShareCommonDivisor = [&Q](Value *X, Value *Y, Value *Z,
5677 bool IsNegative) -> bool {
5678 const APInt *OffsetC;
5679 if (!match(Z, m_APInt(OffsetC)))
5680 return false;
5681
5682 // Fast path for Z == 1/-1.
5683 if (IsNegative ? OffsetC->isAllOnes() : OffsetC->isOne())
5684 return true;
5685
5686 APInt C = *OffsetC;
5687 if (IsNegative)
5688 C.negate();
5689 // Note: -INT_MIN is also negative.
5690 if (!C.isStrictlyPositive())
5691 return false;
5692
5693 return isMultipleOf(X, C, Q) && isMultipleOf(Y, C, Q);
5694 };
5695
5696 // The subtraction-related identities (A -nuw B) shown below require that
5697 // the subtraction does not wrap unsigned (i.e., A >=u B). Canonicalization
5698 // from (A -nuw 1) to (A + -1) means that such combinations ought to never
5699 // occur, as sub nuw ops should have been canonicalized to add ones. It may
5700 // however appear in the form of a or disjoint. Though, or disjoint A, -B
5701 // requires proving A <u B, for which the nowrap precondition can never be
5702 // satisfied. These are therefore skipped.
5703 //
5704 // icmp ult (A - 1), Op1 -> icmp ule A, Op1
5705 // icmp uge (A - 1), Op1 -> icmp ugt A, Op1
5706 // icmp ugt Op0, (C - 1) -> icmp uge Op0, C
5707 // icmp ule Op0, (C - 1) -> icmp ult Op0, C
5708
5709 // icmp slt (A + -1), Op1 -> icmp sle A, Op1
5710 // icmp sge (A + -1), Op1 -> icmp sgt A, Op1
5711 // icmp sle (A + 1), Op1 -> icmp slt A, Op1
5712 // icmp sgt (A + 1), Op1 -> icmp sge A, Op1
5713 // icmp ule (A + 1), Op0 -> icmp ult A, Op1
5714 // icmp ugt (A + 1), Op0 -> icmp uge A, Op1
5715 bool IsNegative = ICmpInst::isLT(Pred) || ICmpInst::isGE(Pred);
5716 bool IsAddOrSignedPred = !IsNegative || ICmpInst::isSigned(Pred);
5717 if (A && NoOp0WrapProblem && IsAddOrSignedPred &&
5718 ShareCommonDivisor(A, Op1, B, IsNegative))
5720 Op1);
5721
5722 // icmp sgt Op0, (C + -1) -> icmp sge Op0, C
5723 // icmp sle Op0, (C + -1) -> icmp slt Op0, C
5724 // icmp sge Op0, (C + 1) -> icmp sgt Op0, C
5725 // icmp slt Op0, (C + 1) -> icmp sle Op0, C
5726 // icmp uge Op0, (C + 1) -> icmp ugt Op0, C
5727 // icmp ult Op0, (C + 1) -> icmp ule Op0, C
5728 if (C && NoOp1WrapProblem &&
5729 ShareCommonDivisor(Op0, C, D,
5730 ICmpInst::isGT(Pred) || ICmpInst::isLE(Pred)))
5732 C);
5733 }
5734
5735 // if C1 has greater magnitude than C2:
5736 // icmp (A + C1), (C + C2) -> icmp (A + C3), C
5737 // s.t. C3 = C1 - C2
5738 //
5739 // if C2 has greater magnitude than C1:
5740 // icmp (A + C1), (C + C2) -> icmp A, (C + C3)
5741 // s.t. C3 = C2 - C1
5742 if (A && C && NoOp0WrapProblem && NoOp1WrapProblem &&
5743 (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned()) {
5744 const APInt *AP1, *AP2;
5745 // TODO: Support non-uniform vectors.
5746 // TODO: Allow poison passthrough if B or D's element is poison.
5747 if (match(B, m_APIntAllowPoison(AP1)) &&
5748 match(D, m_APIntAllowPoison(AP2)) &&
5749 AP1->isNegative() == AP2->isNegative()) {
5750 APInt AP1Abs = AP1->abs();
5751 APInt AP2Abs = AP2->abs();
5752 if (AP1Abs.uge(AP2Abs)) {
5753 APInt Diff = *AP1 - *AP2;
5754 Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff);
5755 Value *NewAdd = Builder.CreateAdd(
5756 A, C3, "", Op0HasNUW && Diff.ule(*AP1), Op0HasNSW);
5757 return new ICmpInst(Pred, NewAdd, C);
5758 } else {
5759 APInt Diff = *AP2 - *AP1;
5760 Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff);
5761 Value *NewAdd = Builder.CreateAdd(
5762 C, C3, "", Op1HasNUW && Diff.ule(*AP2), Op1HasNSW);
5763 return new ICmpInst(Pred, A, NewAdd);
5764 }
5765 }
5766 Constant *Cst1, *Cst2;
5767 if (match(B, m_ImmConstant(Cst1)) && match(D, m_ImmConstant(Cst2)) &&
5768 ICmpInst::isEquality(Pred)) {
5769 Constant *Diff = ConstantExpr::getSub(Cst2, Cst1);
5770 Value *NewAdd = Builder.CreateAdd(C, Diff);
5771 return new ICmpInst(Pred, A, NewAdd);
5772 }
5773 }
5774
5775 // Analyze the case when either Op0 or Op1 is a sub instruction.
5776 // Op0 = A - B (or A and B are null); Op1 = C - D (or C and D are null).
5777 A = nullptr;
5778 B = nullptr;
5779 C = nullptr;
5780 D = nullptr;
5781 if (BO0 && BO0->getOpcode() == Instruction::Sub) {
5782 A = BO0->getOperand(0);
5783 B = BO0->getOperand(1);
5784 }
5785 if (BO1 && BO1->getOpcode() == Instruction::Sub) {
5786 C = BO1->getOperand(0);
5787 D = BO1->getOperand(1);
5788 }
5789
5790 // icmp (A-B), A -> icmp 0, B for equalities or if there is no overflow.
5791 if (A == Op1 && NoOp0WrapProblem)
5792 return new ICmpInst(Pred, Constant::getNullValue(Op1->getType()), B);
5793 // icmp C, (C-D) -> icmp D, 0 for equalities or if there is no overflow.
5794 if (C == Op0 && NoOp1WrapProblem)
5795 return new ICmpInst(Pred, D, Constant::getNullValue(Op0->getType()));
5796
5797 // Convert sub-with-unsigned-overflow comparisons into a comparison of args.
5798 // (A - B) u>/u<= A --> B u>/u<= A
5799 if (A == Op1 && (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE))
5800 return new ICmpInst(Pred, B, A);
5801 // C u</u>= (C - D) --> C u</u>= D
5802 if (C == Op0 && (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))
5803 return new ICmpInst(Pred, C, D);
5804 // (A - B) u>=/u< A --> B u>/u<= A iff B != 0
5805 if (A == Op1 && (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) &&
5806 isKnownNonZero(B, Q))
5808 // C u<=/u> (C - D) --> C u</u>= D iff B != 0
5809 if (C == Op0 && (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
5810 isKnownNonZero(D, Q))
5812
5813 // icmp (0-X) < cst --> x > -cst
5814 if (NoOp0WrapProblem && ICmpInst::isSigned(Pred)) {
5815 Value *X;
5816 if (match(BO0, m_Neg(m_Value(X))))
5817 if (Constant *RHSC = dyn_cast<Constant>(Op1))
5818 if (RHSC->isNotMinSignedValue())
5819 return new ICmpInst(I.getSwappedPredicate(), X,
5820 ConstantExpr::getNeg(RHSC));
5821 }
5822
5823 if (Instruction *R = foldICmpXorXX(I, Q, *this))
5824 return R;
5825 if (Instruction *R = foldICmpOrXX(I, Q, *this))
5826 return R;
5827
5828 {
5829 // Try to remove shared multiplier from comparison:
5830 // X * Z pred Y * Z
5831 Value *X, *Y, *Z;
5832 if ((match(Op0, m_Mul(m_Value(X), m_Value(Z))) &&
5833 match(Op1, m_c_Mul(m_Specific(Z), m_Value(Y)))) ||
5834 (match(Op0, m_Mul(m_Value(Z), m_Value(X))) &&
5835 match(Op1, m_c_Mul(m_Specific(Z), m_Value(Y))))) {
5836 if (ICmpInst::isSigned(Pred)) {
5837 if (Op0HasNSW && Op1HasNSW) {
5839 SQ.getWithInstruction(&I));
5840 if (LessThan && match(LessThan, m_One()))
5841 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Z,
5842 Constant::getNullValue(Z->getType()));
5843 Value *GreaterThan = simplifyICmpInst(ICmpInst::ICMP_SGT, X, Y,
5844 SQ.getWithInstruction(&I));
5845 if (GreaterThan && match(GreaterThan, m_One()))
5846 return new ICmpInst(Pred, Z, Constant::getNullValue(Z->getType()));
5847 }
5848 } else if (ICmpInst::isEquality(Pred)) {
5849 // Handle equality folds beyond direct multiplier cancellation.
5850 // If X != Y, fold (X *nw Z) eq/ne (Y *nw Z) -> Z eq/ne 0
5851 if (((Op0HasNSW && Op1HasNSW) || (Op0HasNUW && Op1HasNUW)) &&
5852 isKnownNonEqual(X, Y, SQ))
5853 return new ICmpInst(Pred, Z, Constant::getNullValue(Z->getType()));
5854
5855 KnownBits ZKnown = computeKnownBits(Z, &I);
5856 // if Z % 2 != 0
5857 // X * Z eq/ne Y * Z -> X eq/ne Y
5858 if (ZKnown.countMaxTrailingZeros() == 0)
5859 return new ICmpInst(Pred, X, Y);
5860 bool NonZero = !ZKnown.One.isZero() || isKnownNonZero(Z, Q);
5861 // if Z != 0 and nsw(X * Z) and nsw(Y * Z)
5862 // X * Z eq/ne Y * Z -> X eq/ne Y
5863 if (NonZero && Op0HasNSW && Op1HasNSW)
5864 return new ICmpInst(Pred, X, Y);
5865
5866 // If Z != 0 and nuw(X * Z) and nuw(Y * Z)
5867 // X * Z eq/ne Y * Z -> X eq/ne Y
5868 if (NonZero && Op0HasNUW && Op1HasNUW)
5869 return new ICmpInst(Pred, X, Y);
5870 }
5871 }
5872 }
5873
5874 BinaryOperator *SRem = nullptr;
5875 // icmp (srem X, Y), Y
5876 if (BO0 && BO0->getOpcode() == Instruction::SRem && Op1 == BO0->getOperand(1))
5877 SRem = BO0;
5878 // icmp Y, (srem X, Y)
5879 else if (BO1 && BO1->getOpcode() == Instruction::SRem &&
5880 Op0 == BO1->getOperand(1))
5881 SRem = BO1;
5882 if (SRem) {
5883 // We don't check hasOneUse to avoid increasing register pressure because
5884 // the value we use is the same value this instruction was already using.
5885 switch (SRem == BO0 ? ICmpInst::getSwappedPredicate(Pred) : Pred) {
5886 default:
5887 break;
5888 case ICmpInst::ICMP_EQ:
5889 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5890 case ICmpInst::ICMP_NE:
5891 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5892 case ICmpInst::ICMP_SGT:
5893 case ICmpInst::ICMP_SGE:
5894 return new ICmpInst(ICmpInst::ICMP_SGT, SRem->getOperand(1),
5896 case ICmpInst::ICMP_SLT:
5897 case ICmpInst::ICMP_SLE:
5898 return new ICmpInst(ICmpInst::ICMP_SLT, SRem->getOperand(1),
5900 }
5901 }
5902
5903 if (BO0 && BO1 && BO0->getOpcode() == BO1->getOpcode() &&
5904 (BO0->hasOneUse() || BO1->hasOneUse()) &&
5905 BO0->getOperand(1) == BO1->getOperand(1)) {
5906 switch (BO0->getOpcode()) {
5907 default:
5908 break;
5909 case Instruction::Add:
5910 case Instruction::Sub:
5911 case Instruction::Xor: {
5912 if (I.isEquality()) // a+x icmp eq/ne b+x --> a icmp b
5913 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5914
5915 const APInt *C;
5916 if (match(BO0->getOperand(1), m_APInt(C))) {
5917 // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b
5918 if (C->isSignMask()) {
5919 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate();
5920 return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0));
5921 }
5922
5923 // icmp u/s (a ^ maxsignval), (b ^ maxsignval) --> icmp s/u' a, b
5924 if (BO0->getOpcode() == Instruction::Xor && C->isMaxSignedValue()) {
5925 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate();
5926 NewPred = I.getSwappedPredicate(NewPred);
5927 return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0));
5928 }
5929 }
5930 break;
5931 }
5932 case Instruction::Mul: {
5933 if (!I.isEquality())
5934 break;
5935
5936 const APInt *C;
5937 if (match(BO0->getOperand(1), m_APInt(C)) && !C->isZero() &&
5938 !C->isOne()) {
5939 // icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask)
5940 // Mask = -1 >> count-trailing-zeros(C).
5941 if (unsigned TZs = C->countr_zero()) {
5942 Constant *Mask = ConstantInt::get(
5943 BO0->getType(),
5944 APInt::getLowBitsSet(C->getBitWidth(), C->getBitWidth() - TZs));
5945 Value *And1 = Builder.CreateAnd(BO0->getOperand(0), Mask);
5946 Value *And2 = Builder.CreateAnd(BO1->getOperand(0), Mask);
5947 return new ICmpInst(Pred, And1, And2);
5948 }
5949 }
5950 break;
5951 }
5952 case Instruction::UDiv:
5953 case Instruction::LShr:
5954 if (I.isSigned() || !BO0->isExact() || !BO1->isExact())
5955 break;
5956 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5957
5958 case Instruction::SDiv:
5959 if (!(I.isEquality() || match(BO0->getOperand(1), m_NonNegative())) ||
5960 !BO0->isExact() || !BO1->isExact())
5961 break;
5962 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5963
5964 case Instruction::AShr:
5965 if (!BO0->isExact() || !BO1->isExact())
5966 break;
5967 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5968
5969 case Instruction::Shl: {
5970 bool NUW = Op0HasNUW && Op1HasNUW;
5971 bool NSW = Op0HasNSW && Op1HasNSW;
5972 if (!NUW && !NSW)
5973 break;
5974 if (!NSW && I.isSigned())
5975 break;
5976 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5977 }
5978 }
5979 }
5980
5981 if (BO0) {
5982 // Transform A & (L - 1) `ult` L --> L != 0
5983 auto LSubOne = m_Add(m_Specific(Op1), m_AllOnes());
5984 auto BitwiseAnd = m_c_And(m_Value(), LSubOne);
5985
5986 if (match(BO0, BitwiseAnd) && Pred == ICmpInst::ICMP_ULT) {
5987 auto *Zero = Constant::getNullValue(BO0->getType());
5988 return new ICmpInst(ICmpInst::ICMP_NE, Op1, Zero);
5989 }
5990 }
5991
5992 // For unsigned predicates / eq / ne:
5993 // icmp pred (x << 1), x --> icmp getSignedPredicate(pred) x, 0
5994 // icmp pred x, (x << 1) --> icmp getSignedPredicate(pred) 0, x
5995 if (!ICmpInst::isSigned(Pred)) {
5996 if (match(Op0, m_Shl(m_Specific(Op1), m_One())))
5997 return new ICmpInst(ICmpInst::getSignedPredicate(Pred), Op1,
5998 Constant::getNullValue(Op1->getType()));
5999 else if (match(Op1, m_Shl(m_Specific(Op0), m_One())))
6000 return new ICmpInst(ICmpInst::getSignedPredicate(Pred),
6001 Constant::getNullValue(Op0->getType()), Op0);
6002 }
6003
6005 return replaceInstUsesWith(I, V);
6006
6007 if (Instruction *R = foldICmpAndXX(I, Q, *this))
6008 return R;
6009
6011 return replaceInstUsesWith(I, V);
6012
6014 return replaceInstUsesWith(I, V);
6015
6016 return nullptr;
6017}
6018
6019/// Fold icmp Pred min|max(X, Y), Z.
6022 Value *Z, CmpPredicate Pred) {
6023 Value *X = MinMax->getLHS();
6024 Value *Y = MinMax->getRHS();
6025 if (ICmpInst::isSigned(Pred) && !MinMax->isSigned())
6026 return nullptr;
6027 if (ICmpInst::isUnsigned(Pred) && MinMax->isSigned()) {
6028 // Revert the transform signed pred -> unsigned pred
6029 // TODO: We can flip the signedness of predicate if both operands of icmp
6030 // are negative.
6031 if (isKnownNonNegative(Z, SQ.getWithInstruction(&I)) &&
6032 isKnownNonNegative(MinMax, SQ.getWithInstruction(&I))) {
6034 } else
6035 return nullptr;
6036 }
6037 SimplifyQuery Q = SQ.getWithInstruction(&I);
6038 auto IsCondKnownTrue = [](Value *Val) -> std::optional<bool> {
6039 if (!Val)
6040 return std::nullopt;
6041 if (match(Val, m_One()))
6042 return true;
6043 if (match(Val, m_Zero()))
6044 return false;
6045 return std::nullopt;
6046 };
6047 // Remove samesign here since it is illegal to keep it when we speculatively
6048 // execute comparisons. For example, `icmp samesign ult umax(X, -46), -32`
6049 // cannot be decomposed into `(icmp samesign ult X, -46) or (icmp samesign ult
6050 // -46, -32)`. `X` is allowed to be non-negative here.
6051 Pred = Pred.dropSameSign();
6052 auto CmpXZ = IsCondKnownTrue(simplifyICmpInst(Pred, X, Z, Q));
6053 auto CmpYZ = IsCondKnownTrue(simplifyICmpInst(Pred, Y, Z, Q));
6054 if (!CmpXZ.has_value() && !CmpYZ.has_value())
6055 return nullptr;
6056 if (!CmpXZ.has_value()) {
6057 std::swap(X, Y);
6058 std::swap(CmpXZ, CmpYZ);
6059 }
6060
6061 auto FoldIntoCmpYZ = [&]() -> Instruction * {
6062 if (CmpYZ.has_value())
6063 return replaceInstUsesWith(I, ConstantInt::getBool(I.getType(), *CmpYZ));
6064 return ICmpInst::Create(Instruction::ICmp, Pred, Y, Z);
6065 };
6066
6067 switch (Pred) {
6068 case ICmpInst::ICMP_EQ:
6069 case ICmpInst::ICMP_NE: {
6070 // If X == Z:
6071 // Expr Result
6072 // min(X, Y) == Z X <= Y
6073 // max(X, Y) == Z X >= Y
6074 // min(X, Y) != Z X > Y
6075 // max(X, Y) != Z X < Y
6076 if ((Pred == ICmpInst::ICMP_EQ) == *CmpXZ) {
6077 ICmpInst::Predicate NewPred =
6078 ICmpInst::getNonStrictPredicate(MinMax->getPredicate());
6079 if (Pred == ICmpInst::ICMP_NE)
6080 NewPred = ICmpInst::getInversePredicate(NewPred);
6081 return ICmpInst::Create(Instruction::ICmp, NewPred, X, Y);
6082 }
6083 // Otherwise (X != Z):
6084 ICmpInst::Predicate NewPred = MinMax->getPredicate();
6085 auto MinMaxCmpXZ = IsCondKnownTrue(simplifyICmpInst(NewPred, X, Z, Q));
6086 if (!MinMaxCmpXZ.has_value()) {
6087 std::swap(X, Y);
6088 std::swap(CmpXZ, CmpYZ);
6089 // Re-check pre-condition X != Z
6090 if (!CmpXZ.has_value() || (Pred == ICmpInst::ICMP_EQ) == *CmpXZ)
6091 break;
6092 MinMaxCmpXZ = IsCondKnownTrue(simplifyICmpInst(NewPred, X, Z, Q));
6093 }
6094 if (!MinMaxCmpXZ.has_value())
6095 break;
6096 if (*MinMaxCmpXZ) {
6097 // Expr Fact Result
6098 // min(X, Y) == Z X < Z false
6099 // max(X, Y) == Z X > Z false
6100 // min(X, Y) != Z X < Z true
6101 // max(X, Y) != Z X > Z true
6102 return replaceInstUsesWith(
6103 I, ConstantInt::getBool(I.getType(), Pred == ICmpInst::ICMP_NE));
6104 } else {
6105 // Expr Fact Result
6106 // min(X, Y) == Z X > Z Y == Z
6107 // max(X, Y) == Z X < Z Y == Z
6108 // min(X, Y) != Z X > Z Y != Z
6109 // max(X, Y) != Z X < Z Y != Z
6110 return FoldIntoCmpYZ();
6111 }
6112 break;
6113 }
6114 case ICmpInst::ICMP_SLT:
6115 case ICmpInst::ICMP_ULT:
6116 case ICmpInst::ICMP_SLE:
6117 case ICmpInst::ICMP_ULE:
6118 case ICmpInst::ICMP_SGT:
6119 case ICmpInst::ICMP_UGT:
6120 case ICmpInst::ICMP_SGE:
6121 case ICmpInst::ICMP_UGE: {
6122 bool IsSame = MinMax->getPredicate() == ICmpInst::getStrictPredicate(Pred);
6123 if (*CmpXZ) {
6124 if (IsSame) {
6125 // Expr Fact Result
6126 // min(X, Y) < Z X < Z true
6127 // min(X, Y) <= Z X <= Z true
6128 // max(X, Y) > Z X > Z true
6129 // max(X, Y) >= Z X >= Z true
6130 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
6131 } else {
6132 // Expr Fact Result
6133 // max(X, Y) < Z X < Z Y < Z
6134 // max(X, Y) <= Z X <= Z Y <= Z
6135 // min(X, Y) > Z X > Z Y > Z
6136 // min(X, Y) >= Z X >= Z Y >= Z
6137 return FoldIntoCmpYZ();
6138 }
6139 } else {
6140 if (IsSame) {
6141 // Expr Fact Result
6142 // min(X, Y) < Z X >= Z Y < Z
6143 // min(X, Y) <= Z X > Z Y <= Z
6144 // max(X, Y) > Z X <= Z Y > Z
6145 // max(X, Y) >= Z X < Z Y >= Z
6146 return FoldIntoCmpYZ();
6147 } else {
6148 // Expr Fact Result
6149 // max(X, Y) < Z X >= Z false
6150 // max(X, Y) <= Z X > Z false
6151 // min(X, Y) > Z X <= Z false
6152 // min(X, Y) >= Z X < Z false
6153 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
6154 }
6155 }
6156 break;
6157 }
6158 default:
6159 break;
6160 }
6161
6162 return nullptr;
6163}
6164
6165/// Match and fold patterns like:
6166/// icmp eq/ne X, min(max(X, Lo), Hi)
6167/// which represents a range check and can be represented as a ConstantRange.
6168///
6169/// For icmp eq, build ConstantRange [Lo, Hi + 1) and convert to:
6170/// (X - Lo) u< (Hi + 1 - Lo)
6171/// For icmp ne, build ConstantRange [Hi + 1, Lo) and convert to:
6172/// (X - (Hi + 1)) u< (Lo - (Hi + 1))
6174 MinMaxIntrinsic *Min) {
6175 if (!I.isEquality() || !Min->hasOneUse() || !Min->isMin())
6176 return nullptr;
6177
6178 const APInt *Lo = nullptr, *Hi = nullptr;
6179 if (Min->isSigned()) {
6180 if (!match(Min->getLHS(), m_OneUse(m_SMax(m_Specific(X), m_APInt(Lo)))) ||
6181 !match(Min->getRHS(), m_APInt(Hi)) || !Lo->slt(*Hi))
6182 return nullptr;
6183 } else {
6184 if (!match(Min->getLHS(), m_OneUse(m_UMax(m_Specific(X), m_APInt(Lo)))) ||
6185 !match(Min->getRHS(), m_APInt(Hi)) || !Lo->ult(*Hi))
6186 return nullptr;
6187 }
6188
6191 APInt C, Offset;
6192 if (I.getPredicate() == ICmpInst::ICMP_EQ)
6193 CR.getEquivalentICmp(Pred, C, Offset);
6194 else
6195 CR.inverse().getEquivalentICmp(Pred, C, Offset);
6196
6197 if (!Offset.isZero())
6198 X = Builder.CreateAdd(X, ConstantInt::get(X->getType(), Offset));
6199
6200 return replaceInstUsesWith(
6201 I, Builder.CreateICmp(Pred, X, ConstantInt::get(X->getType(), C)));
6202}
6203
6204// Canonicalize checking for a power-of-2-or-zero value:
6206 InstCombiner::BuilderTy &Builder) {
6207 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
6208 const CmpInst::Predicate Pred = I.getPredicate();
6209 Value *A = nullptr;
6210 bool CheckIs;
6211 if (I.isEquality()) {
6212 // (A & (A-1)) == 0 --> ctpop(A) < 2 (two commuted variants)
6213 // ((A-1) & A) != 0 --> ctpop(A) > 1 (two commuted variants)
6214 if (!match(Op0, m_OneUse(m_c_And(m_Add(m_Value(A), m_AllOnes()),
6215 m_Deferred(A)))) ||
6216 !match(Op1, m_ZeroInt()))
6217 A = nullptr;
6218
6219 // (A & -A) == A --> ctpop(A) < 2 (four commuted variants)
6220 // (-A & A) != A --> ctpop(A) > 1 (four commuted variants)
6221 if (match(Op0, m_OneUse(m_c_And(m_Neg(m_Specific(Op1)), m_Specific(Op1)))))
6222 A = Op1;
6223 else if (match(Op1,
6225 A = Op0;
6226
6227 CheckIs = Pred == ICmpInst::ICMP_EQ;
6228 } else if (ICmpInst::isUnsigned(Pred)) {
6229 // (A ^ (A-1)) u>= A --> ctpop(A) < 2 (two commuted variants)
6230 // ((A-1) ^ A) u< A --> ctpop(A) > 1 (two commuted variants)
6231
6234 m_Specific(Op1))))) {
6235 A = Op1;
6236 CheckIs = Pred == ICmpInst::ICMP_UGE;
6237 } else if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) &&
6239 m_Specific(Op0))))) {
6240 A = Op0;
6241 CheckIs = Pred == ICmpInst::ICMP_ULE;
6242 }
6243 }
6244
6245 if (A) {
6246 Type *Ty = A->getType();
6247 Value *CtPop = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, A);
6248 return CheckIs ? new ICmpInst(ICmpInst::ICMP_ULT, CtPop,
6249 ConstantInt::get(Ty, 2))
6250 : new ICmpInst(ICmpInst::ICMP_UGT, CtPop,
6251 ConstantInt::get(Ty, 1));
6252 }
6253
6254 return nullptr;
6255}
6256
6257/// Find all possible pairs (BinOp, RHS) that BinOp V, RHS can be simplified.
6258using OffsetOp = std::pair<Instruction::BinaryOps, Value *>;
6260 bool AllowRecursion) {
6262 if (!Inst || !Inst->hasOneUse())
6263 return;
6264
6265 switch (Inst->getOpcode()) {
6266 case Instruction::Add:
6267 Offsets.emplace_back(Instruction::Sub, Inst->getOperand(1));
6268 Offsets.emplace_back(Instruction::Sub, Inst->getOperand(0));
6269 break;
6270 case Instruction::Sub:
6271 Offsets.emplace_back(Instruction::Add, Inst->getOperand(1));
6272 break;
6273 case Instruction::Xor:
6274 Offsets.emplace_back(Instruction::Xor, Inst->getOperand(1));
6275 Offsets.emplace_back(Instruction::Xor, Inst->getOperand(0));
6276 break;
6277 case Instruction::Shl:
6278 if (Inst->hasNoSignedWrap())
6279 Offsets.emplace_back(Instruction::AShr, Inst->getOperand(1));
6280 if (Inst->hasNoUnsignedWrap())
6281 Offsets.emplace_back(Instruction::LShr, Inst->getOperand(1));
6282 break;
6283 case Instruction::Select:
6284 if (AllowRecursion) {
6285 collectOffsetOp(Inst->getOperand(1), Offsets, /*AllowRecursion=*/false);
6286 collectOffsetOp(Inst->getOperand(2), Offsets, /*AllowRecursion=*/false);
6287 }
6288 break;
6289 default:
6290 break;
6291 }
6292}
6293
6295
6300
6302 return {OffsetKind::Invalid, nullptr, nullptr, nullptr, nullptr};
6303 }
6305 return {OffsetKind::Value, V, nullptr, nullptr, nullptr};
6306 }
6307 static OffsetResult select(Value *Cond, Value *TrueV, Value *FalseV,
6309 return {OffsetKind::Select, Cond, TrueV, FalseV, MDFrom};
6310 }
6311 bool isValid() const { return Kind != OffsetKind::Invalid; }
6313 switch (Kind) {
6315 llvm_unreachable("Invalid offset result");
6316 case OffsetKind::Value:
6317 return V0;
6318 case OffsetKind::Select:
6319 return Builder.CreateSelect(V0, V1, V2, "", MDFrom);
6320 }
6321 llvm_unreachable("Unknown OffsetKind enum");
6322 }
6323};
6324
6325/// Offset both sides of an equality icmp to see if we can save some
6326/// instructions: icmp eq/ne X, Y -> icmp eq/ne X op Z, Y op Z.
6327/// Note: This operation should not introduce poison.
6329 InstCombiner::BuilderTy &Builder,
6330 const SimplifyQuery &SQ) {
6331 assert(I.isEquality() && "Expected an equality icmp");
6332 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
6333 if (!Op0->getType()->isIntOrIntVectorTy())
6334 return nullptr;
6335
6336 SmallVector<OffsetOp, 4> OffsetOps;
6337 collectOffsetOp(Op0, OffsetOps, /*AllowRecursion=*/true);
6338 collectOffsetOp(Op1, OffsetOps, /*AllowRecursion=*/true);
6339
6340 auto ApplyOffsetImpl = [&](Value *V, unsigned BinOpc, Value *RHS) -> Value * {
6341 switch (BinOpc) {
6342 // V = shl nsw X, RHS => X = ashr V, RHS
6343 case Instruction::AShr: {
6344 const APInt *CV, *CRHS;
6345 if (!(match(V, m_APInt(CV)) && match(RHS, m_APInt(CRHS)) &&
6346 CV->ashr(*CRHS).shl(*CRHS) == *CV) &&
6348 return nullptr;
6349 break;
6350 }
6351 // V = shl nuw X, RHS => X = lshr V, RHS
6352 case Instruction::LShr: {
6353 const APInt *CV, *CRHS;
6354 if (!(match(V, m_APInt(CV)) && match(RHS, m_APInt(CRHS)) &&
6355 CV->lshr(*CRHS).shl(*CRHS) == *CV) &&
6357 return nullptr;
6358 break;
6359 }
6360 default:
6361 break;
6362 }
6363
6364 Value *Simplified = simplifyBinOp(BinOpc, V, RHS, SQ);
6365 if (!Simplified)
6366 return nullptr;
6367 // Reject constant expressions as they don't simplify things.
6368 if (isa<Constant>(Simplified) && !match(Simplified, m_ImmConstant()))
6369 return nullptr;
6370 // Check if the transformation introduces poison.
6371 return impliesPoison(RHS, V) ? Simplified : nullptr;
6372 };
6373
6374 auto ApplyOffset = [&](Value *V, unsigned BinOpc,
6375 Value *RHS) -> OffsetResult {
6376 if (auto *Sel = dyn_cast<SelectInst>(V)) {
6377 if (!Sel->hasOneUse())
6378 return OffsetResult::invalid();
6379 Value *TrueVal = ApplyOffsetImpl(Sel->getTrueValue(), BinOpc, RHS);
6380 if (!TrueVal)
6381 return OffsetResult::invalid();
6382 Value *FalseVal = ApplyOffsetImpl(Sel->getFalseValue(), BinOpc, RHS);
6383 if (!FalseVal)
6384 return OffsetResult::invalid();
6385 return OffsetResult::select(Sel->getCondition(), TrueVal, FalseVal, Sel);
6386 }
6387 if (Value *Simplified = ApplyOffsetImpl(V, BinOpc, RHS))
6388 return OffsetResult::value(Simplified);
6389 return OffsetResult::invalid();
6390 };
6391
6392 for (auto [BinOp, RHS] : OffsetOps) {
6393 auto BinOpc = static_cast<unsigned>(BinOp);
6394
6395 auto Op0Result = ApplyOffset(Op0, BinOpc, RHS);
6396 if (!Op0Result.isValid())
6397 continue;
6398 auto Op1Result = ApplyOffset(Op1, BinOpc, RHS);
6399 if (!Op1Result.isValid())
6400 continue;
6401
6402 Value *NewLHS = Op0Result.materialize(Builder);
6403 Value *NewRHS = Op1Result.materialize(Builder);
6404 return new ICmpInst(I.getPredicate(), NewLHS, NewRHS);
6405 }
6406
6407 return nullptr;
6408}
6409
6411 if (!I.isEquality())
6412 return nullptr;
6413
6414 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
6415 const CmpInst::Predicate Pred = I.getPredicate();
6416 Value *A, *B, *C, *D;
6417 if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) {
6418 if (A == Op1 || B == Op1) { // (A^B) == A -> B == 0
6419 Value *OtherVal = A == Op1 ? B : A;
6420 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType()));
6421 }
6422
6423 if (match(Op1, m_Xor(m_Value(C), m_Value(D)))) {
6424 // A^c1 == C^c2 --> A == C^(c1^c2)
6425 ConstantInt *C1, *C2;
6426 if (match(B, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2)) &&
6427 Op1->hasOneUse()) {
6428 Constant *NC = Builder.getInt(C1->getValue() ^ C2->getValue());
6429 Value *Xor = Builder.CreateXor(C, NC);
6430 return new ICmpInst(Pred, A, Xor);
6431 }
6432
6433 // A^B == A^D -> B == D
6434 if (A == C)
6435 return new ICmpInst(Pred, B, D);
6436 if (A == D)
6437 return new ICmpInst(Pred, B, C);
6438 if (B == C)
6439 return new ICmpInst(Pred, A, D);
6440 if (B == D)
6441 return new ICmpInst(Pred, A, C);
6442 }
6443 }
6444
6445 if (match(Op1, m_Xor(m_Value(A), m_Value(B))) && (A == Op0 || B == Op0)) {
6446 // A == (A^B) -> B == 0
6447 Value *OtherVal = A == Op0 ? B : A;
6448 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType()));
6449 }
6450
6451 // (X&Z) == (Y&Z) -> (X^Y) & Z == 0
6452 if (match(Op0, m_And(m_Value(A), m_Value(B))) &&
6453 match(Op1, m_And(m_Value(C), m_Value(D)))) {
6454 Value *X = nullptr, *Y = nullptr, *Z = nullptr;
6455
6456 if (A == C) {
6457 X = B;
6458 Y = D;
6459 Z = A;
6460 } else if (A == D) {
6461 X = B;
6462 Y = C;
6463 Z = A;
6464 } else if (B == C) {
6465 X = A;
6466 Y = D;
6467 Z = B;
6468 } else if (B == D) {
6469 X = A;
6470 Y = C;
6471 Z = B;
6472 }
6473
6474 if (X) {
6475 // If X^Y is a negative power of two, then `icmp eq/ne (Z & NegP2), 0`
6476 // will fold to `icmp ult/uge Z, -NegP2` incurringb no additional
6477 // instructions.
6478 const APInt *C0, *C1;
6479 bool XorIsNegP2 = match(X, m_APInt(C0)) && match(Y, m_APInt(C1)) &&
6480 (*C0 ^ *C1).isNegatedPowerOf2();
6481
6482 // If either Op0/Op1 are both one use or X^Y will constant fold and one of
6483 // Op0/Op1 are one use, proceed. In those cases we are instruction neutral
6484 // but `icmp eq/ne A, 0` is easier to analyze than `icmp eq/ne A, B`.
6485 int UseCnt =
6486 int(Op0->hasOneUse()) + int(Op1->hasOneUse()) +
6487 (int(match(X, m_ImmConstant()) && match(Y, m_ImmConstant())));
6488 if (XorIsNegP2 || UseCnt >= 2) {
6489 // Build (X^Y) & Z
6490 Op1 = Builder.CreateXor(X, Y);
6491 Op1 = Builder.CreateAnd(Op1, Z);
6492 return new ICmpInst(Pred, Op1, Constant::getNullValue(Op1->getType()));
6493 }
6494 }
6495 }
6496
6497 {
6498 // Similar to above, but specialized for constant because invert is needed:
6499 // (X | C) == (Y | C) --> (X ^ Y) & ~C == 0
6500 Value *X, *Y;
6501 Constant *C;
6502 if (match(Op0, m_OneUse(m_Or(m_Value(X), m_Constant(C)))) &&
6503 match(Op1, m_OneUse(m_Or(m_Value(Y), m_Specific(C))))) {
6504 Value *Xor = Builder.CreateXor(X, Y);
6505 Value *And = Builder.CreateAnd(Xor, ConstantExpr::getNot(C));
6506 return new ICmpInst(Pred, And, Constant::getNullValue(And->getType()));
6507 }
6508 }
6509
6510 if (match(Op1, m_ZExt(m_Value(A))) &&
6511 (Op0->hasOneUse() || Op1->hasOneUse())) {
6512 // (B & (Pow2C-1)) == zext A --> A == trunc B
6513 // (B & (Pow2C-1)) != zext A --> A != trunc B
6514 const APInt *MaskC;
6515 if (match(Op0, m_And(m_Value(B), m_LowBitMask(MaskC))) &&
6516 MaskC->countr_one() == A->getType()->getScalarSizeInBits())
6517 return new ICmpInst(Pred, A, Builder.CreateTrunc(B, A->getType()));
6518 }
6519
6520 // (A >> C) == (B >> C) --> (A^B) u< (1 << C)
6521 // For lshr and ashr pairs.
6522 const APInt *AP1, *AP2;
6523 if ((match(Op0, m_OneUse(m_LShr(m_Value(A), m_APIntAllowPoison(AP1)))) &&
6524 match(Op1, m_OneUse(m_LShr(m_Value(B), m_APIntAllowPoison(AP2))))) ||
6525 (match(Op0, m_OneUse(m_AShr(m_Value(A), m_APIntAllowPoison(AP1)))) &&
6526 match(Op1, m_OneUse(m_AShr(m_Value(B), m_APIntAllowPoison(AP2)))))) {
6527 if (*AP1 != *AP2)
6528 return nullptr;
6529 unsigned TypeBits = AP1->getBitWidth();
6530 unsigned ShAmt = AP1->getLimitedValue(TypeBits);
6531 if (ShAmt < TypeBits && ShAmt != 0) {
6532 ICmpInst::Predicate NewPred =
6534 Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted");
6535 APInt CmpVal = APInt::getOneBitSet(TypeBits, ShAmt);
6536 return new ICmpInst(NewPred, Xor, ConstantInt::get(A->getType(), CmpVal));
6537 }
6538 }
6539
6540 // (A << C) == (B << C) --> ((A^B) & (~0U >> C)) == 0
6541 ConstantInt *Cst1;
6542 if (match(Op0, m_OneUse(m_Shl(m_Value(A), m_ConstantInt(Cst1)))) &&
6543 match(Op1, m_OneUse(m_Shl(m_Value(B), m_Specific(Cst1))))) {
6544 unsigned TypeBits = Cst1->getBitWidth();
6545 unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits);
6546 if (ShAmt < TypeBits && ShAmt != 0) {
6547 Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted");
6548 APInt AndVal = APInt::getLowBitsSet(TypeBits, TypeBits - ShAmt);
6549 Value *And =
6550 Builder.CreateAnd(Xor, Builder.getInt(AndVal), I.getName() + ".mask");
6551 return new ICmpInst(Pred, And, Constant::getNullValue(Cst1->getType()));
6552 }
6553 }
6554
6555 // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to
6556 // "icmp (and X, mask), cst"
6557 uint64_t ShAmt = 0;
6558 if (Op0->hasOneUse() &&
6559 match(Op0, m_Trunc(m_OneUse(m_LShr(m_Value(A), m_ConstantInt(ShAmt))))) &&
6560 match(Op1, m_ConstantInt(Cst1)) &&
6561 // Only do this when A has multiple uses. This is most important to do
6562 // when it exposes other optimizations.
6563 !A->hasOneUse()) {
6564 unsigned ASize = cast<IntegerType>(A->getType())->getPrimitiveSizeInBits();
6565
6566 if (ShAmt < ASize) {
6567 APInt MaskV =
6569 MaskV <<= ShAmt;
6570
6571 APInt CmpV = Cst1->getValue().zext(ASize);
6572 CmpV <<= ShAmt;
6573
6574 Value *Mask = Builder.CreateAnd(A, Builder.getInt(MaskV));
6575 return new ICmpInst(Pred, Mask, Builder.getInt(CmpV));
6576 }
6577 }
6578
6580 return ICmp;
6581
6582 // Match icmp eq (trunc (lshr A, BW), (ashr (trunc A), BW-1)), which checks
6583 // the top BW/2 + 1 bits are all the same. Create "A >=s INT_MIN && A <=s
6584 // INT_MAX", which we generate as "icmp ult (add A, 2^(BW-1)), 2^BW" to skip a
6585 // few steps of instcombine.
6586 unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
6587 if (match(Op0, m_AShr(m_Trunc(m_Value(A)), m_SpecificInt(BitWidth - 1))) &&
6589 A->getType()->getScalarSizeInBits() == BitWidth * 2 &&
6590 (I.getOperand(0)->hasOneUse() || I.getOperand(1)->hasOneUse())) {
6592 Value *Add = Builder.CreateAdd(A, ConstantInt::get(A->getType(), C));
6593 return new ICmpInst(Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULT
6595 Add, ConstantInt::get(A->getType(), C.shl(1)));
6596 }
6597
6598 // Canonicalize:
6599 // Assume B_Pow2 != 0
6600 // 1. A & B_Pow2 != B_Pow2 -> A & B_Pow2 == 0
6601 // 2. A & B_Pow2 == B_Pow2 -> A & B_Pow2 != 0
6602 if (match(Op0, m_c_And(m_Specific(Op1), m_Value())) &&
6603 isKnownToBeAPowerOfTwo(Op1, /* OrZero */ false, &I))
6604 return new ICmpInst(CmpInst::getInversePredicate(Pred), Op0,
6606
6607 if (match(Op1, m_c_And(m_Specific(Op0), m_Value())) &&
6608 isKnownToBeAPowerOfTwo(Op0, /* OrZero */ false, &I))
6609 return new ICmpInst(CmpInst::getInversePredicate(Pred), Op1,
6610 ConstantInt::getNullValue(Op1->getType()));
6611
6612 // Canonicalize:
6613 // icmp eq/ne X, OneUse(rotate-right(X))
6614 // -> icmp eq/ne X, rotate-left(X)
6615 // We generally try to convert rotate-right -> rotate-left, this just
6616 // canonicalizes another case.
6617 if (match(&I, m_c_ICmp(m_Value(A),
6619 m_Deferred(A), m_Deferred(A), m_Value(B))))))
6620 return new ICmpInst(
6621 Pred, A,
6622 Builder.CreateIntrinsic(Op0->getType(), Intrinsic::fshl, {A, A, B}));
6623
6624 // Canonicalize:
6625 // icmp eq/ne OneUse(A ^ Cst), B --> icmp eq/ne (A ^ B), Cst
6626 Constant *Cst;
6629 return new ICmpInst(Pred, Builder.CreateXor(A, B), Cst);
6630
6631 {
6632 // (icmp eq/ne (and (add/sub/xor X, P2), P2), P2)
6633 auto m_Matcher =
6636 m_Sub(m_Value(B), m_Deferred(A)));
6637 std::optional<bool> IsZero = std::nullopt;
6638 if (match(&I, m_c_ICmp(m_OneUse(m_c_And(m_Value(A), m_Matcher)),
6639 m_Deferred(A))))
6640 IsZero = false;
6641 // (icmp eq/ne (and (add/sub/xor X, P2), P2), 0)
6642 else if (match(&I,
6643 m_ICmp(m_OneUse(m_c_And(m_Value(A), m_Matcher)), m_Zero())))
6644 IsZero = true;
6645
6646 if (IsZero && isKnownToBeAPowerOfTwo(A, /* OrZero */ true, &I))
6647 // (icmp eq/ne (and (add/sub/xor X, P2), P2), P2)
6648 // -> (icmp eq/ne (and X, P2), 0)
6649 // (icmp eq/ne (and (add/sub/xor X, P2), P2), 0)
6650 // -> (icmp eq/ne (and X, P2), P2)
6651 return new ICmpInst(Pred, Builder.CreateAnd(B, A),
6652 *IsZero ? A
6653 : ConstantInt::getNullValue(A->getType()));
6654 }
6655
6656 if (auto *Res = foldICmpEqualityWithOffset(
6657 I, Builder, getSimplifyQuery().getWithInstruction(&I)))
6658 return Res;
6659
6660 return nullptr;
6661}
6662
6664 ICmpInst::Predicate Pred = ICmp.getPredicate();
6665 Value *Op0 = ICmp.getOperand(0), *Op1 = ICmp.getOperand(1);
6666
6667 // Try to canonicalize trunc + compare-to-constant into a mask + cmp.
6668 // The trunc masks high bits while the compare may effectively mask low bits.
6669 Value *X;
6670 const APInt *C;
6671 if (!match(Op0, m_OneUse(m_Trunc(m_Value(X)))) || !match(Op1, m_APInt(C)))
6672 return nullptr;
6673
6674 // This matches patterns corresponding to tests of the signbit as well as:
6675 // (trunc X) pred C2 --> (X & Mask) == C
6676 if (auto Res = decomposeBitTestICmp(Op0, Op1, Pred, /*LookThroughTrunc=*/true,
6677 /*AllowNonZeroC=*/true)) {
6678 Value *And = Builder.CreateAnd(Res->X, Res->Mask);
6679 Constant *C = ConstantInt::get(Res->X->getType(), Res->C);
6680 return new ICmpInst(Res->Pred, And, C);
6681 }
6682
6683 unsigned SrcBits = X->getType()->getScalarSizeInBits();
6684 if (auto *II = dyn_cast<IntrinsicInst>(X)) {
6685 if (II->getIntrinsicID() == Intrinsic::cttz ||
6686 II->getIntrinsicID() == Intrinsic::ctlz) {
6687 unsigned MaxRet = SrcBits;
6688 // If the "is_zero_poison" argument is set, then we know at least
6689 // one bit is set in the input, so the result is always at least one
6690 // less than the full bitwidth of that input.
6691 if (match(II->getArgOperand(1), m_One()))
6692 MaxRet--;
6693
6694 // Make sure the destination is wide enough to hold the largest output of
6695 // the intrinsic.
6696 if (llvm::Log2_32(MaxRet) + 1 <= Op0->getType()->getScalarSizeInBits())
6697 if (Instruction *I =
6698 foldICmpIntrinsicWithConstant(ICmp, II, C->zext(SrcBits)))
6699 return I;
6700 }
6701 }
6702
6703 return nullptr;
6704}
6705
6707 assert(isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0");
6708 auto *CastOp0 = cast<CastInst>(ICmp.getOperand(0));
6709 Value *X;
6710 if (!match(CastOp0, m_ZExtOrSExt(m_Value(X))))
6711 return nullptr;
6712
6713 bool IsSignedExt = CastOp0->getOpcode() == Instruction::SExt;
6714 bool IsSignedCmp = ICmp.isSigned();
6715
6716 // icmp Pred (ext X), (ext Y)
6717 Value *Y;
6718 if (match(ICmp.getOperand(1), m_ZExtOrSExt(m_Value(Y)))) {
6719 bool IsZext0 = isa<ZExtInst>(ICmp.getOperand(0));
6720 bool IsZext1 = isa<ZExtInst>(ICmp.getOperand(1));
6721
6722 if (IsZext0 != IsZext1) {
6723 // If X and Y and both i1
6724 // (icmp eq/ne (zext X) (sext Y))
6725 // eq -> (icmp eq (or X, Y), 0)
6726 // ne -> (icmp ne (or X, Y), 0)
6727 if (ICmp.isEquality() && X->getType()->isIntOrIntVectorTy(1) &&
6728 Y->getType()->isIntOrIntVectorTy(1))
6729 return new ICmpInst(ICmp.getPredicate(), Builder.CreateOr(X, Y),
6730 Constant::getNullValue(X->getType()));
6731
6732 // If we have mismatched casts and zext has the nneg flag, we can
6733 // treat the "zext nneg" as "sext". Otherwise, we cannot fold and quit.
6734
6735 auto *NonNegInst0 = dyn_cast<PossiblyNonNegInst>(ICmp.getOperand(0));
6736 auto *NonNegInst1 = dyn_cast<PossiblyNonNegInst>(ICmp.getOperand(1));
6737
6738 bool IsNonNeg0 = NonNegInst0 && NonNegInst0->hasNonNeg();
6739 bool IsNonNeg1 = NonNegInst1 && NonNegInst1->hasNonNeg();
6740
6741 if ((IsZext0 && IsNonNeg0) || (IsZext1 && IsNonNeg1))
6742 IsSignedExt = true;
6743 else
6744 return nullptr;
6745 }
6746
6747 // Not an extension from the same type?
6748 Type *XTy = X->getType(), *YTy = Y->getType();
6749 if (XTy != YTy) {
6750 // One of the casts must have one use because we are creating a new cast.
6751 if (!ICmp.getOperand(0)->hasOneUse() && !ICmp.getOperand(1)->hasOneUse())
6752 return nullptr;
6753 // Extend the narrower operand to the type of the wider operand.
6754 CastInst::CastOps CastOpcode =
6755 IsSignedExt ? Instruction::SExt : Instruction::ZExt;
6756 if (XTy->getScalarSizeInBits() < YTy->getScalarSizeInBits())
6757 X = Builder.CreateCast(CastOpcode, X, YTy);
6758 else if (YTy->getScalarSizeInBits() < XTy->getScalarSizeInBits())
6759 Y = Builder.CreateCast(CastOpcode, Y, XTy);
6760 else
6761 return nullptr;
6762 }
6763
6764 // (zext X) == (zext Y) --> X == Y
6765 // (sext X) == (sext Y) --> X == Y
6766 if (ICmp.isEquality())
6767 return new ICmpInst(ICmp.getPredicate(), X, Y);
6768
6769 // A signed comparison of sign extended values simplifies into a
6770 // signed comparison.
6771 if (IsSignedCmp && IsSignedExt)
6772 return new ICmpInst(ICmp.getPredicate(), X, Y);
6773
6774 // The other three cases all fold into an unsigned comparison.
6775 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Y);
6776 }
6777
6778 // Below here, we are only folding a compare with constant.
6779 auto *C = dyn_cast<Constant>(ICmp.getOperand(1));
6780 if (!C)
6781 return nullptr;
6782
6783 // If a lossless truncate is possible...
6784 Type *SrcTy = CastOp0->getSrcTy();
6785 Constant *Res = getLosslessInvCast(C, SrcTy, CastOp0->getOpcode(), DL);
6786 if (Res) {
6787 if (ICmp.isEquality())
6788 return new ICmpInst(ICmp.getPredicate(), X, Res);
6789
6790 // A signed comparison of sign extended values simplifies into a
6791 // signed comparison.
6792 if (IsSignedExt && IsSignedCmp)
6793 return new ICmpInst(ICmp.getPredicate(), X, Res);
6794
6795 // The other three cases all fold into an unsigned comparison.
6796 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Res);
6797 }
6798
6799 // The re-extended constant changed, partly changed (in the case of a vector),
6800 // or could not be determined to be equal (in the case of a constant
6801 // expression), so the constant cannot be represented in the shorter type.
6802 // All the cases that fold to true or false will have already been handled
6803 // by simplifyICmpInst, so only deal with the tricky case.
6804 if (IsSignedCmp || !IsSignedExt || !isa<ConstantInt>(C))
6805 return nullptr;
6806
6807 // Is source op positive?
6808 // icmp ult (sext X), C --> icmp sgt X, -1
6809 if (ICmp.getPredicate() == ICmpInst::ICMP_ULT)
6811
6812 // Is source op negative?
6813 // icmp ugt (sext X), C --> icmp slt X, 0
6814 assert(ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!");
6816}
6817
6818/// Handle icmp (cast x), (cast or constant).
6820 // If any operand of ICmp is a inttoptr roundtrip cast then remove it as
6821 // icmp compares only pointer's value.
6822 // icmp (inttoptr (ptrtoint p1)), p2 --> icmp p1, p2.
6823 Value *SimplifiedOp0 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(0));
6824 Value *SimplifiedOp1 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(1));
6825 if (SimplifiedOp0 || SimplifiedOp1)
6826 return new ICmpInst(ICmp.getPredicate(),
6827 SimplifiedOp0 ? SimplifiedOp0 : ICmp.getOperand(0),
6828 SimplifiedOp1 ? SimplifiedOp1 : ICmp.getOperand(1));
6829
6830 auto *CastOp0 = dyn_cast<CastInst>(ICmp.getOperand(0));
6831 Value *Op1 = ICmp.getOperand(1);
6832 if (!CastOp0)
6833 return nullptr;
6834 if (!isa<Constant>(ICmp.getOperand(1)) && !isa<CastInst>(ICmp.getOperand(1)))
6835 return nullptr;
6836
6837 Value *Op0Src = CastOp0->getOperand(0);
6838 Type *SrcTy = CastOp0->getSrcTy();
6839 Type *DestTy = CastOp0->getDestTy();
6840
6841 // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the
6842 // integer type is the same size as the pointer type.
6843 auto CompatibleSizes = [&](Type *PtrTy, Type *IntTy) {
6844 unsigned IntWidth = IntTy->getScalarType()->getIntegerBitWidth();
6845 unsigned IndexWidth = DL.getAddressSizeInBits(PtrTy);
6846 unsigned PtrWidth = DL.getPointerTypeSizeInBits(PtrTy);
6847 // For ptrtoint/inttoptr, we must check that IntWidth == IndexWidth and also
6848 // IndexWidth == PtrWidth to (not) handle non-integral pointers.
6849 return IntWidth == IndexWidth && IndexWidth == PtrWidth;
6850 };
6851 if (isa<PtrToIntInst, PtrToAddrInst>(CastOp0)) {
6852 bool HasPtrToInt = isa<PtrToIntInst>(CastOp0);
6853 Value *NewOp1 = nullptr;
6854 if (auto *PtrToIntOp1 = dyn_cast<PtrToIntOperator>(Op1)) {
6855 NewOp1 = PtrToIntOp1->getOperand(0);
6856 HasPtrToInt = true;
6857 } else if (auto *PtrToAddrOp1 = dyn_cast<PtrToAddrOperator>(Op1)) {
6858 NewOp1 = PtrToAddrOp1->getOperand(0);
6859 } else if (auto *RHSC = dyn_cast<Constant>(Op1)) {
6860 NewOp1 = ConstantExpr::getIntToPtr(RHSC, SrcTy);
6861 }
6862
6863 // For ptrtoaddr, IntWidth == IndexWidth is implied and we don't need to
6864 // check PtrWidth.
6865 if ((!HasPtrToInt || CompatibleSizes(SrcTy, DestTy)) &&
6866 (NewOp1 && NewOp1->getType() == Op0Src->getType()))
6867 return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1);
6868 }
6869
6870 // Do the same in the other direction for icmp (inttoptr x), (inttoptr/c).
6871 if (CastOp0->getOpcode() == Instruction::IntToPtr &&
6872 CompatibleSizes(DestTy, SrcTy)) {
6873 Value *NewOp1 = nullptr;
6874 if (auto *IntToPtrOp1 = dyn_cast<IntToPtrInst>(Op1)) {
6875 Value *IntSrc = IntToPtrOp1->getOperand(0);
6876 if (IntSrc->getType() == Op0Src->getType())
6877 NewOp1 = IntToPtrOp1->getOperand(0);
6878 } else if (auto *RHSC = dyn_cast<Constant>(Op1)) {
6879 NewOp1 = ConstantFoldConstant(ConstantExpr::getPtrToInt(RHSC, SrcTy), DL);
6880 }
6881
6882 if (NewOp1)
6883 return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1);
6884 }
6885
6886 if (Instruction *R = foldICmpWithTrunc(ICmp))
6887 return R;
6888
6889 return foldICmpWithZextOrSext(ICmp);
6890}
6891
6893 bool IsSigned) {
6894 switch (BinaryOp) {
6895 default:
6896 llvm_unreachable("Unsupported binary op");
6897 case Instruction::Add:
6898 case Instruction::Sub:
6899 return match(RHS, m_Zero());
6900 case Instruction::Mul:
6901 return !(RHS->getType()->isIntOrIntVectorTy(1) && IsSigned) &&
6902 match(RHS, m_One());
6903 }
6904}
6905
6908 bool IsSigned, Value *LHS, Value *RHS,
6909 Instruction *CtxI) const {
6910 switch (BinaryOp) {
6911 default:
6912 llvm_unreachable("Unsupported binary op");
6913 case Instruction::Add:
6914 if (IsSigned)
6915 return computeOverflowForSignedAdd(LHS, RHS, CtxI);
6916 else
6917 return computeOverflowForUnsignedAdd(LHS, RHS, CtxI);
6918 case Instruction::Sub:
6919 if (IsSigned)
6920 return computeOverflowForSignedSub(LHS, RHS, CtxI);
6921 else
6922 return computeOverflowForUnsignedSub(LHS, RHS, CtxI);
6923 case Instruction::Mul:
6924 if (IsSigned)
6925 return computeOverflowForSignedMul(LHS, RHS, CtxI);
6926 else
6927 return computeOverflowForUnsignedMul(LHS, RHS, CtxI);
6928 }
6929}
6930
6931bool InstCombinerImpl::OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp,
6932 bool IsSigned, Value *LHS,
6933 Value *RHS, Instruction &OrigI,
6934 Value *&Result,
6935 Constant *&Overflow) {
6936 if (OrigI.isCommutative() && isa<Constant>(LHS) && !isa<Constant>(RHS))
6937 std::swap(LHS, RHS);
6938
6939 // If the overflow check was an add followed by a compare, the insertion point
6940 // may be pointing to the compare. We want to insert the new instructions
6941 // before the add in case there are uses of the add between the add and the
6942 // compare.
6943 Builder.SetInsertPoint(&OrigI);
6944
6945 Type *OverflowTy = Type::getInt1Ty(LHS->getContext());
6946 if (auto *LHSTy = dyn_cast<VectorType>(LHS->getType()))
6947 OverflowTy = VectorType::get(OverflowTy, LHSTy->getElementCount());
6948
6949 if (isNeutralValue(BinaryOp, RHS, IsSigned)) {
6950 Result = LHS;
6951 Overflow = ConstantInt::getFalse(OverflowTy);
6952 return true;
6953 }
6954
6955 switch (computeOverflow(BinaryOp, IsSigned, LHS, RHS, &OrigI)) {
6957 return false;
6960 Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
6961 Result->takeName(&OrigI);
6962 Overflow = ConstantInt::getTrue(OverflowTy);
6963 return true;
6965 Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
6966 Result->takeName(&OrigI);
6967 Overflow = ConstantInt::getFalse(OverflowTy);
6968 if (auto *Inst = dyn_cast<Instruction>(Result)) {
6969 if (IsSigned)
6970 Inst->setHasNoSignedWrap();
6971 else
6972 Inst->setHasNoUnsignedWrap();
6973 }
6974 return true;
6975 }
6976
6977 llvm_unreachable("Unexpected overflow result");
6978}
6979
6980/// Recognize and process idiom involving test for unsigned
6981/// overflow.
6982///
6983/// The caller has matched a pattern of the form:
6984/// I = cmp u (add(zext A, zext B), V
6985/// I = cmp u (mul(zext A, zext B), V
6986/// The function checks if this is a test for overflow and if so replaces
6987/// addition/multiplication with call to the umul intrinsic or the canonical
6988/// form of uadd overflow.
6989///
6990/// \param I Compare instruction.
6991/// \param Val Result of add/mul instruction. It is one of the arguments of
6992/// the compare instruction. Must be of integer type.
6993/// \param OtherVal The other argument of compare instruction.
6994/// \returns Instruction which must replace the compare instruction, NULL if no
6995/// replacement required.
6997 const APInt *OtherVal,
6998 InstCombinerImpl &IC) {
6999 // Don't bother doing this transformation for pointers, don't do it for
7000 // vectors.
7001 if (!isa<IntegerType>(Val->getType()))
7002 return nullptr;
7003
7004 auto *Instr = cast<Instruction>(Val);
7005 unsigned Opcode = Instr->getOpcode();
7006 assert(Opcode == Instruction::Add || Opcode == Instruction::Mul);
7007
7008 auto *LHS = cast<ZExtInst>(Instr->getOperand(0)),
7009 *RHS = cast<ZExtInst>(Instr->getOperand(1));
7010 Value *A = LHS->getOperand(0), *B = RHS->getOperand(0);
7011
7012 // Calculate type and width of the result produced by add/mul.with.overflow.
7013 Type *TyA = A->getType(), *TyB = B->getType();
7014 unsigned WidthA = TyA->getPrimitiveSizeInBits(),
7015 WidthB = TyB->getPrimitiveSizeInBits();
7016 unsigned ResultWidth;
7017 Type *ResultType;
7018 if (WidthB > WidthA) {
7019 ResultWidth = WidthB;
7020 ResultType = TyB;
7021 } else {
7022 ResultWidth = WidthA;
7023 ResultType = TyA;
7024 }
7025
7026 // In order to replace the original result with a narrower one, all uses must
7027 // ignore upper bits of the result. The number of used low bits must be not
7028 // greater than the width of add or mul.with.overflow.
7029 if (Val->hasNUsesOrMore(2))
7030 for (User *U : Val->users()) {
7031 if (U == &I)
7032 continue;
7033 if (TruncInst *TI = dyn_cast<TruncInst>(U)) {
7034 // Check if truncation ignores bits above ResultWidth.
7035 unsigned TruncWidth = TI->getType()->getPrimitiveSizeInBits();
7036 if (TruncWidth > ResultWidth)
7037 return nullptr;
7038 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) {
7039 // Check if AND ignores bits above ResultWidth.
7040 if (BO->getOpcode() != Instruction::And)
7041 return nullptr;
7042 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->getOperand(1))) {
7043 const APInt &CVal = CI->getValue();
7044 if (CVal.getBitWidth() - CVal.countl_zero() > ResultWidth)
7045 return nullptr;
7046 } else {
7047 // In this case we could have the operand of the binary operation
7048 // being defined in another block, and performing the replacement
7049 // could break the dominance relation.
7050 return nullptr;
7051 }
7052 } else {
7053 // Other uses prohibit this transformation.
7054 return nullptr;
7055 }
7056 }
7057
7058 // Recognize patterns
7059 switch (I.getPredicate()) {
7060 case ICmpInst::ICMP_UGT: {
7061 // Recognize pattern:
7062 // val = add/mul(zext A, zext B)
7063 // cmp ugt val, max
7064 APInt MaxVal = APInt::getMaxValue(ResultWidth);
7065 MaxVal = MaxVal.zext(OtherVal->getBitWidth());
7066 if (MaxVal.eq(*OtherVal))
7067 break; // Recognized
7068 return nullptr;
7069 }
7070
7071 case ICmpInst::ICMP_ULT: {
7072 // Recognize pattern:
7073 // val = add/mul(zext A, zext B)
7074 // cmp ult val, max + 1
7075 APInt MaxVal = APInt::getOneBitSet(OtherVal->getBitWidth(), ResultWidth);
7076 if (MaxVal.eq(*OtherVal))
7077 break; // Recognized
7078 return nullptr;
7079 }
7080
7081 default:
7082 return nullptr;
7083 }
7084
7085 InstCombiner::BuilderTy &Builder = IC.Builder;
7086 Builder.SetInsertPoint(Instr);
7087
7088 // Replace: add/mul(zext A, zext B) --> canonical add/mul + overflow check
7089 Value *ResultA = A, *ResultB = B;
7090 if (WidthA < ResultWidth)
7091 ResultA = Builder.CreateZExt(A, ResultType);
7092 if (WidthB < ResultWidth)
7093 ResultB = Builder.CreateZExt(B, ResultType);
7094
7095 Value *ArithResult;
7096 Value *OverflowCheck;
7097
7098 if (Opcode == Instruction::Add) {
7099 // Canonical add overflow check: add + compare
7100 ArithResult = Builder.CreateAdd(ResultA, ResultB, "add");
7101 // Overflow if result < either operand (for unsigned add)
7102 if (I.getPredicate() == ICmpInst::ICMP_ULT)
7103 OverflowCheck =
7104 Builder.CreateICmpUGE(ArithResult, ResultA, "not.add.overflow");
7105 else
7106 OverflowCheck =
7107 Builder.CreateICmpULT(ArithResult, ResultA, "add.overflow");
7108 } else {
7109 // For multiplication, the intrinsic is actually the canonical form
7110 Value *Call = Builder.CreateIntrinsic(Intrinsic::umul_with_overflow,
7111 ResultType, {ResultA, ResultB},
7112 /*FMFSource=*/nullptr, "umul");
7113 ArithResult = Builder.CreateExtractValue(Call, 0, "umul.value");
7114 OverflowCheck = Builder.CreateExtractValue(Call, 1, "umul.overflow");
7115 if (I.getPredicate() == ICmpInst::ICMP_ULT)
7116 OverflowCheck = Builder.CreateNot(OverflowCheck);
7117 }
7118
7119 IC.addToWorklist(Instr);
7120
7121 // Replace uses of the original add/mul result with the new arithmetic result
7122 if (Val->hasNUsesOrMore(2)) {
7123 for (User *U : make_early_inc_range(Val->users())) {
7124 if (U == &I)
7125 continue;
7126 if (TruncInst *TI = dyn_cast<TruncInst>(U)) {
7127 if (TI->getType()->getPrimitiveSizeInBits() == ResultWidth)
7128 IC.replaceInstUsesWith(*TI, ArithResult);
7129 else
7130 TI->setOperand(0, ArithResult);
7131 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) {
7132 assert(BO->getOpcode() == Instruction::And);
7133 // Replace (ArithResult & mask) --> zext (ArithResult & short_mask)
7134 ConstantInt *CI = cast<ConstantInt>(BO->getOperand(1));
7135 APInt ShortMask = CI->getValue().trunc(ResultWidth);
7136 Value *ShortAnd = Builder.CreateAnd(ArithResult, ShortMask);
7137 Value *Zext = Builder.CreateZExt(ShortAnd, BO->getType());
7138 IC.replaceInstUsesWith(*BO, Zext);
7139 } else {
7140 llvm_unreachable("Unexpected Binary operation");
7141 }
7143 }
7144 }
7145
7146 return IC.replaceInstUsesWith(I, OverflowCheck);
7147}
7148
7149/// When performing a comparison against a constant, it is possible that not all
7150/// the bits in the LHS are demanded. This helper method computes the mask that
7151/// IS demanded.
7153 const APInt *RHS;
7154 if (!match(I.getOperand(1), m_APInt(RHS)))
7156
7157 // If this is a normal comparison, it demands all bits. If it is a sign bit
7158 // comparison, it only demands the sign bit.
7159 bool UnusedBit;
7160 if (isSignBitCheck(I.getPredicate(), *RHS, UnusedBit))
7162
7163 switch (I.getPredicate()) {
7164 // For a UGT comparison, we don't care about any bits that
7165 // correspond to the trailing ones of the comparand. The value of these
7166 // bits doesn't impact the outcome of the comparison, because any value
7167 // greater than the RHS must differ in a bit higher than these due to carry.
7168 case ICmpInst::ICMP_UGT:
7169 return APInt::getBitsSetFrom(BitWidth, RHS->countr_one());
7170
7171 // Similarly, for a ULT comparison, we don't care about the trailing zeros.
7172 // Any value less than the RHS must differ in a higher bit because of carries.
7173 case ICmpInst::ICMP_ULT:
7174 return APInt::getBitsSetFrom(BitWidth, RHS->countr_zero());
7175
7176 default:
7178 }
7179}
7180
7181/// Check that one use is in the same block as the definition and all
7182/// other uses are in blocks dominated by a given block.
7183///
7184/// \param DI Definition
7185/// \param UI Use
7186/// \param DB Block that must dominate all uses of \p DI outside
7187/// the parent block
7188/// \return true when \p UI is the only use of \p DI in the parent block
7189/// and all other uses of \p DI are in blocks dominated by \p DB.
7190///
7192 const Instruction *UI,
7193 const BasicBlock *DB) const {
7194 assert(DI && UI && "Instruction not defined\n");
7195 // Ignore incomplete definitions.
7196 if (!DI->getParent())
7197 return false;
7198 // DI and UI must be in the same block.
7199 if (DI->getParent() != UI->getParent())
7200 return false;
7201 // Protect from self-referencing blocks.
7202 if (DI->getParent() == DB)
7203 return false;
7204 for (const User *U : DI->users()) {
7205 auto *Usr = cast<Instruction>(U);
7206 if (Usr != UI && !DT.dominates(DB, Usr->getParent()))
7207 return false;
7208 }
7209 return true;
7210}
7211
7212/// Return true when the instruction sequence within a block is select-cmp-br.
7214 const BasicBlock *BB = SI->getParent();
7215 if (!BB)
7216 return false;
7217 auto *BI = dyn_cast<CondBrInst>(BB->getTerminator());
7218 if (!BI)
7219 return false;
7220 auto *IC = dyn_cast<ICmpInst>(BI->getCondition());
7221 if (!IC || (IC->getOperand(0) != SI && IC->getOperand(1) != SI))
7222 return false;
7223 return true;
7224}
7225
7226/// True when a select result is replaced by one of its operands
7227/// in select-icmp sequence. This will eventually result in the elimination
7228/// of the select.
7229///
7230/// \param SI Select instruction
7231/// \param Icmp Compare instruction
7232/// \param SIOpd Operand that replaces the select
7233///
7234/// Notes:
7235/// - The replacement is global and requires dominator information
7236/// - The caller is responsible for the actual replacement
7237///
7238/// Example:
7239///
7240/// entry:
7241/// %4 = select i1 %3, %C* %0, %C* null
7242/// %5 = icmp eq %C* %4, null
7243/// br i1 %5, label %9, label %7
7244/// ...
7245/// ; <label>:7 ; preds = %entry
7246/// %8 = getelementptr inbounds %C* %4, i64 0, i32 0
7247/// ...
7248///
7249/// can be transformed to
7250///
7251/// %5 = icmp eq %C* %0, null
7252/// %6 = select i1 %3, i1 %5, i1 true
7253/// br i1 %6, label %9, label %7
7254/// ...
7255/// ; <label>:7 ; preds = %entry
7256/// %8 = getelementptr inbounds %C* %0, i64 0, i32 0 // replace by %0!
7257///
7258/// Similar when the first operand of the select is a constant or/and
7259/// the compare is for not equal rather than equal.
7260///
7261/// NOTE: The function is only called when the select and compare constants
7262/// are equal, the optimization can work only for EQ predicates. This is not a
7263/// major restriction since a NE compare should be 'normalized' to an equal
7264/// compare, which usually happens in the combiner and test case
7265/// select-cmp-br.ll checks for it.
7267 const ICmpInst *Icmp,
7268 const unsigned SIOpd) {
7269 assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!");
7271 BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(1);
7272 // The check for the single predecessor is not the best that can be
7273 // done. But it protects efficiently against cases like when SI's
7274 // home block has two successors, Succ and Succ1, and Succ1 predecessor
7275 // of Succ. Then SI can't be replaced by SIOpd because the use that gets
7276 // replaced can be reached on either path. So the uniqueness check
7277 // guarantees that the path all uses of SI (outside SI's parent) are on
7278 // is disjoint from all other paths out of SI. But that information
7279 // is more expensive to compute, and the trade-off here is in favor
7280 // of compile-time. It should also be noticed that we check for a single
7281 // predecessor and not only uniqueness. This to handle the situation when
7282 // Succ and Succ1 points to the same basic block.
7283 if (Succ->getSinglePredecessor() && dominatesAllUses(SI, Icmp, Succ)) {
7284 NumSel++;
7285 SI->replaceUsesOutsideBlock(SI->getOperand(SIOpd), SI->getParent());
7286 return true;
7287 }
7288 }
7289 return false;
7290}
7291
7292/// Try to fold the comparison based on range information we can get by checking
7293/// whether bits are known to be zero or one in the inputs.
7295 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
7296 Type *Ty = Op0->getType();
7297 ICmpInst::Predicate Pred = I.getPredicate();
7298
7299 // Get scalar or pointer size.
7300 unsigned BitWidth = Ty->isIntOrIntVectorTy()
7301 ? Ty->getScalarSizeInBits()
7302 : DL.getPointerTypeSizeInBits(Ty->getScalarType());
7303
7304 if (!BitWidth)
7305 return nullptr;
7306
7307 KnownBits Op0Known(BitWidth);
7308 KnownBits Op1Known(BitWidth);
7309
7310 {
7311 // Don't use dominating conditions when folding icmp using known bits. This
7312 // may convert signed into unsigned predicates in ways that other passes
7313 // (especially IndVarSimplify) may not be able to reliably undo.
7314 SimplifyQuery Q = SQ.getWithoutDomCondCache().getWithInstruction(&I);
7316 Op0Known, Q))
7317 return &I;
7318
7319 if (SimplifyDemandedBits(&I, 1, APInt::getAllOnes(BitWidth), Op1Known, Q))
7320 return &I;
7321 }
7322
7323 // If an unsigned samesign comparison is not poison, both operands have the
7324 // same sign bit. Propagate a known sign bit between the temporary KnownBits
7325 // values so the existing range folds can use that constraint.
7326 if (I.hasSameSign() && I.isUnsigned()) {
7327 auto PropagateSignBit = [](const KnownBits &From, KnownBits &To) {
7328 if (To.isNegative() || To.isNonNegative())
7329 return;
7330 if (From.isNegative())
7331 To.makeNegative();
7332 else if (From.isNonNegative())
7333 To.makeNonNegative();
7334 };
7335 PropagateSignBit(Op0Known, Op1Known);
7336 PropagateSignBit(Op1Known, Op0Known);
7337 }
7338
7339 if (!isa<Constant>(Op0) && Op0Known.isConstant())
7340 return new ICmpInst(
7341 Pred, ConstantExpr::getIntegerValue(Ty, Op0Known.getConstant()), Op1);
7342 if (!isa<Constant>(Op1) && Op1Known.isConstant())
7343 return new ICmpInst(
7344 Pred, Op0, ConstantExpr::getIntegerValue(Ty, Op1Known.getConstant()));
7345
7346 if (std::optional<bool> Res = ICmpInst::compare(Op0Known, Op1Known, Pred))
7347 return replaceInstUsesWith(I, ConstantInt::getBool(I.getType(), *Res));
7348
7349 // Given the known and unknown bits, compute a range that the LHS could be
7350 // in. Compute the Min, Max and RHS values based on the known bits. For the
7351 // EQ and NE we use unsigned values.
7352 APInt Op0Min(BitWidth, 0), Op0Max(BitWidth, 0);
7353 APInt Op1Min(BitWidth, 0), Op1Max(BitWidth, 0);
7354 if (I.isSigned()) {
7355 Op0Min = Op0Known.getSignedMinValue();
7356 Op0Max = Op0Known.getSignedMaxValue();
7357 Op1Min = Op1Known.getSignedMinValue();
7358 Op1Max = Op1Known.getSignedMaxValue();
7359 } else {
7360 Op0Min = Op0Known.getMinValue();
7361 Op0Max = Op0Known.getMaxValue();
7362 Op1Min = Op1Known.getMinValue();
7363 Op1Max = Op1Known.getMaxValue();
7364 }
7365
7366 // Don't break up a clamp pattern -- (min(max X, Y), Z) -- by replacing a
7367 // min/max canonical compare with some other compare. That could lead to
7368 // conflict with select canonicalization and infinite looping.
7369 // FIXME: This constraint may go away if min/max intrinsics are canonical.
7370 auto isMinMaxCmp = [&](Instruction &Cmp) {
7371 if (!Cmp.hasOneUse())
7372 return false;
7373 Value *A, *B;
7374 SelectPatternFlavor SPF = matchSelectPattern(Cmp.user_back(), A, B).Flavor;
7376 return false;
7377 return match(Op0, m_MaxOrMin(m_Value(), m_Value())) ||
7378 match(Op1, m_MaxOrMin(m_Value(), m_Value()));
7379 };
7380 if (!isMinMaxCmp(I)) {
7381 switch (Pred) {
7382 default:
7383 break;
7384 case ICmpInst::ICMP_ULT: {
7385 if (Op1Min == Op0Max) // A <u B -> A != B if max(A) == min(B)
7386 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
7387 const APInt *CmpC;
7388 if (match(Op1, m_APInt(CmpC))) {
7389 // A <u C -> A == C-1 if min(A)+1 == C
7390 if (*CmpC == Op0Min + 1)
7391 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
7392 ConstantInt::get(Op1->getType(), *CmpC - 1));
7393 // X <u C --> X == 0, if the number of zero bits in the bottom of X
7394 // exceeds the log2 of C.
7395 if (Op0Known.countMinTrailingZeros() >= CmpC->ceilLogBase2())
7396 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
7397 Constant::getNullValue(Op1->getType()));
7398 }
7399 break;
7400 }
7401 case ICmpInst::ICMP_UGT: {
7402 if (Op1Max == Op0Min) // A >u B -> A != B if min(A) == max(B)
7403 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
7404 const APInt *CmpC;
7405 if (match(Op1, m_APInt(CmpC))) {
7406 // A >u C -> A == C+1 if max(a)-1 == C
7407 if (*CmpC == Op0Max - 1)
7408 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
7409 ConstantInt::get(Op1->getType(), *CmpC + 1));
7410 // X >u C --> X != 0, if the number of zero bits in the bottom of X
7411 // exceeds the log2 of C.
7412 if (Op0Known.countMinTrailingZeros() >= CmpC->getActiveBits())
7413 return new ICmpInst(ICmpInst::ICMP_NE, Op0,
7414 Constant::getNullValue(Op1->getType()));
7415 }
7416 break;
7417 }
7418 case ICmpInst::ICMP_SLT: {
7419 if (Op1Min == Op0Max) // A <s B -> A != B if max(A) == min(B)
7420 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
7421 const APInt *CmpC;
7422 if (match(Op1, m_APInt(CmpC))) {
7423 if (*CmpC == Op0Min + 1) // A <s C -> A == C-1 if min(A)+1 == C
7424 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
7425 ConstantInt::get(Op1->getType(), *CmpC - 1));
7426 }
7427 break;
7428 }
7429 case ICmpInst::ICMP_SGT: {
7430 if (Op1Max == Op0Min) // A >s B -> A != B if min(A) == max(B)
7431 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
7432 const APInt *CmpC;
7433 if (match(Op1, m_APInt(CmpC))) {
7434 if (*CmpC == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C
7435 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
7436 ConstantInt::get(Op1->getType(), *CmpC + 1));
7437 }
7438 break;
7439 }
7440 }
7441 }
7442
7443 // Based on the range information we know about the LHS, see if we can
7444 // simplify this comparison. For example, (x&4) < 8 is always true.
7445 switch (Pred) {
7446 default:
7447 break;
7448 case ICmpInst::ICMP_EQ:
7449 case ICmpInst::ICMP_NE: {
7450 // If all bits are known zero except for one, then we know at most one bit
7451 // is set. If the comparison is against zero, then this is a check to see if
7452 // *that* bit is set.
7453 APInt Op0KnownZeroInverted = ~Op0Known.Zero;
7454 if (Op1Known.isZero()) {
7455 // If the LHS is an AND with the same constant, look through it.
7456 Value *LHS = nullptr;
7457 const APInt *LHSC;
7458 if (!match(Op0, m_And(m_Value(LHS), m_APInt(LHSC))) ||
7459 *LHSC != Op0KnownZeroInverted)
7460 LHS = Op0;
7461
7462 Value *X;
7463 const APInt *C1;
7464 if (match(LHS, m_Shl(m_Power2(C1), m_Value(X)))) {
7465 Type *XTy = X->getType();
7466 unsigned Log2C1 = C1->countr_zero();
7467 APInt C2 = Op0KnownZeroInverted;
7468 APInt C2Pow2 = (C2 & ~(*C1 - 1)) + *C1;
7469 if (C2Pow2.isPowerOf2()) {
7470 // iff (C1 is pow2) & ((C2 & ~(C1-1)) + C1) is pow2):
7471 // ((C1 << X) & C2) == 0 -> X >= (Log2(C2+C1) - Log2(C1))
7472 // ((C1 << X) & C2) != 0 -> X < (Log2(C2+C1) - Log2(C1))
7473 unsigned Log2C2 = C2Pow2.countr_zero();
7474 auto *CmpC = ConstantInt::get(XTy, Log2C2 - Log2C1);
7475 auto NewPred =
7477 return new ICmpInst(NewPred, X, CmpC);
7478 }
7479 }
7480 }
7481
7482 // Op0 eq C_Pow2 -> Op0 ne 0 if Op0 is known to be C_Pow2 or zero.
7483 if (Op1Known.isConstant() && Op1Known.getConstant().isPowerOf2() &&
7484 (Op0Known & Op1Known) == Op0Known)
7485 return new ICmpInst(CmpInst::getInversePredicate(Pred), Op0,
7486 ConstantInt::getNullValue(Op1->getType()));
7487 break;
7488 }
7489 case ICmpInst::ICMP_SGE:
7490 if (Op1Min == Op0Max) // A >=s B -> A == B if max(A) == min(B)
7491 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
7492 break;
7493 case ICmpInst::ICMP_SLE:
7494 if (Op1Max == Op0Min) // A <=s B -> A == B if min(A) == max(B)
7495 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
7496 break;
7497 case ICmpInst::ICMP_UGE:
7498 if (Op1Min == Op0Max) // A >=u B -> A == B if max(A) == min(B)
7499 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
7500 break;
7501 case ICmpInst::ICMP_ULE:
7502 if (Op1Max == Op0Min) // A <=u B -> A == B if min(A) == max(B)
7503 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
7504 break;
7505 }
7506
7507 // Turn a signed comparison into an unsigned one if both operands are known to
7508 // have the same sign. Set samesign if possible (except for equality
7509 // predicates).
7510 if ((I.isSigned() || (I.isUnsigned() && !I.hasSameSign())) &&
7511 ((Op0Known.Zero.isNegative() && Op1Known.Zero.isNegative()) ||
7512 (Op0Known.One.isNegative() && Op1Known.One.isNegative()))) {
7513 I.setPredicate(I.getUnsignedPredicate());
7514 I.setSameSign();
7515 return &I;
7516 }
7517
7518 return nullptr;
7519}
7520
7521/// If one operand of an icmp is effectively a bool (value range of {0,1}),
7522/// then try to reduce patterns based on that limit.
7524 Value *X, *Y;
7525 CmpPredicate Pred;
7526
7527 // X must be 0 and bool must be true for "ULT":
7528 // X <u (zext i1 Y) --> (X == 0) & Y
7529 if (match(&I, m_c_ICmp(Pred, m_Value(X), m_OneUse(m_ZExt(m_Value(Y))))) &&
7530 Y->getType()->isIntOrIntVectorTy(1) && Pred == ICmpInst::ICMP_ULT)
7531 return BinaryOperator::CreateAnd(Builder.CreateIsNull(X), Y);
7532
7533 // X must be 0 or bool must be true for "ULE":
7534 // X <=u (sext i1 Y) --> (X == 0) | Y
7535 if (match(&I, m_c_ICmp(Pred, m_Value(X), m_OneUse(m_SExt(m_Value(Y))))) &&
7536 Y->getType()->isIntOrIntVectorTy(1) && Pred == ICmpInst::ICMP_ULE)
7537 return BinaryOperator::CreateOr(Builder.CreateIsNull(X), Y);
7538
7539 // icmp eq/ne X, (zext/sext (icmp eq/ne X, C))
7540 CmpPredicate Pred1, Pred2;
7541 const APInt *C;
7542 Instruction *ExtI;
7543 if (match(&I, m_c_ICmp(Pred1, m_Value(X),
7546 m_APInt(C)))))) &&
7547 ICmpInst::isEquality(Pred1) && ICmpInst::isEquality(Pred2)) {
7548 bool IsSExt = ExtI->getOpcode() == Instruction::SExt;
7549 bool HasOneUse = ExtI->hasOneUse() && ExtI->getOperand(0)->hasOneUse();
7550 auto CreateRangeCheck = [&] {
7551 Value *CmpV1 =
7552 Builder.CreateICmp(Pred1, X, Constant::getNullValue(X->getType()));
7553 Value *CmpV2 = Builder.CreateICmp(
7554 Pred1, X, ConstantInt::getSigned(X->getType(), IsSExt ? -1 : 1));
7556 Pred1 == ICmpInst::ICMP_EQ ? Instruction::Or : Instruction::And,
7557 CmpV1, CmpV2);
7558 };
7559 if (C->isZero()) {
7560 if (Pred2 == ICmpInst::ICMP_EQ) {
7561 // icmp eq X, (zext/sext (icmp eq X, 0)) --> false
7562 // icmp ne X, (zext/sext (icmp eq X, 0)) --> true
7563 return replaceInstUsesWith(
7564 I, ConstantInt::getBool(I.getType(), Pred1 == ICmpInst::ICMP_NE));
7565 } else if (!IsSExt || HasOneUse) {
7566 // icmp eq X, (zext (icmp ne X, 0)) --> X == 0 || X == 1
7567 // icmp ne X, (zext (icmp ne X, 0)) --> X != 0 && X != 1
7568 // icmp eq X, (sext (icmp ne X, 0)) --> X == 0 || X == -1
7569 // icmp ne X, (sext (icmp ne X, 0)) --> X != 0 && X != -1
7570 return CreateRangeCheck();
7571 }
7572 } else if (IsSExt ? C->isAllOnes() : C->isOne()) {
7573 if (Pred2 == ICmpInst::ICMP_NE) {
7574 // icmp eq X, (zext (icmp ne X, 1)) --> false
7575 // icmp ne X, (zext (icmp ne X, 1)) --> true
7576 // icmp eq X, (sext (icmp ne X, -1)) --> false
7577 // icmp ne X, (sext (icmp ne X, -1)) --> true
7578 return replaceInstUsesWith(
7579 I, ConstantInt::getBool(I.getType(), Pred1 == ICmpInst::ICMP_NE));
7580 } else if (!IsSExt || HasOneUse) {
7581 // icmp eq X, (zext (icmp eq X, 1)) --> X == 0 || X == 1
7582 // icmp ne X, (zext (icmp eq X, 1)) --> X != 0 && X != 1
7583 // icmp eq X, (sext (icmp eq X, -1)) --> X == 0 || X == -1
7584 // icmp ne X, (sext (icmp eq X, -1)) --> X != 0 && X == -1
7585 return CreateRangeCheck();
7586 }
7587 } else {
7588 // when C != 0 && C != 1:
7589 // icmp eq X, (zext (icmp eq X, C)) --> icmp eq X, 0
7590 // icmp eq X, (zext (icmp ne X, C)) --> icmp eq X, 1
7591 // icmp ne X, (zext (icmp eq X, C)) --> icmp ne X, 0
7592 // icmp ne X, (zext (icmp ne X, C)) --> icmp ne X, 1
7593 // when C != 0 && C != -1:
7594 // icmp eq X, (sext (icmp eq X, C)) --> icmp eq X, 0
7595 // icmp eq X, (sext (icmp ne X, C)) --> icmp eq X, -1
7596 // icmp ne X, (sext (icmp eq X, C)) --> icmp ne X, 0
7597 // icmp ne X, (sext (icmp ne X, C)) --> icmp ne X, -1
7598 return ICmpInst::Create(
7599 Instruction::ICmp, Pred1, X,
7600 ConstantInt::getSigned(X->getType(), Pred2 == ICmpInst::ICMP_NE
7601 ? (IsSExt ? -1 : 1)
7602 : 0));
7603 }
7604 }
7605
7606 return nullptr;
7607}
7608
7609/// If we have an icmp le or icmp ge instruction with a constant operand, turn
7610/// it into the appropriate icmp lt or icmp gt instruction. This transform
7611/// allows them to be folded in visitICmpInst.
7613 CmpPredicate Pred = I.getCmpPredicate();
7616 return nullptr;
7617
7618 Value *Op0 = I.getOperand(0);
7619 Value *Op1 = I.getOperand(1);
7620 auto *Op1C = dyn_cast<Constant>(Op1);
7621 if (!Op1C)
7622 return nullptr;
7623
7624 auto FlippedStrictness = getFlippedStrictnessPredicateAndConstant(Pred, Op1C);
7625 if (!FlippedStrictness)
7626 return nullptr;
7627
7628 auto *NewCmp =
7629 new ICmpInst(FlippedStrictness->first, Op0, FlippedStrictness->second);
7630 NewCmp->setSameSign(FlippedStrictness->first.hasSameSign());
7631 return NewCmp;
7632}
7633
7634/// If we have a comparison with a non-canonical predicate, if we can update
7635/// all the users, invert the predicate and adjust all the users.
7637 // Is the predicate already canonical?
7638 CmpInst::Predicate Pred = I.getPredicate();
7640 return nullptr;
7641
7642 // Can all users be adjusted to predicate inversion?
7643 if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr))
7644 return nullptr;
7645
7646 // Ok, we can canonicalize comparison!
7647 // Let's first invert the comparison's predicate.
7648 I.setPredicate(CmpInst::getInversePredicate(Pred));
7649 I.setName(I.getName() + ".not");
7650
7651 // And, adapt users.
7653
7654 return &I;
7655}
7656
7657/// Integer compare with boolean values can always be turned into bitwise ops.
7659 InstCombiner::BuilderTy &Builder) {
7660 Value *A = I.getOperand(0), *B = I.getOperand(1);
7661 assert(A->getType()->isIntOrIntVectorTy(1) && "Bools only");
7662
7663 // A boolean compared to true/false can be simplified to Op0/true/false in
7664 // 14 out of the 20 (10 predicates * 2 constants) possible combinations.
7665 // Cases not handled by InstSimplify are always 'not' of Op0.
7666 if (match(B, m_Zero())) {
7667 switch (I.getPredicate()) {
7668 case CmpInst::ICMP_EQ: // A == 0 -> !A
7669 case CmpInst::ICMP_ULE: // A <=u 0 -> !A
7670 case CmpInst::ICMP_SGE: // A >=s 0 -> !A
7672 default:
7673 llvm_unreachable("ICmp i1 X, C not simplified as expected.");
7674 }
7675 } else if (match(B, m_One())) {
7676 switch (I.getPredicate()) {
7677 case CmpInst::ICMP_NE: // A != 1 -> !A
7678 case CmpInst::ICMP_ULT: // A <u 1 -> !A
7679 case CmpInst::ICMP_SGT: // A >s -1 -> !A
7681 default:
7682 llvm_unreachable("ICmp i1 X, C not simplified as expected.");
7683 }
7684 }
7685
7686 switch (I.getPredicate()) {
7687 default:
7688 llvm_unreachable("Invalid icmp instruction!");
7689 case ICmpInst::ICMP_EQ:
7690 // icmp eq i1 A, B -> ~(A ^ B)
7691 return BinaryOperator::CreateNot(Builder.CreateXor(A, B));
7692
7693 case ICmpInst::ICMP_NE:
7694 // icmp ne i1 A, B -> A ^ B
7695 return BinaryOperator::CreateXor(A, B);
7696
7697 case ICmpInst::ICMP_UGT:
7698 // icmp ugt -> icmp ult
7699 std::swap(A, B);
7700 [[fallthrough]];
7701 case ICmpInst::ICMP_ULT:
7702 // icmp ult i1 A, B -> ~A & B
7703 return BinaryOperator::CreateAnd(Builder.CreateNot(A), B);
7704
7705 case ICmpInst::ICMP_SGT:
7706 // icmp sgt -> icmp slt
7707 std::swap(A, B);
7708 [[fallthrough]];
7709 case ICmpInst::ICMP_SLT:
7710 // icmp slt i1 A, B -> A & ~B
7711 return BinaryOperator::CreateAnd(Builder.CreateNot(B), A);
7712
7713 case ICmpInst::ICMP_UGE:
7714 // icmp uge -> icmp ule
7715 std::swap(A, B);
7716 [[fallthrough]];
7717 case ICmpInst::ICMP_ULE:
7718 // icmp ule i1 A, B -> ~A | B
7719 return BinaryOperator::CreateOr(Builder.CreateNot(A), B);
7720
7721 case ICmpInst::ICMP_SGE:
7722 // icmp sge -> icmp sle
7723 std::swap(A, B);
7724 [[fallthrough]];
7725 case ICmpInst::ICMP_SLE:
7726 // icmp sle i1 A, B -> A | ~B
7727 return BinaryOperator::CreateOr(Builder.CreateNot(B), A);
7728 }
7729}
7730
7731// Transform pattern like:
7732// (1 << Y) u<= X or ~(-1 << Y) u< X or ((1 << Y)+(-1)) u< X
7733// (1 << Y) u> X or ~(-1 << Y) u>= X or ((1 << Y)+(-1)) u>= X
7734// Into:
7735// (X l>> Y) != 0
7736// (X l>> Y) == 0
7738 InstCombiner::BuilderTy &Builder) {
7739 CmpPredicate Pred, NewPred;
7740 Value *X, *Y;
7741 if (match(&Cmp,
7743 switch (Pred) {
7744 case ICmpInst::ICMP_ULE:
7745 NewPred = ICmpInst::ICMP_NE;
7746 break;
7747 case ICmpInst::ICMP_UGT:
7748 NewPred = ICmpInst::ICMP_EQ;
7749 break;
7750 default:
7751 return nullptr;
7752 }
7753 } else if (match(&Cmp, m_c_ICmp(Pred,
7756 m_Add(m_Shl(m_One(), m_Value(Y)),
7757 m_AllOnes()))),
7758 m_Value(X)))) {
7759 // The variant with 'add' is not canonical, (the variant with 'not' is)
7760 // we only get it because it has extra uses, and can't be canonicalized,
7761
7762 switch (Pred) {
7763 case ICmpInst::ICMP_ULT:
7764 NewPred = ICmpInst::ICMP_NE;
7765 break;
7766 case ICmpInst::ICMP_UGE:
7767 NewPred = ICmpInst::ICMP_EQ;
7768 break;
7769 default:
7770 return nullptr;
7771 }
7772 } else
7773 return nullptr;
7774
7775 Value *NewX = Builder.CreateLShr(X, Y, X->getName() + ".highbits");
7777 return CmpInst::Create(Instruction::ICmp, NewPred, NewX, Zero);
7778}
7779
7781 InstCombiner::BuilderTy &Builder) {
7782 const CmpInst::Predicate Pred = Cmp.getPredicate();
7783 Value *LHS = Cmp.getOperand(0), *RHS = Cmp.getOperand(1);
7784 Value *V1, *V2;
7785
7786 auto createCmpReverse = [&](CmpInst::Predicate Pred, Value *X, Value *Y) {
7787 Value *V = Builder.CreateCmp(Pred, X, Y, Cmp.getName());
7788 if (auto *I = dyn_cast<Instruction>(V))
7789 I->copyIRFlags(&Cmp);
7790 Module *M = Cmp.getModule();
7792 M, Intrinsic::vector_reverse, V->getType());
7793 return CallInst::Create(F, V);
7794 };
7795
7796 if (match(LHS, m_VecReverse(m_Value(V1)))) {
7797 // cmp Pred, rev(V1), rev(V2) --> rev(cmp Pred, V1, V2)
7798 if (match(RHS, m_VecReverse(m_Value(V2))) &&
7799 (LHS->hasOneUse() || RHS->hasOneUse()))
7800 return createCmpReverse(Pred, V1, V2);
7801
7802 // cmp Pred, rev(V1), RHSSplat --> rev(cmp Pred, V1, RHSSplat)
7803 if (LHS->hasOneUse() && isSplatValue(RHS))
7804 return createCmpReverse(Pred, V1, RHS);
7805 }
7806 // cmp Pred, LHSSplat, rev(V2) --> rev(cmp Pred, LHSSplat, V2)
7807 else if (isSplatValue(LHS) && match(RHS, m_OneUse(m_VecReverse(m_Value(V2)))))
7808 return createCmpReverse(Pred, LHS, V2);
7809
7810 ArrayRef<int> M;
7811 if (!match(LHS, m_Shuffle(m_Value(V1), m_Undef(), m_Mask(M))))
7812 return nullptr;
7813
7814 // If both arguments of the cmp are shuffles that use the same mask and
7815 // shuffle within a single vector, move the shuffle after the cmp:
7816 // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M
7817 Type *V1Ty = V1->getType();
7818 if (match(RHS, m_Shuffle(m_Value(V2), m_Undef(), m_SpecificMask(M))) &&
7819 V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) {
7820 Value *NewCmp = Builder.CreateCmp(Pred, V1, V2);
7821 return new ShuffleVectorInst(NewCmp, M);
7822 }
7823
7824 // Try to canonicalize compare with splatted operand and splat constant.
7825 // TODO: We could generalize this for more than splats. See/use the code in
7826 // InstCombiner::foldVectorBinop().
7827 Constant *C;
7828 if (!LHS->hasOneUse() || !match(RHS, m_Constant(C)))
7829 return nullptr;
7830
7831 // Length-changing splats are ok, so adjust the constants as needed:
7832 // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M
7833 Constant *ScalarC = C->getSplatValue(/* AllowPoison */ true);
7834 int MaskSplatIndex;
7835 if (ScalarC && match(M, m_SplatOrPoisonMask(MaskSplatIndex))) {
7836 // We allow poison in matching, but this transform removes it for safety.
7837 // Demanded elements analysis should be able to recover some/all of that.
7838 C = ConstantVector::getSplat(cast<VectorType>(V1Ty)->getElementCount(),
7839 ScalarC);
7840 SmallVector<int, 8> NewM(M.size(), MaskSplatIndex);
7841 Value *NewCmp = Builder.CreateCmp(Pred, V1, C);
7842 return new ShuffleVectorInst(NewCmp, NewM);
7843 }
7844
7845 return nullptr;
7846}
7847
7848// extract(uadd.with.overflow(A, B), 0) ult A
7849// -> extract(uadd.with.overflow(A, B), 1)
7851 CmpInst::Predicate Pred = I.getPredicate();
7852 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
7853
7854 Value *UAddOv;
7855 Value *A, *B;
7856 auto UAddOvResultPat = m_ExtractValue<0>(
7858 if (match(Op0, UAddOvResultPat) &&
7859 ((Pred == ICmpInst::ICMP_ULT && (Op1 == A || Op1 == B)) ||
7860 (Pred == ICmpInst::ICMP_EQ && match(Op1, m_ZeroInt()) &&
7861 (match(A, m_One()) || match(B, m_One()))) ||
7862 (Pred == ICmpInst::ICMP_NE && match(Op1, m_AllOnes()) &&
7863 (match(A, m_AllOnes()) || match(B, m_AllOnes())))))
7864 // extract(uadd.with.overflow(A, B), 0) < A
7865 // extract(uadd.with.overflow(A, 1), 0) == 0
7866 // extract(uadd.with.overflow(A, -1), 0) != -1
7867 UAddOv = cast<ExtractValueInst>(Op0)->getAggregateOperand();
7868 else if (match(Op1, UAddOvResultPat) && Pred == ICmpInst::ICMP_UGT &&
7869 (Op0 == A || Op0 == B))
7870 // A > extract(uadd.with.overflow(A, B), 0)
7871 UAddOv = cast<ExtractValueInst>(Op1)->getAggregateOperand();
7872 else
7873 return nullptr;
7874
7875 return ExtractValueInst::Create(UAddOv, 1);
7876}
7877
7879 if (!I.getOperand(0)->getType()->isPointerTy() ||
7881 I.getParent()->getParent(),
7882 I.getOperand(0)->getType()->getPointerAddressSpace())) {
7883 return nullptr;
7884 }
7885 Value *Ptr;
7886 if (match(I.getOperand(0),
7888 match(I.getOperand(1), m_Zero())) {
7889 return ICmpInst::Create(Instruction::ICmp, I.getPredicate(), Ptr,
7890 I.getOperand(1));
7891 }
7892 return nullptr;
7893}
7894
7896 IRBuilderBase &Builder) {
7897 if (!ICmpInst::isEquality(I.getPredicate()))
7898 return nullptr;
7899
7900 // The caller puts constants after non-constants.
7901 Value *Op = I.getOperand(0);
7902 Value *Const = I.getOperand(1);
7903
7904 // For Cond an equality condition, fold
7905 //
7906 // icmp (eq|ne) (vreduce_(or|and) Op), (Zero|AllOnes) ->
7907 // icmp (eq|ne) Op, (Zero|AllOnes)
7908 //
7909 // with a bitcast.
7910 Value *Vec;
7911 if ((match(Const, m_ZeroInt()) &&
7913 m_Value(Vec))))) ||
7914 (match(Const, m_AllOnes()) &&
7916 m_Value(Vec)))))) {
7917 auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
7918 if (!VecTy)
7919 return nullptr;
7920 Type *VecEltTy = VecTy->getElementType();
7921 unsigned ScalarBW =
7922 DL.getTypeSizeInBits(VecEltTy) * VecTy->getNumElements();
7923 if (!DL.fitsInLegalInteger(ScalarBW))
7924 return nullptr;
7925 Type *ScalarTy = IntegerType::get(I.getContext(), ScalarBW);
7926 Value *NewConst = match(Const, m_ZeroInt())
7927 ? ConstantInt::get(ScalarTy, 0)
7928 : ConstantInt::getAllOnesValue(ScalarTy);
7929 return CmpInst::Create(Instruction::ICmp, I.getPredicate(),
7930 Builder.CreateBitCast(Vec, ScalarTy), NewConst);
7931 }
7932 return nullptr;
7933}
7934
7935/// This function folds patterns produced by lowering of reduce idioms, such as
7936/// llvm.vector.reduce.and which are lowered into instruction chains. This code
7937/// attempts to generate fewer number of scalar comparisons instead of vector
7938/// comparisons when possible.
7940 InstCombiner::BuilderTy &Builder,
7941 const DataLayout &DL) {
7942 if (I.getType()->isVectorTy())
7943 return nullptr;
7944 CmpPredicate OuterPred, InnerPred;
7945 Value *LHS, *RHS;
7946
7947 // Match lowering of @llvm.vector.reduce.and. Turn
7948 /// %vec_ne = icmp ne <8 x i8> %lhs, %rhs
7949 /// %scalar_ne = bitcast <8 x i1> %vec_ne to i8
7950 /// %res = icmp <pred> i8 %scalar_ne, 0
7951 ///
7952 /// into
7953 ///
7954 /// %lhs.scalar = bitcast <8 x i8> %lhs to i64
7955 /// %rhs.scalar = bitcast <8 x i8> %rhs to i64
7956 /// %res = icmp <pred> i64 %lhs.scalar, %rhs.scalar
7957 ///
7958 /// for <pred> in {ne, eq}.
7959 if (!match(&I, m_ICmp(OuterPred,
7961 m_ICmp(InnerPred, m_Value(LHS), m_Value(RHS))))),
7962 m_Zero())))
7963 return nullptr;
7964 auto *LHSTy = dyn_cast<FixedVectorType>(LHS->getType());
7965 if (!LHSTy || !LHSTy->getElementType()->isIntegerTy())
7966 return nullptr;
7967 unsigned NumBits =
7968 LHSTy->getNumElements() * LHSTy->getElementType()->getIntegerBitWidth();
7969 // TODO: Relax this to "not wider than max legal integer type"?
7970 if (!DL.isLegalInteger(NumBits))
7971 return nullptr;
7972
7973 if (ICmpInst::isEquality(OuterPred) && InnerPred == ICmpInst::ICMP_NE) {
7974 auto *ScalarTy = Builder.getIntNTy(NumBits);
7975 LHS = Builder.CreateBitCast(LHS, ScalarTy, LHS->getName() + ".scalar");
7976 RHS = Builder.CreateBitCast(RHS, ScalarTy, RHS->getName() + ".scalar");
7977 return ICmpInst::Create(Instruction::ICmp, OuterPred, LHS, RHS,
7978 I.getName());
7979 }
7980
7981 return nullptr;
7982}
7983
7984// This helper will be called with icmp operands in both orders.
7986 Value *Op0, Value *Op1,
7987 ICmpInst &CtxI) {
7988 // Try to optimize 'icmp GEP, P' or 'icmp P, GEP'.
7989 if (auto *GEP = dyn_cast<GEPOperator>(Op0))
7990 if (Instruction *NI = foldGEPICmp(GEP, Op1, Pred, CtxI))
7991 return NI;
7992
7993 if (auto *SI = dyn_cast<SelectInst>(Op0))
7994 if (Instruction *NI = foldSelectICmp(Pred, SI, Op1, CtxI))
7995 return NI;
7996
7997 if (auto *MinMax = dyn_cast<MinMaxIntrinsic>(Op0)) {
7998 if (Instruction *Res = foldICmpWithMinMax(CtxI, MinMax, Op1, Pred))
7999 return Res;
8000
8001 if (Instruction *Res = foldICmpWithClamp(CtxI, Op1, MinMax))
8002 return Res;
8003 }
8004
8005 {
8006 Value *X;
8007 const APInt *C;
8008 // icmp X+Cst, X
8009 if (match(Op0, m_Add(m_Value(X), m_APInt(C))) && Op1 == X)
8010 return foldICmpAddOpConst(X, *C, Pred);
8011 }
8012
8013 // abs(X) >= X --> true
8014 // abs(X) u<= X --> true
8015 // abs(X) < X --> false
8016 // abs(X) u> X --> false
8017 // abs(X) u>= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
8018 // abs(X) <= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
8019 // abs(X) == X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
8020 // abs(X) u< X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
8021 // abs(X) > X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
8022 // abs(X) != X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
8023 {
8024 Value *X;
8025 Constant *C;
8027 match(Op1, m_Specific(X))) {
8028 Value *NullValue = Constant::getNullValue(X->getType());
8029 Value *AllOnesValue = Constant::getAllOnesValue(X->getType());
8030 const APInt SMin =
8031 APInt::getSignedMinValue(X->getType()->getScalarSizeInBits());
8032 bool IsIntMinPosion = C->isAllOnesValue();
8033 switch (Pred) {
8034 case CmpInst::ICMP_ULE:
8035 case CmpInst::ICMP_SGE:
8036 return replaceInstUsesWith(CtxI, ConstantInt::getTrue(CtxI.getType()));
8037 case CmpInst::ICMP_UGT:
8038 case CmpInst::ICMP_SLT:
8040 case CmpInst::ICMP_UGE:
8041 case CmpInst::ICMP_SLE:
8042 case CmpInst::ICMP_EQ: {
8043 return replaceInstUsesWith(
8044 CtxI, IsIntMinPosion
8045 ? Builder.CreateICmpSGT(X, AllOnesValue)
8046 : Builder.CreateICmpULT(
8047 X, ConstantInt::get(X->getType(), SMin + 1)));
8048 }
8049 case CmpInst::ICMP_ULT:
8050 case CmpInst::ICMP_SGT:
8051 case CmpInst::ICMP_NE: {
8052 return replaceInstUsesWith(
8053 CtxI, IsIntMinPosion
8054 ? Builder.CreateICmpSLT(X, NullValue)
8055 : Builder.CreateICmpUGT(
8056 X, ConstantInt::get(X->getType(), SMin)));
8057 }
8058 default:
8059 llvm_unreachable("Invalid predicate!");
8060 }
8061 }
8062 }
8063
8064 {
8065 // For a nonzero constant C:
8066 // usub.sat(X, C) == X --> X == 0
8067 // usub.sat(X, C) != X --> X != 0
8068 // usub.sat(X, C) < X --> X != 0
8070 m_NonZeroInt())) &&
8071 (CmpInst::isEquality(Pred) || Pred == ICmpInst::ICMP_ULT)) {
8072 ICmpInst::Predicate NewPred =
8073 CmpInst::isEquality(Pred) ? Pred.dropSameSign() : ICmpInst::ICMP_NE;
8074 return new ICmpInst(NewPred, Op1, Constant::getNullValue(Op1->getType()));
8075 }
8076 }
8077
8078 const SimplifyQuery Q = SQ.getWithInstruction(&CtxI);
8079 if (Value *V = foldICmpWithLowBitMaskedVal(Pred, Op0, Op1, Q, *this))
8080 return replaceInstUsesWith(CtxI, V);
8081
8082 // Folding (X / Y) pred X => X swap(pred) 0 for constant Y other than 0 or 1
8083 auto CheckUGT1 = [](const APInt &Divisor) { return Divisor.ugt(1); };
8084 {
8085 if (match(Op0, m_UDiv(m_Specific(Op1), m_CheckedInt(CheckUGT1)))) {
8086 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
8088 }
8089
8090 if (!ICmpInst::isUnsigned(Pred) &&
8091 match(Op0, m_SDiv(m_Specific(Op1), m_CheckedInt(CheckUGT1)))) {
8092 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
8094 }
8095 }
8096
8097 // Another case of this fold is (X >> Y) pred X => X swap(pred) 0 if Y != 0
8098 auto CheckNE0 = [](const APInt &Shift) { return !Shift.isZero(); };
8099 {
8100 if (match(Op0, m_LShr(m_Specific(Op1), m_CheckedInt(CheckNE0)))) {
8101 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
8103 }
8104
8105 if ((Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SGE) &&
8106 match(Op0, m_AShr(m_Specific(Op1), m_CheckedInt(CheckNE0)))) {
8107 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
8109 }
8110 }
8111
8112 // icmp (shl nsw/nuw X, L), (add nsw/nuw (shl nsw/nuw Y, L), K)
8113 // -> icmp X, (add nsw/nuw Y, K >> L)
8114 // We use AShr for nsw and LShr for nuw to safely peel off the shift.
8115 Value *X;
8116 uint64_t ShAmt;
8117 if (match(Op0, m_NUWShl(m_Value(X), m_ConstantInt(ShAmt))) &&
8118 !CtxI.isSigned()) {
8119 if (ShAmt >= X->getType()->getScalarSizeInBits())
8120 return nullptr;
8121 if (canEvaluateShifted(Op1, ShAmt, /*IsLeftShift=*/false,
8122 ShiftSemantics::Unsigned, &CtxI)) {
8123 Value *NewOp1 = getShiftedValue(Op1, ShAmt, /*IsLeftShift=*/false,
8125 return new ICmpInst(Pred, X, NewOp1);
8126 }
8127 }
8128
8129 if (match(Op0, m_NSWShl(m_Value(X), m_ConstantInt(ShAmt))) &&
8130 !CtxI.isUnsigned()) {
8131 if (ShAmt >= X->getType()->getScalarSizeInBits())
8132 return nullptr;
8133 if (canEvaluateShifted(Op1, ShAmt, /*IsLeftShift=*/false,
8134 ShiftSemantics::Signed, &CtxI)) {
8135 Value *NewOp1 = getShiftedValue(Op1, ShAmt, /*IsLeftShift=*/false,
8137 return new ICmpInst(Pred, X, NewOp1);
8138 }
8139 }
8140 return nullptr;
8141}
8142
8144 bool Changed = false;
8145 const SimplifyQuery Q = SQ.getWithInstruction(&I);
8146 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
8147 unsigned Op0Cplxity = getComplexity(Op0);
8148 unsigned Op1Cplxity = getComplexity(Op1);
8149
8150 /// Orders the operands of the compare so that they are listed from most
8151 /// complex to least complex. This puts constants before unary operators,
8152 /// before binary operators.
8153 if (Op0Cplxity < Op1Cplxity) {
8154 I.swapOperands();
8155 std::swap(Op0, Op1);
8156 Changed = true;
8157 }
8158
8159 if (Value *V = simplifyICmpInst(I.getCmpPredicate(), Op0, Op1, Q))
8160 return replaceInstUsesWith(I, V);
8161
8163 return Res;
8164
8165 if (Op0->getType()->isIntOrIntVectorTy(1))
8167 return Res;
8168
8170 return Res;
8171
8173 return Res;
8174
8176 return Res;
8177
8179 return Res;
8180
8182 return Res;
8183
8185 return Res;
8186
8188 return Res;
8189
8190 // Test if the ICmpInst instruction is used exclusively by a select as
8191 // part of a minimum or maximum operation. If so, refrain from doing
8192 // any other folding. This helps out other analyses which understand
8193 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
8194 // and CodeGen. And in this case, at least one of the comparison
8195 // operands has at least one user besides the compare (the select),
8196 // which would often largely negate the benefit of folding anyway.
8197 //
8198 // Do the same for the other patterns recognized by matchSelectPattern.
8199 if (I.hasOneUse())
8200 if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) {
8201 Value *A, *B;
8203 if (SPR.Flavor != SPF_UNKNOWN)
8204 return nullptr;
8205 }
8206
8207 // Do this after checking for min/max to prevent infinite looping.
8208 if (Instruction *Res = foldICmpWithZero(I))
8209 return Res;
8210
8211 Value *X;
8212 const APInt *C;
8213 if (I.getPredicate() == ICmpInst::ICMP_UGT &&
8214 match(Op0, m_UMax(m_Value(X), m_APInt(C))) &&
8215 match(Op1, m_Not(m_Specific(X)))) {
8216 if (C->isNonNegative())
8217 return new ICmpInst(ICmpInst::ICMP_SLT, X,
8218 Constant::getNullValue(X->getType()));
8219 return new ICmpInst(ICmpInst::ICMP_UGT, X,
8220 ConstantInt::get(X->getType(), ~*C));
8221 }
8222
8223 if (I.getPredicate() == ICmpInst::ICMP_ULT &&
8224 match(Op0, m_UMax(m_Value(X), m_APInt(C))) &&
8225 match(Op1, m_Not(m_Specific(X)))) {
8226 if (C->isNonNegative())
8227 return new ICmpInst(ICmpInst::ICMP_SGT, X,
8228 Constant::getAllOnesValue(X->getType()));
8229 return new ICmpInst(ICmpInst::ICMP_ULT, X,
8230 ConstantInt::get(X->getType(), ~*C));
8231 }
8232
8233 // FIXME: We only do this after checking for min/max to prevent infinite
8234 // looping caused by a reverse canonicalization of these patterns for min/max.
8235 // FIXME: The organization of folds is a mess. These would naturally go into
8236 // canonicalizeCmpWithConstant(), but we can't move all of the above folds
8237 // down here after the min/max restriction.
8238 ICmpInst::Predicate Pred = I.getPredicate();
8239 if (match(Op1, m_APInt(C))) {
8240 // For i32: x >u 2147483647 -> x <s 0 -> true if sign bit set
8241 if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) {
8243 return new ICmpInst(ICmpInst::ICMP_SLT, Op0, Zero);
8244 }
8245
8246 // For i32: x <u 2147483648 -> x >s -1 -> true if sign bit clear
8247 if (Pred == ICmpInst::ICMP_ULT && C->isMinSignedValue()) {
8249 return new ICmpInst(ICmpInst::ICMP_SGT, Op0, AllOnes);
8250 }
8251 }
8252
8253 // The folds in here may rely on wrapping flags and special constants, so
8254 // they can break up min/max idioms in some cases but not seemingly similar
8255 // patterns.
8256 // FIXME: It may be possible to enhance select folding to make this
8257 // unnecessary. It may also be moot if we canonicalize to min/max
8258 // intrinsics.
8259 if (Instruction *Res = foldICmpBinOp(I, Q))
8260 return Res;
8261
8263 return Res;
8264
8265 // Try to match comparison as a sign bit test. Intentionally do this after
8266 // foldICmpInstWithConstant() to potentially let other folds to happen first.
8267 if (Instruction *New = foldSignBitTest(I))
8268 return New;
8269
8270 if (auto *PN = dyn_cast<PHINode>(Op0))
8271 if (Instruction *NV = foldOpIntoPhi(I, PN))
8272 return NV;
8273 if (auto *PN = dyn_cast<PHINode>(Op1))
8274 if (Instruction *NV = foldOpIntoPhi(I, PN))
8275 return NV;
8276
8278 return Res;
8279
8280 if (Instruction *Res = foldICmpCommutative(I.getCmpPredicate(), Op0, Op1, I))
8281 return Res;
8282 if (Instruction *Res =
8283 foldICmpCommutative(I.getSwappedCmpPredicate(), Op1, Op0, I))
8284 return Res;
8285
8286 if (I.isCommutative()) {
8287 if (auto Pair = matchSymmetricPair(I.getOperand(0), I.getOperand(1))) {
8288 replaceOperand(I, 0, Pair->first);
8289 replaceOperand(I, 1, Pair->second);
8290 return &I;
8291 }
8292 }
8293
8294 // Fold icmp pred (select C1, TV1, FV1), (select C2, TV2, FV2)
8295 // when all select arms are constants, via truth table.
8297 return R;
8298
8299 // In case of a comparison with two select instructions having the same
8300 // condition, check whether one of the resulting branches can be simplified.
8301 // If so, just compare the other branch and select the appropriate result.
8302 // For example:
8303 // %tmp1 = select i1 %cmp, i32 %y, i32 %x
8304 // %tmp2 = select i1 %cmp, i32 %z, i32 %x
8305 // %cmp2 = icmp slt i32 %tmp2, %tmp1
8306 // The icmp will result false for the false value of selects and the result
8307 // will depend upon the comparison of true values of selects if %cmp is
8308 // true. Thus, transform this into:
8309 // %cmp = icmp slt i32 %y, %z
8310 // %sel = select i1 %cond, i1 %cmp, i1 false
8311 // This handles similar cases to transform.
8312 {
8313 Value *Cond, *A, *B, *C, *D;
8314 if (match(Op0, m_Select(m_Value(Cond), m_Value(A), m_Value(B))) &&
8316 (Op0->hasOneUse() || Op1->hasOneUse())) {
8317 // Check whether comparison of TrueValues can be simplified
8318 if (Value *Res = simplifyICmpInst(Pred, A, C, SQ)) {
8319 Value *NewICMP = Builder.CreateICmp(Pred, B, D);
8320 return SelectInst::Create(Cond, Res, NewICMP, /*NameStr=*/"",
8321 /*InsertBefore=*/nullptr,
8322 cast<Instruction>(Op0));
8323 }
8324 // Check whether comparison of FalseValues can be simplified
8325 if (Value *Res = simplifyICmpInst(Pred, B, D, SQ)) {
8326 Value *NewICMP = Builder.CreateICmp(Pred, A, C);
8327 return SelectInst::Create(Cond, NewICMP, Res, /*NameStr=*/"",
8328 /*InsertBefore=*/nullptr,
8329 cast<Instruction>(Op0));
8330 }
8331 }
8332 }
8333
8334 // icmp slt (sub nsw x, y), (add nsw x, y) --> icmp sgt y, 0
8335 // icmp ult (sub nuw x, y), (add nuw x, y) --> icmp ugt y, 0
8336 // icmp eq (sub nsw/nuw x, y), (add nsw/nuw x, y) --> icmp eq y, 0
8337 {
8338 Value *A, *B;
8339 CmpPredicate CmpPred;
8340 if (match(&I, m_c_ICmp(CmpPred, m_Sub(m_Value(A), m_Value(B)),
8342 auto *I0 = cast<OverflowingBinaryOperator>(Op0);
8343 auto *I1 = cast<OverflowingBinaryOperator>(Op1);
8344 bool I0NUW = I0->hasNoUnsignedWrap();
8345 bool I1NUW = I1->hasNoUnsignedWrap();
8346 bool I0NSW = I0->hasNoSignedWrap();
8347 bool I1NSW = I1->hasNoSignedWrap();
8348 if ((ICmpInst::isUnsigned(Pred) && I0NUW && I1NUW) ||
8349 (ICmpInst::isSigned(Pred) && I0NSW && I1NSW) ||
8350 (ICmpInst::isEquality(Pred) &&
8351 ((I0NUW || I0NSW) && (I1NUW || I1NSW)))) {
8352 return new ICmpInst(CmpPredicate::getSwapped(CmpPred), B,
8353 ConstantInt::get(Op0->getType(), 0));
8354 }
8355 }
8356 }
8357
8358 // Try to optimize equality comparisons against alloca-based pointers.
8359 if (Op0->getType()->isPointerTy() && I.isEquality()) {
8360 assert(Op1->getType()->isPointerTy() &&
8361 "Comparing pointer with non-pointer?");
8362 if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0)))
8363 if (foldAllocaCmp(Alloca))
8364 return nullptr;
8365 if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op1)))
8366 if (foldAllocaCmp(Alloca))
8367 return nullptr;
8368 }
8369
8370 if (Instruction *Res = foldICmpBitCast(I))
8371 return Res;
8372
8373 // TODO: Hoist this above the min/max bailout.
8375 return R;
8376
8377 // icmp (zext X), (and (trunc Y), Mask) -> icmp X, trunc Y IFF Mask exactly
8378 // covers the bits of X
8379 {
8380 Value *Y;
8381 const APInt *Mask;
8382 if (match(I.getOperand(1), m_ZExt(m_Value(X))) &&
8383 match(I.getOperand(0),
8384 m_OneUse(m_And(m_Trunc(m_Value(Y)), m_APInt(Mask))))) {
8385 Type *SmallType = X->getType();
8386 unsigned SmallWidth = SmallType->getScalarSizeInBits();
8387 if (Mask->isMask(SmallWidth) &&
8388 shouldChangeType(I.getOperand(0)->getType(), SmallType)) {
8389 Value *NewTrunc = Builder.CreateTrunc(Y, SmallType);
8390 return new ICmpInst(I.getUnsignedPredicate(), NewTrunc, X);
8391 }
8392 }
8393 }
8394
8395 {
8396 Value *X, *Y;
8397 // Transform (X & ~Y) == 0 --> (X & Y) != 0
8398 // and (X & ~Y) != 0 --> (X & Y) == 0
8399 // if A is a power of 2.
8400 if (match(Op0, m_And(m_Value(X), m_Not(m_Value(Y)))) &&
8401 match(Op1, m_Zero()) && isKnownToBeAPowerOfTwo(X, false, &I) &&
8402 I.isEquality())
8403 return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(X, Y),
8404 Op1);
8405
8406 // Op0 pred Op1 -> ~Op1 pred ~Op0, if this allows us to drop an instruction.
8407 if (Op0->getType()->isIntOrIntVectorTy()) {
8408 bool ConsumesOp0, ConsumesOp1;
8409 if (isFreeToInvert(Op0, Op0->hasOneUse(), ConsumesOp0) &&
8410 isFreeToInvert(Op1, Op1->hasOneUse(), ConsumesOp1) &&
8411 (ConsumesOp0 || ConsumesOp1)) {
8412 Value *InvOp0 = getFreelyInverted(Op0, Op0->hasOneUse(), &Builder);
8413 Value *InvOp1 = getFreelyInverted(Op1, Op1->hasOneUse(), &Builder);
8414 assert(InvOp0 && InvOp1 &&
8415 "Mismatch between isFreeToInvert and getFreelyInverted");
8416 return new ICmpInst(I.getSwappedPredicate(), InvOp0, InvOp1);
8417 }
8418 }
8419
8420 Instruction *AddI = nullptr;
8422 m_Instruction(AddI))) &&
8423 isa<IntegerType>(X->getType())) {
8424 Value *Result;
8425 Constant *Overflow;
8426 // m_UAddWithOverflow can match patterns that do not include an explicit
8427 // "add" instruction, so check the opcode of the matched op.
8428 if (AddI->getOpcode() == Instruction::Add &&
8429 OptimizeOverflowCheck(Instruction::Add, /*Signed*/ false, X, Y, *AddI,
8430 Result, Overflow)) {
8431 replaceInstUsesWith(*AddI, Result);
8432 eraseInstFromFunction(*AddI);
8433 return replaceInstUsesWith(I, Overflow);
8434 }
8435 }
8436
8437 // (zext X) + (zext Y) --> add + overflow check.
8438 // (zext X) * (zext Y) --> llvm.umul.with.overflow.
8439 if ((match(Op0, m_NUWAdd(m_ZExt(m_Value(X)), m_ZExt(m_Value(Y)))) ||
8440 match(Op0, m_NUWMul(m_ZExt(m_Value(X)), m_ZExt(m_Value(Y))))) &&
8441 match(Op1, m_APInt(C))) {
8442 if (Instruction *R = processUZExtIdiom(I, Op0, C, *this))
8443 return R;
8444 }
8445
8446 // Signbit test folds
8447 // Fold (X u>> BitWidth - 1 Pred ZExt(i1)) --> X s< 0 Pred i1
8448 // Fold (X s>> BitWidth - 1 Pred SExt(i1)) --> X s< 0 Pred i1
8449 Instruction *ExtI;
8450 if ((I.isUnsigned() || I.isEquality()) &&
8451 match(Op1,
8453 Y->getType()->getScalarSizeInBits() == 1 &&
8454 (Op0->hasOneUse() || Op1->hasOneUse())) {
8455 unsigned OpWidth = Op0->getType()->getScalarSizeInBits();
8456 Instruction *ShiftI;
8457 if (match(Op0, m_CombineAnd(m_Instruction(ShiftI),
8459 OpWidth - 1))))) {
8460 unsigned ExtOpc = ExtI->getOpcode();
8461 unsigned ShiftOpc = ShiftI->getOpcode();
8462 if ((ExtOpc == Instruction::ZExt && ShiftOpc == Instruction::LShr) ||
8463 (ExtOpc == Instruction::SExt && ShiftOpc == Instruction::AShr)) {
8464 Value *SLTZero =
8465 Builder.CreateICmpSLT(X, Constant::getNullValue(X->getType()));
8466 Value *Cmp = Builder.CreateICmp(Pred, SLTZero, Y, I.getName());
8467 return replaceInstUsesWith(I, Cmp);
8468 }
8469 }
8470 }
8471 }
8472
8473 if (Instruction *Res = foldICmpEquality(I))
8474 return Res;
8475
8477 return Res;
8478
8479 if (Instruction *Res = foldICmpOfUAddOv(I))
8480 return Res;
8481
8483 return Res;
8484
8485 // The 'cmpxchg' instruction returns an aggregate containing the old value and
8486 // an i1 which indicates whether or not we successfully did the swap.
8487 //
8488 // Replace comparisons between the old value and the expected value with the
8489 // indicator that 'cmpxchg' returns.
8490 //
8491 // N.B. This transform is only valid when the 'cmpxchg' is not permitted to
8492 // spuriously fail. In those cases, the old value may equal the expected
8493 // value but it is possible for the swap to not occur.
8494 if (I.getPredicate() == ICmpInst::ICMP_EQ)
8495 if (auto *EVI = dyn_cast<ExtractValueInst>(Op0))
8496 if (auto *ACXI = dyn_cast<AtomicCmpXchgInst>(EVI->getAggregateOperand()))
8497 if (EVI->getIndices()[0] == 0 && ACXI->getCompareOperand() == Op1 &&
8498 !ACXI->isWeak())
8499 return ExtractValueInst::Create(ACXI, 1);
8500
8502 return Res;
8503
8504 if (I.getType()->isVectorTy())
8505 if (Instruction *Res = foldVectorCmp(I, Builder))
8506 return Res;
8507
8509 return Res;
8510
8512 return Res;
8513
8514 {
8515 Value *A;
8516 const APInt *C1, *C2;
8517 ICmpInst::Predicate Pred = I.getPredicate();
8518 if (ICmpInst::isEquality(Pred)) {
8519 // sext(a) & c1 == c2 --> a & c3 == trunc(c2)
8520 // sext(a) & c1 != c2 --> a & c3 != trunc(c2)
8521 if (match(Op0, m_And(m_SExt(m_Value(A)), m_APInt(C1))) &&
8522 match(Op1, m_APInt(C2))) {
8523 Type *InputTy = A->getType();
8524 unsigned InputBitWidth = InputTy->getScalarSizeInBits();
8525 // c2 must be non-negative at the bitwidth of a.
8526 if (C2->getActiveBits() < InputBitWidth) {
8527 APInt TruncC1 = C1->trunc(InputBitWidth);
8528 // Check if there are 1s in C1 high bits of size InputBitWidth.
8529 if (C1->uge(APInt::getOneBitSet(C1->getBitWidth(), InputBitWidth)))
8530 TruncC1.setBit(InputBitWidth - 1);
8531 Value *AndInst = Builder.CreateAnd(A, TruncC1);
8532 return new ICmpInst(
8533 Pred, AndInst,
8534 ConstantInt::get(InputTy, C2->trunc(InputBitWidth)));
8535 }
8536 }
8537 }
8538 }
8539
8540 return Changed ? &I : nullptr;
8541}
8542
8543/// Fold fcmp ([us]itofp x, cst) if possible.
8545 Instruction *LHSI,
8546 Constant *RHSC) {
8547 const APFloat *RHS;
8548 if (!match(RHSC, m_APFloat(RHS)))
8549 return nullptr;
8550
8551 // Get the width of the mantissa. We don't want to hack on conversions that
8552 // might lose information from the integer, e.g. "i64 -> float"
8553 int MantissaWidth = LHSI->getType()->getFPMantissaWidth();
8554 if (MantissaWidth == -1)
8555 return nullptr; // Unknown.
8556
8557 Type *IntTy = LHSI->getOperand(0)->getType();
8558 unsigned IntWidth = IntTy->getScalarSizeInBits();
8559 bool LHSUnsigned = isa<UIToFPInst>(LHSI);
8560
8561 if (I.isEquality()) {
8562 FCmpInst::Predicate P = I.getPredicate();
8563 bool IsExact = false;
8564 APSInt RHSCvt(IntWidth, LHSUnsigned);
8565 RHS->convertToInteger(RHSCvt, APFloat::rmNearestTiesToEven, &IsExact);
8566
8567 // If the floating point constant isn't an integer value, we know if we will
8568 // ever compare equal / not equal to it.
8569 if (!IsExact) {
8570 // TODO: Can never be -0.0 and other non-representable values
8571 APFloat RHSRoundInt(*RHS);
8573 if (*RHS != RHSRoundInt) {
8575 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8576
8578 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8579 }
8580 }
8581
8582 // TODO: If the constant is exactly representable, is it always OK to do
8583 // equality compares as integer?
8584 }
8585
8586 // Check to see that the input is converted from an integer type that is small
8587 // enough that preserves all bits. TODO: check here for "known" sign bits.
8588 // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e.
8589
8590 // Following test does NOT adjust IntWidth downwards for signed inputs,
8591 // because the most negative value still requires all the mantissa bits
8592 // to distinguish it from one less than that value.
8593 if ((int)IntWidth > MantissaWidth) {
8594 // Conversion would lose accuracy. Check if loss can impact comparison.
8595 int Exp = ilogb(*RHS);
8596 if (Exp == APFloat::IEK_Inf) {
8597 int MaxExponent = ilogb(APFloat::getLargest(RHS->getSemantics()));
8598 if (MaxExponent < (int)IntWidth - !LHSUnsigned)
8599 // Conversion could create infinity.
8600 return nullptr;
8601 } else {
8602 // Note that if RHS is zero or NaN, then Exp is negative
8603 // and first condition is trivially false.
8604 if (MantissaWidth <= Exp && Exp <= (int)IntWidth - !LHSUnsigned)
8605 // Conversion could affect comparison.
8606 return nullptr;
8607 }
8608 }
8609
8610 // Otherwise, we can potentially simplify the comparison. We know that it
8611 // will always come through as an integer value and we know the constant is
8612 // not a NAN (it would have been previously simplified).
8613 assert(!RHS->isNaN() && "NaN comparison not already folded!");
8614
8616 switch (I.getPredicate()) {
8617 default:
8618 llvm_unreachable("Unexpected predicate!");
8619 case FCmpInst::FCMP_UEQ:
8620 case FCmpInst::FCMP_OEQ:
8621 Pred = ICmpInst::ICMP_EQ;
8622 break;
8623 case FCmpInst::FCMP_UGT:
8624 case FCmpInst::FCMP_OGT:
8625 Pred = LHSUnsigned ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_SGT;
8626 break;
8627 case FCmpInst::FCMP_UGE:
8628 case FCmpInst::FCMP_OGE:
8629 Pred = LHSUnsigned ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_SGE;
8630 break;
8631 case FCmpInst::FCMP_ULT:
8632 case FCmpInst::FCMP_OLT:
8633 Pred = LHSUnsigned ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_SLT;
8634 break;
8635 case FCmpInst::FCMP_ULE:
8636 case FCmpInst::FCMP_OLE:
8637 Pred = LHSUnsigned ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_SLE;
8638 break;
8639 case FCmpInst::FCMP_UNE:
8640 case FCmpInst::FCMP_ONE:
8641 Pred = ICmpInst::ICMP_NE;
8642 break;
8643 case FCmpInst::FCMP_ORD:
8644 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8645 case FCmpInst::FCMP_UNO:
8646 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8647 }
8648
8649 // Now we know that the APFloat is a normal number, zero or inf.
8650
8651 // See if the FP constant is too large for the integer. For example,
8652 // comparing an i8 to 300.0.
8653 if (!LHSUnsigned) {
8654 // If the RHS value is > SignedMax, fold the comparison. This handles +INF
8655 // and large values.
8656 APFloat SMax(RHS->getSemantics());
8657 SMax.convertFromAPInt(APInt::getSignedMaxValue(IntWidth), true,
8659 if (SMax < *RHS) { // smax < 13123.0
8660 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT ||
8661 Pred == ICmpInst::ICMP_SLE)
8662 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8663 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8664 }
8665 } else {
8666 // If the RHS value is > UnsignedMax, fold the comparison. This handles
8667 // +INF and large values.
8668 APFloat UMax(RHS->getSemantics());
8669 UMax.convertFromAPInt(APInt::getMaxValue(IntWidth), false,
8671 if (UMax < *RHS) { // umax < 13123.0
8672 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT ||
8673 Pred == ICmpInst::ICMP_ULE)
8674 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8675 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8676 }
8677 }
8678
8679 if (!LHSUnsigned) {
8680 // See if the RHS value is < SignedMin.
8681 APFloat SMin(RHS->getSemantics());
8682 SMin.convertFromAPInt(APInt::getSignedMinValue(IntWidth), true,
8684 if (SMin > *RHS) { // smin > 12312.0
8685 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT ||
8686 Pred == ICmpInst::ICMP_SGE)
8687 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8688 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8689 }
8690 } else {
8691 // See if the RHS value is < UnsignedMin.
8692 APFloat UMin(RHS->getSemantics());
8693 UMin.convertFromAPInt(APInt::getMinValue(IntWidth), false,
8695 if (UMin > *RHS) { // umin > 12312.0
8696 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT ||
8697 Pred == ICmpInst::ICMP_UGE)
8698 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8699 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8700 }
8701 }
8702
8703 // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or
8704 // [0, UMAX], but it may still be fractional. Check whether this is the case
8705 // using the IsExact flag.
8706 // Don't do this for zero, because -0.0 is not fractional.
8707 APSInt RHSInt(IntWidth, LHSUnsigned);
8708 bool IsExact;
8709 RHS->convertToInteger(RHSInt, APFloat::rmTowardZero, &IsExact);
8710 if (!RHS->isZero()) {
8711 if (!IsExact) {
8712 // If we had a comparison against a fractional value, we have to adjust
8713 // the compare predicate and sometimes the value. RHSC is rounded towards
8714 // zero at this point.
8715 switch (Pred) {
8716 default:
8717 llvm_unreachable("Unexpected integer comparison!");
8718 case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true
8719 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8720 case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false
8721 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8722 case ICmpInst::ICMP_ULE:
8723 // (float)int <= 4.4 --> int <= 4
8724 // (float)int <= -4.4 --> false
8725 if (RHS->isNegative())
8726 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8727 break;
8728 case ICmpInst::ICMP_SLE:
8729 // (float)int <= 4.4 --> int <= 4
8730 // (float)int <= -4.4 --> int < -4
8731 if (RHS->isNegative())
8732 Pred = ICmpInst::ICMP_SLT;
8733 break;
8734 case ICmpInst::ICMP_ULT:
8735 // (float)int < -4.4 --> false
8736 // (float)int < 4.4 --> int <= 4
8737 if (RHS->isNegative())
8738 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8739 Pred = ICmpInst::ICMP_ULE;
8740 break;
8741 case ICmpInst::ICMP_SLT:
8742 // (float)int < -4.4 --> int < -4
8743 // (float)int < 4.4 --> int <= 4
8744 if (!RHS->isNegative())
8745 Pred = ICmpInst::ICMP_SLE;
8746 break;
8747 case ICmpInst::ICMP_UGT:
8748 // (float)int > 4.4 --> int > 4
8749 // (float)int > -4.4 --> true
8750 if (RHS->isNegative())
8751 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8752 break;
8753 case ICmpInst::ICMP_SGT:
8754 // (float)int > 4.4 --> int > 4
8755 // (float)int > -4.4 --> int >= -4
8756 if (RHS->isNegative())
8757 Pred = ICmpInst::ICMP_SGE;
8758 break;
8759 case ICmpInst::ICMP_UGE:
8760 // (float)int >= -4.4 --> true
8761 // (float)int >= 4.4 --> int > 4
8762 if (RHS->isNegative())
8763 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8764 Pred = ICmpInst::ICMP_UGT;
8765 break;
8766 case ICmpInst::ICMP_SGE:
8767 // (float)int >= -4.4 --> int >= -4
8768 // (float)int >= 4.4 --> int > 4
8769 if (!RHS->isNegative())
8770 Pred = ICmpInst::ICMP_SGT;
8771 break;
8772 }
8773 }
8774 }
8775
8776 // Lower this FP comparison into an appropriate integer version of the
8777 // comparison.
8778 return new ICmpInst(Pred, LHSI->getOperand(0),
8779 ConstantInt::get(LHSI->getOperand(0)->getType(), RHSInt));
8780}
8781
8782/// Fold fcmp/icmp pred (select C1, TV1, FV1), (select C2, TV2, FV2)
8783/// where all true/false values are constants that allow the compare to be
8784/// constant-folded for every combination of C1 and C2.
8785/// We compute a 4-entry truth table and use createLogicFromTable to
8786/// synthesize a boolean expression of C1 and C2.
8788 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
8789 Value *C1, *C2;
8790 Constant *TV1, *FV1, *TV2, *FV2;
8791
8792 if (!match(Op0, m_Select(m_Value(C1), m_Constant(TV1), m_Constant(FV1))) ||
8793 !match(Op1, m_Select(m_Value(C2), m_Constant(TV2), m_Constant(FV2))))
8794 return nullptr;
8795
8796 if (I.getType() != C1->getType() || I.getType() != C2->getType())
8797 return nullptr;
8798
8799 unsigned Pred = I.getPredicate();
8800 const DataLayout &DL = I.getDataLayout();
8801
8802 Constant *Res00 = ConstantFoldCompareInstOperands(Pred, FV1, FV2, DL);
8803 Constant *Res01 = ConstantFoldCompareInstOperands(Pred, FV1, TV2, DL);
8804 Constant *Res10 = ConstantFoldCompareInstOperands(Pred, TV1, FV2, DL);
8805 Constant *Res11 = ConstantFoldCompareInstOperands(Pred, TV1, TV2, DL);
8806
8807 if (!Res00 || !Res01 || !Res10 || !Res11)
8808 return nullptr;
8809
8810 if ((!Res00->isNullValue() && !Res00->isAllOnesValue()) ||
8811 (!Res01->isNullValue() && !Res01->isAllOnesValue()) ||
8812 (!Res10->isNullValue() && !Res10->isAllOnesValue()) ||
8813 (!Res11->isNullValue() && !Res11->isAllOnesValue()))
8814 return nullptr;
8815
8816 std::bitset<4> Table;
8817 if (!Res00->isNullValue())
8818 Table.set(0);
8819 if (!Res01->isNullValue())
8820 Table.set(1);
8821 if (!Res10->isNullValue())
8822 Table.set(2);
8823 if (!Res11->isNullValue())
8824 Table.set(3);
8825
8827 Op0->hasOneUse() && Op1->hasOneUse());
8828 if (!Res)
8829 return nullptr;
8830 return replaceInstUsesWith(I, Res);
8831}
8832
8833/// Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary.
8835 Constant *RHSC) {
8836 // When C is not 0.0 and infinities are not allowed:
8837 // (C / X) < 0.0 is a sign-bit test of X
8838 // (C / X) < 0.0 --> X < 0.0 (if C is positive)
8839 // (C / X) < 0.0 --> X > 0.0 (if C is negative, swap the predicate)
8840 //
8841 // Proof:
8842 // Multiply (C / X) < 0.0 by X * X / C.
8843 // - X is non zero, if it is the flag 'ninf' is violated.
8844 // - C defines the sign of X * X * C. Thus it also defines whether to swap
8845 // the predicate. C is also non zero by definition.
8846 //
8847 // Thus X * X / C is non zero and the transformation is valid. [qed]
8848
8849 FCmpInst::Predicate Pred = I.getPredicate();
8850
8851 // Check that predicates are valid.
8854 return nullptr;
8855
8856 // Check that RHS operand is zero.
8857 if (!match(RHSC, m_AnyZeroFP()))
8858 return nullptr;
8859
8860 // Check fastmath flags ('ninf').
8861 if (!LHSI->hasNoInfs() || !I.hasNoInfs())
8862 return nullptr;
8863
8864 // Check the properties of the dividend. It must not be zero to avoid a
8865 // division by zero (see Proof).
8866 const APFloat *C;
8867 if (!match(LHSI->getOperand(0), m_APFloat(C)))
8868 return nullptr;
8869
8870 if (C->isZero())
8871 return nullptr;
8872
8873 // Get swapped predicate if necessary.
8874 if (C->isNegative())
8875 Pred = I.getSwappedPredicate();
8876
8877 return new FCmpInst(Pred, LHSI->getOperand(1), RHSC, "", &I);
8878}
8879
8880// Transform 'fptrunc(x) cmp C' to 'x cmp ext(C)' if possible.
8881// Patterns include:
8882// fptrunc(x) < C --> x < ext(C)
8883// fptrunc(x) <= C --> x <= ext(C)
8884// fptrunc(x) > C --> x > ext(C)
8885// fptrunc(x) >= C --> x >= ext(C)
8886// fptrunc(x) ord/uno C --> x ord/uno 0
8887// where 'ext(C)' is the extension of 'C' to the type of 'x' with a small bias
8888// due to precision loss.
8890 const Constant &C) {
8891 FCmpInst::Predicate Pred = I.getPredicate();
8892 Type *DestType = FPTrunc.getOperand(0)->getType();
8893
8894 const APFloat *CValue;
8895 // TODO: support vec
8896 if (!match(&C, m_APFloat(CValue)))
8897 return nullptr;
8898
8899 // Handle ord/uno
8901 assert(!CValue->isNaN() &&
8902 "X ord/uno NaN should be folded away by simplifyFCmpInst()");
8903 return new FCmpInst(Pred, FPTrunc.getOperand(0),
8904 ConstantFP::getZero(DestType), "", &I);
8905 }
8906
8907 // Handle <, >, <=, >=
8908 bool RoundDown = false;
8909
8912 RoundDown = true;
8913 else if (Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_UGT ||
8915 RoundDown = false;
8916 else
8917 return nullptr;
8918
8919 if (CValue->isNaN() || CValue->isInfinity())
8920 return nullptr;
8921
8922 auto ConvertFltSema = [](const APFloat &Src, const fltSemantics &Sema) {
8923 bool LosesInfo;
8924 APFloat Dest = Src;
8925 Dest.convert(Sema, APFloat::rmNearestTiesToEven, &LosesInfo);
8926 return Dest;
8927 };
8928
8929 auto NextValue = [](const APFloat &Value, bool RoundDown) {
8930 APFloat NextValue = Value;
8931 NextValue.next(RoundDown);
8932 return NextValue;
8933 };
8934
8935 APFloat NextCValue = NextValue(*CValue, RoundDown);
8936
8937 const fltSemantics &DestFltSema =
8938 DestType->getScalarType()->getFltSemantics();
8939
8940 APFloat ExtCValue = ConvertFltSema(*CValue, DestFltSema);
8941 APFloat ExtNextCValue = ConvertFltSema(NextCValue, DestFltSema);
8942
8943 // When 'NextCValue' is infinity, use an imaged 'NextCValue' that equals
8944 // 'CValue + bias' to avoid the infinity after conversion. The bias is
8945 // estimated as 'CValue - PrevCValue', where 'PrevCValue' is the previous
8946 // value of 'CValue'.
8947 if (NextCValue.isInfinity()) {
8948 APFloat PrevCValue = NextValue(*CValue, !RoundDown);
8949 APFloat Bias = ConvertFltSema(*CValue - PrevCValue, DestFltSema);
8950
8951 ExtNextCValue = ExtCValue + Bias;
8952 }
8953
8954 APFloat ExtMidValue =
8955 scalbn(ExtCValue + ExtNextCValue, -1, APFloat::rmNearestTiesToEven);
8956
8957 const fltSemantics &SrcFltSema =
8958 C.getType()->getScalarType()->getFltSemantics();
8959
8960 // 'MidValue' might be rounded to 'NextCValue'. Correct it here.
8961 APFloat MidValue = ConvertFltSema(ExtMidValue, SrcFltSema);
8962 if (MidValue != *CValue)
8963 ExtMidValue.next(!RoundDown);
8964
8965 // Check whether 'ExtMidValue' is a valid result since the assumption on
8966 // imaged 'NextCValue' might not hold for new float types.
8967 // ppc_fp128 can't pass here when converting from max float because of
8968 // APFloat implementation.
8969 if (NextCValue.isInfinity()) {
8970 // ExtMidValue --- narrowed ---> Finite
8971 if (ConvertFltSema(ExtMidValue, SrcFltSema).isInfinity())
8972 return nullptr;
8973
8974 // NextExtMidValue --- narrowed ---> Infinity
8975 APFloat NextExtMidValue = NextValue(ExtMidValue, RoundDown);
8976 if (ConvertFltSema(NextExtMidValue, SrcFltSema).isFinite())
8977 return nullptr;
8978 }
8979
8980 return new FCmpInst(Pred, FPTrunc.getOperand(0),
8981 ConstantFP::get(DestType, ExtMidValue), "", &I);
8982}
8983
8984/// Optimize fabs(X) compared with zero.
8986 Value *X;
8987 if (!match(I.getOperand(0), m_FAbs(m_Value(X))))
8988 return nullptr;
8989
8990 const APFloat *C;
8991 if (!match(I.getOperand(1), m_APFloat(C)))
8992 return nullptr;
8993
8994 if (!C->isPosZero()) {
8995 if (!C->isSmallestNormalized())
8996 return nullptr;
8997
8998 const Function *F = I.getFunction();
8999 DenormalMode Mode = F->getDenormalMode(C->getSemantics());
9000 if (Mode.Input == DenormalMode::PreserveSign ||
9002
9003 auto replaceFCmp = [](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
9004 Constant *Zero = ConstantFP::getZero(X->getType());
9005 return new FCmpInst(P, X, Zero, "", I);
9006 };
9007
9008 switch (I.getPredicate()) {
9009 case FCmpInst::FCMP_OLT:
9010 // fcmp olt fabs(x), smallest_normalized_number -> fcmp oeq x, 0.0
9011 return replaceFCmp(&I, FCmpInst::FCMP_OEQ, X);
9012 case FCmpInst::FCMP_UGE:
9013 // fcmp uge fabs(x), smallest_normalized_number -> fcmp une x, 0.0
9014 return replaceFCmp(&I, FCmpInst::FCMP_UNE, X);
9015 case FCmpInst::FCMP_OGE:
9016 // fcmp oge fabs(x), smallest_normalized_number -> fcmp one x, 0.0
9017 return replaceFCmp(&I, FCmpInst::FCMP_ONE, X);
9018 case FCmpInst::FCMP_ULT:
9019 // fcmp ult fabs(x), smallest_normalized_number -> fcmp ueq x, 0.0
9020 return replaceFCmp(&I, FCmpInst::FCMP_UEQ, X);
9021 default:
9022 break;
9023 }
9024 }
9025
9026 return nullptr;
9027 }
9028
9029 auto replacePredAndOp0 = [&IC](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
9030 I->setPredicate(P);
9031 return IC.replaceOperand(*I, 0, X);
9032 };
9033
9034 switch (I.getPredicate()) {
9035 case FCmpInst::FCMP_UGE:
9036 case FCmpInst::FCMP_OLT:
9037 // fabs(X) >= 0.0 --> true
9038 // fabs(X) < 0.0 --> false
9039 llvm_unreachable("fcmp should have simplified");
9040
9041 case FCmpInst::FCMP_OGT:
9042 // fabs(X) > 0.0 --> X != 0.0
9043 return replacePredAndOp0(&I, FCmpInst::FCMP_ONE, X);
9044
9045 case FCmpInst::FCMP_UGT:
9046 // fabs(X) u> 0.0 --> X u!= 0.0
9047 return replacePredAndOp0(&I, FCmpInst::FCMP_UNE, X);
9048
9049 case FCmpInst::FCMP_OLE:
9050 // fabs(X) <= 0.0 --> X == 0.0
9051 return replacePredAndOp0(&I, FCmpInst::FCMP_OEQ, X);
9052
9053 case FCmpInst::FCMP_ULE:
9054 // fabs(X) u<= 0.0 --> X u== 0.0
9055 return replacePredAndOp0(&I, FCmpInst::FCMP_UEQ, X);
9056
9057 case FCmpInst::FCMP_OGE:
9058 // fabs(X) >= 0.0 --> !isnan(X)
9059 assert(!I.hasNoNaNs() && "fcmp should have simplified");
9060 return replacePredAndOp0(&I, FCmpInst::FCMP_ORD, X);
9061
9062 case FCmpInst::FCMP_ULT:
9063 // fabs(X) u< 0.0 --> isnan(X)
9064 assert(!I.hasNoNaNs() && "fcmp should have simplified");
9065 return replacePredAndOp0(&I, FCmpInst::FCMP_UNO, X);
9066
9067 case FCmpInst::FCMP_OEQ:
9068 case FCmpInst::FCMP_UEQ:
9069 case FCmpInst::FCMP_ONE:
9070 case FCmpInst::FCMP_UNE:
9071 case FCmpInst::FCMP_ORD:
9072 case FCmpInst::FCMP_UNO:
9073 // Look through the fabs() because it doesn't change anything but the sign.
9074 // fabs(X) == 0.0 --> X == 0.0,
9075 // fabs(X) != 0.0 --> X != 0.0
9076 // isnan(fabs(X)) --> isnan(X)
9077 // !isnan(fabs(X) --> !isnan(X)
9078 return replacePredAndOp0(&I, I.getPredicate(), X);
9079
9080 default:
9081 return nullptr;
9082 }
9083}
9084
9085/// Optimize sqrt(X) compared with zero.
9087 Value *X;
9088 if (!match(I.getOperand(0), m_Sqrt(m_Value(X))))
9089 return nullptr;
9090
9091 if (!match(I.getOperand(1), m_PosZeroFP()))
9092 return nullptr;
9093
9094 auto ReplacePredAndOp0 = [&](FCmpInst::Predicate P) {
9095 I.setPredicate(P);
9096 return IC.replaceOperand(I, 0, X);
9097 };
9098
9099 // Clear ninf flag if sqrt doesn't have it.
9100 if (!cast<Instruction>(I.getOperand(0))->hasNoInfs())
9101 I.setHasNoInfs(false);
9102
9103 switch (I.getPredicate()) {
9104 case FCmpInst::FCMP_OLT:
9105 case FCmpInst::FCMP_UGE:
9106 // sqrt(X) < 0.0 --> false
9107 // sqrt(X) u>= 0.0 --> true
9108 llvm_unreachable("fcmp should have simplified");
9109 case FCmpInst::FCMP_ULT:
9110 case FCmpInst::FCMP_ULE:
9111 case FCmpInst::FCMP_OGT:
9112 case FCmpInst::FCMP_OGE:
9113 case FCmpInst::FCMP_OEQ:
9114 case FCmpInst::FCMP_UNE:
9115 // sqrt(X) u< 0.0 --> X u< 0.0
9116 // sqrt(X) u<= 0.0 --> X u<= 0.0
9117 // sqrt(X) > 0.0 --> X > 0.0
9118 // sqrt(X) >= 0.0 --> X >= 0.0
9119 // sqrt(X) == 0.0 --> X == 0.0
9120 // sqrt(X) u!= 0.0 --> X u!= 0.0
9121 return IC.replaceOperand(I, 0, X);
9122
9123 case FCmpInst::FCMP_OLE:
9124 // sqrt(X) <= 0.0 --> X == 0.0
9125 return ReplacePredAndOp0(FCmpInst::FCMP_OEQ);
9126 case FCmpInst::FCMP_UGT:
9127 // sqrt(X) u> 0.0 --> X u!= 0.0
9128 return ReplacePredAndOp0(FCmpInst::FCMP_UNE);
9129 case FCmpInst::FCMP_UEQ:
9130 // sqrt(X) u== 0.0 --> X u<= 0.0
9131 return ReplacePredAndOp0(FCmpInst::FCMP_ULE);
9132 case FCmpInst::FCMP_ONE:
9133 // sqrt(X) != 0.0 --> X > 0.0
9134 return ReplacePredAndOp0(FCmpInst::FCMP_OGT);
9135 case FCmpInst::FCMP_ORD:
9136 // !isnan(sqrt(X)) --> X >= 0.0
9137 return ReplacePredAndOp0(FCmpInst::FCMP_OGE);
9138 case FCmpInst::FCMP_UNO:
9139 // isnan(sqrt(X)) --> X u< 0.0
9140 return ReplacePredAndOp0(FCmpInst::FCMP_ULT);
9141 default:
9142 llvm_unreachable("Unexpected predicate!");
9143 }
9144}
9145
9147 CmpInst::Predicate Pred = I.getPredicate();
9148 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
9149
9150 // Canonicalize fneg as Op1.
9151 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
9152 if (match(Op0, m_FNegOrFSubNegZero(m_Value())) &&
9153 !match(Op1, m_FNegOrFSubNegZero(m_Value()))) {
9154 std::swap(Op0, Op1);
9155 Pred = I.getSwappedPredicate();
9156 }
9157
9158 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
9159 if (!match(Op1, m_FNegOrFSubNegZero(m_Specific(Op0))))
9160 return nullptr;
9161
9162 // Replace the negated operand with 0.0:
9163 // fcmp Pred Op0, -Op0 --> fcmp Pred Op0, 0.0
9165 return new FCmpInst(Pred, Op0, Zero, "", &I);
9166}
9167
9169 Constant *RHSC, InstCombinerImpl &CI) {
9170 const CmpInst::Predicate Pred = I.getPredicate();
9171 Value *X = LHSI->getOperand(0);
9172 Value *Y = LHSI->getOperand(1);
9173 switch (Pred) {
9174 default:
9175 break;
9176 case FCmpInst::FCMP_UGT:
9177 case FCmpInst::FCMP_ULT:
9178 case FCmpInst::FCMP_UNE:
9179 case FCmpInst::FCMP_OEQ:
9180 case FCmpInst::FCMP_OGE:
9181 case FCmpInst::FCMP_OLE:
9182 // The optimization is not valid if X and Y are infinities of the same
9183 // sign, i.e. the inf - inf = nan case. If the fsub has the ninf or nnan
9184 // flag then we can assume we do not have that case. Otherwise we might be
9185 // able to prove that either X or Y is not infinity.
9186 if (!LHSI->hasNoNaNs() && !LHSI->hasNoInfs() &&
9190 break;
9191
9192 [[fallthrough]];
9193 case FCmpInst::FCMP_OGT:
9194 case FCmpInst::FCMP_OLT:
9195 case FCmpInst::FCMP_ONE:
9196 case FCmpInst::FCMP_UEQ:
9197 case FCmpInst::FCMP_UGE:
9198 case FCmpInst::FCMP_ULE:
9199 // fcmp pred (x - y), 0 --> fcmp pred x, y
9200 if (match(RHSC, m_AnyZeroFP()) &&
9201 I.getFunction()->getDenormalMode(
9202 LHSI->getType()->getScalarType()->getFltSemantics()) ==
9204 CI.replaceOperand(I, 0, X);
9205 CI.replaceOperand(I, 1, Y);
9206 I.setHasNoInfs(LHSI->hasNoInfs());
9207 if (LHSI->hasNoNaNs())
9208 I.setHasNoNaNs(true);
9209 return &I;
9210 }
9211 // fcmp `pred (C - Y), C` -> `fcmp swap(pred), Y, 0`
9212 // where C and Y can't be arbitrary floating-point values.
9213 // For example, with `C = 1.0f` and `Y = 0x1p-149`, `1.0f - Y` rounds back
9214 // to `1.0f`, so the source compare is false while the rewritten compare is
9215 // true.
9216 // We need to make sure (C - Y) never rounds back to C
9217 const APFloat *C;
9218 Value *IntSrc;
9219 if (match(RHSC, m_APFloat(C)) &&
9220 match(LHSI, m_FSub(m_Specific(RHSC), m_IToFP(m_Value(IntSrc)))) &&
9221 C->isNormal()) {
9222 // Requirements on C and Y:
9223 // 1. C is finite, nonzero, normal.
9224 // 2. C shouldn't be too large, that is, ULP(C) <= 1.
9225 // 3. Y must be the form of `[su]itofp`, so the finite nonzero result of Y
9226 // must be integer-valued with an absolute value of at least 1;
9227 // as long as the step size near C does not exceed 1,
9228 // C - Y cannot be rounded back to C when Y != 0.
9229 // 4. If Y = 0, `fcmp pred (C - 0), C` are equivalent to `fcmp swap(pred)
9230 // 0, 0` for ordered and unordered predicates as long as C is finite and
9231 // nonzero.
9232 int MantissaWidth = LHSI->getType()->getFPMantissaWidth();
9233 if (MantissaWidth != -1 && ilogb(*C) < MantissaWidth) {
9234 Constant *ZeroC = ConstantFP::getZero(LHSI->getType());
9235 I.setPredicate(I.getSwappedPredicate());
9236 CI.replaceOperand(I, 0, Y);
9237 CI.replaceOperand(I, 1, ZeroC);
9238 return &I;
9239 }
9240 }
9241 break;
9242 }
9243
9244 return nullptr;
9245}
9246
9247/// Fold: fabs(uitofp(a) - uitofp(b)) pred C --> a == b
9248/// where 'pred' is olt, ult, ogt, ugt, oge or uge and C is a positive, Non-NaN
9249/// float when the uitofp casts are exact and C is in the valid range.
9250///
9251/// Since exact uitofp means distinct integers map to distinct floats, the only
9252/// values fabs(uitofp(a) - uitofp(b)) can take are {0.0, 1.0, 2.0, ...}.
9253/// There are no values in the open interval (0, 1), so:
9254/// fabs(...) < C where 0 < C <= 1.0 --> a == b (strict lt: C=1.0 ok)
9255// fabs(..) >= C where C >= 1.0 -> a != b
9256///
9257/// The same logic applies to sitofp.
9259 Value *FAbsArg;
9260 if (!match(I.getOperand(0), m_FAbs(m_Value(FAbsArg))))
9261 return nullptr;
9262
9263 const APFloat *C;
9264 if (!match(I.getOperand(1), PatternMatch::m_FiniteNonZero(C)))
9265 return nullptr;
9266
9267 FCmpInst::Predicate Pred = I.getPredicate();
9268 bool IsStrictLt = Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_ULT;
9269 bool IsLe = Pred == FCmpInst::FCMP_OLE || Pred == FCmpInst::FCMP_ULE;
9270 bool IsStrictGt = Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_UGT;
9271 bool IsGe = Pred == FCmpInst::FCMP_OGE || Pred == FCmpInst::FCMP_UGE;
9272 if (!IsStrictLt && !IsStrictGt && !IsGe)
9273 return nullptr;
9274
9275 APFloat One = APFloat::getOne(C->getSemantics());
9276 APFloat::cmpResult Cmp = C->compare(One);
9277
9278 // For strict-lt (olt/ult): C must be in (0, 1.0] -- C == 1.0 is fine since
9279 // the next possible value after 0.0 is 1.0, and < 1.0 excludes it.
9280 if (IsStrictLt && Cmp == APFloat::cmpGreaterThan)
9281 return nullptr;
9282 if (IsGe && Cmp == APFloat::cmpGreaterThan)
9283 return nullptr;
9284 if (IsLe && Cmp != APFloat::cmpGreaterThan)
9285 return nullptr;
9286 if (IsStrictGt && Cmp != APFloat::cmpLessThan)
9287 return nullptr;
9288
9289 // Match: fsub(uitofp(A), uitofp(B)) where both casts are uitofp or sitofp
9290 Value *A, *B;
9291 bool IsSigned;
9292 if (match(FAbsArg, m_FSub(m_UIToFP(m_Value(A)), m_UIToFP(m_Value(B))))) {
9293 IsSigned = false;
9294 } else if (match(FAbsArg,
9296 IsSigned = true;
9297 } else {
9298 return nullptr;
9299 }
9300
9301 // A and B must have the same integer type
9302 if (A->getType() != B->getType())
9303 return nullptr;
9304
9305 Type *FPTy = FAbsArg->getType();
9306 if (!IC.canBeCastedExactlyIntToFP(A, FPTy, IsSigned, &I) ||
9307 !IC.canBeCastedExactlyIntToFP(B, FPTy, IsSigned, &I))
9308 return nullptr;
9309 ICmpInst::Predicate ResultPred =
9310 IsStrictLt || IsLe ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
9311 return new ICmpInst(ResultPred, A, B);
9312}
9313
9315 InstCombinerImpl &IC) {
9316 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
9317 Type *OpType = LHS->getType();
9318 CmpInst::Predicate Pred = I.getPredicate();
9319
9322
9323 if (!FloorX && !CeilX) {
9326 std::swap(LHS, RHS);
9327 Pred = I.getSwappedPredicate();
9328 }
9329 }
9330
9331 if ((FloorX || CeilX) && FCmpInst::isCommutative(Pred) && LHS->hasOneUse()) {
9332 // fcmp pred floor(x), x => fcmp pred trunc(x), x
9333 // fcmp pred ceil(x), x => fcmp pred trunc(x), x
9334 // where pred is oeq, one, ord, ueq, une, uno.
9335 Value *TruncX = IC.Builder.CreateUnaryIntrinsic(Intrinsic::trunc, RHS);
9336 return new FCmpInst(Pred, TruncX, RHS, "", &I);
9337 }
9338
9339 switch (Pred) {
9340 case FCmpInst::FCMP_OLE:
9341 // fcmp ole floor(x), x => fcmp ord x, 0
9342 if (FloorX)
9344 "", &I);
9345 break;
9346 case FCmpInst::FCMP_OGT:
9347 // fcmp ogt floor(x), x => false
9348 if (FloorX)
9349 return IC.replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
9350 break;
9351 case FCmpInst::FCMP_OGE:
9352 // fcmp oge ceil(x), x => fcmp ord x, 0
9353 if (CeilX)
9355 "", &I);
9356 break;
9357 case FCmpInst::FCMP_OLT:
9358 // fcmp olt ceil(x), x => false
9359 if (CeilX)
9360 return IC.replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
9361 break;
9362 case FCmpInst::FCMP_ULE:
9363 // fcmp ule floor(x), x => true
9364 if (FloorX)
9365 return IC.replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
9366 break;
9367 case FCmpInst::FCMP_UGT:
9368 // fcmp ugt floor(x), x => fcmp uno x, 0
9369 if (FloorX)
9371 "", &I);
9372 break;
9373 case FCmpInst::FCMP_UGE:
9374 // fcmp uge ceil(x), x => true
9375 if (CeilX)
9376 return IC.replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
9377 break;
9378 case FCmpInst::FCMP_ULT:
9379 // fcmp ult ceil(x), x => fcmp uno x, 0
9380 if (CeilX)
9382 "", &I);
9383 break;
9384 default:
9385 break;
9386 }
9387
9388 return nullptr;
9389}
9390
9391/// Returns true if a select that implements a min/max is redundant and
9392/// select result can be replaced with its non-constant operand, e.g.,
9393/// select ( (si/ui-to-fp A) <= C ), C, (si/ui-to-fp A)
9394/// where C is the FP constant equal to the minimum integer value
9395/// representable by A.
9397 Value *B) {
9398 const APFloat *APF;
9399 if (!match(B, m_APFloat(APF)))
9400 return false;
9401
9402 auto *I = dyn_cast<Instruction>(A);
9403 if (!I || !(I->getOpcode() == Instruction::SIToFP ||
9404 I->getOpcode() == Instruction::UIToFP))
9405 return false;
9406
9407 bool IsUnsigned = I->getOpcode() == Instruction::UIToFP;
9408 unsigned BitWidth = I->getOperand(0)->getType()->getScalarSizeInBits();
9409 APSInt IntBoundary = (Flavor == SPF_FMAXNUM)
9410 ? APSInt::getMinValue(BitWidth, IsUnsigned)
9411 : APSInt::getMaxValue(BitWidth, IsUnsigned);
9412 APSInt ConvertedInt(BitWidth, IsUnsigned);
9413 bool IsExact;
9415 APF->convertToInteger(ConvertedInt, APFloat::rmTowardZero, &IsExact);
9416 return Status == APFloat::opOK && IsExact && ConvertedInt == IntBoundary;
9417}
9418
9420 bool Changed = false;
9421
9422 /// Orders the operands of the compare so that they are listed from most
9423 /// complex to least complex. This puts constants before unary operators,
9424 /// before binary operators.
9425 if (getComplexity(I.getOperand(0)) < getComplexity(I.getOperand(1))) {
9426 I.swapOperands();
9427 Changed = true;
9428 }
9429
9430 const CmpInst::Predicate Pred = I.getPredicate();
9431 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
9432 if (Value *V = simplifyFCmpInst(Pred, Op0, Op1, I.getFastMathFlags(),
9433 SQ.getWithInstruction(&I)))
9434 return replaceInstUsesWith(I, V);
9435
9436 // Simplify 'fcmp pred X, X'
9437 Type *OpType = Op0->getType();
9438 assert(OpType == Op1->getType() && "fcmp with different-typed operands?");
9439 if (Op0 == Op1) {
9440 switch (Pred) {
9441 default:
9442 break;
9443 case FCmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y)
9444 case FCmpInst::FCMP_ULT: // True if unordered or less than
9445 case FCmpInst::FCMP_UGT: // True if unordered or greater than
9446 case FCmpInst::FCMP_UNE: // True if unordered or not equal
9447 // Canonicalize these to be 'fcmp uno %X, 0.0'.
9448 I.setPredicate(FCmpInst::FCMP_UNO);
9449 I.setOperand(1, Constant::getNullValue(OpType));
9450 return &I;
9451
9452 case FCmpInst::FCMP_ORD: // True if ordered (no nans)
9453 case FCmpInst::FCMP_OEQ: // True if ordered and equal
9454 case FCmpInst::FCMP_OGE: // True if ordered and greater than or equal
9455 case FCmpInst::FCMP_OLE: // True if ordered and less than or equal
9456 // Canonicalize these to be 'fcmp ord %X, 0.0'.
9457 I.setPredicate(FCmpInst::FCMP_ORD);
9458 I.setOperand(1, Constant::getNullValue(OpType));
9459 return &I;
9460 }
9461 }
9462
9463 if (I.isCommutative()) {
9464 if (auto Pair = matchSymmetricPair(I.getOperand(0), I.getOperand(1))) {
9465 replaceOperand(I, 0, Pair->first);
9466 replaceOperand(I, 1, Pair->second);
9467 return &I;
9468 }
9469 }
9470
9471 // If we're just checking for a NaN (ORD/UNO) and have a non-NaN operand,
9472 // then canonicalize the operand to 0.0.
9473 if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
9474 if (!match(Op0, m_PosZeroFP()) &&
9475 isKnownNeverNaN(Op0, getSimplifyQuery().getWithInstruction(&I)))
9476 return replaceOperand(I, 0, ConstantFP::getZero(OpType));
9477
9478 if (!match(Op1, m_PosZeroFP()) &&
9479 isKnownNeverNaN(Op1, getSimplifyQuery().getWithInstruction(&I)))
9480 return replaceOperand(I, 1, ConstantFP::getZero(OpType));
9481 }
9482
9483 // fcmp pred (fneg X), (fneg Y) -> fcmp swap(pred) X, Y
9484 Value *X, *Y;
9485 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
9486 if (match(Op0, m_FNegOrFSubNegZero(m_Value(X))) &&
9488 return new FCmpInst(I.getSwappedPredicate(), X, Y, "", &I);
9489
9491 return R;
9492
9493 // Test if the FCmpInst instruction is used exclusively by a select as
9494 // part of a minimum or maximum operation. If so, refrain from doing
9495 // any other folding. This helps out other analyses which understand
9496 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
9497 // and CodeGen. And in this case, at least one of the comparison
9498 // operands has at least one user besides the compare (the select),
9499 // which would often largely negate the benefit of folding anyway.
9500 if (I.hasOneUse())
9501 if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) {
9502 Value *A, *B;
9504 bool IsRedundantMinMaxClamp =
9505 (SPR.Flavor == SPF_FMAXNUM || SPR.Flavor == SPF_FMINNUM) &&
9507 if (SPR.Flavor != SPF_UNKNOWN && !IsRedundantMinMaxClamp)
9508 return nullptr;
9509 }
9510
9511 // The sign of 0.0 is ignored by fcmp, so canonicalize to +0.0:
9512 // fcmp Pred X, -0.0 --> fcmp Pred X, 0.0
9513 if (match(Op1, m_AnyZeroFP()) && !match(Op1, m_PosZeroFP()))
9514 return replaceOperand(I, 1, ConstantFP::getZero(OpType));
9515
9516 // Canonicalize:
9517 // fcmp olt X, +inf -> fcmp one X, +inf
9518 // fcmp ole X, +inf -> fcmp ord X, 0
9519 // fcmp ogt X, +inf -> false
9520 // fcmp oge X, +inf -> fcmp oeq X, +inf
9521 // fcmp ult X, +inf -> fcmp une X, +inf
9522 // fcmp ule X, +inf -> true
9523 // fcmp ugt X, +inf -> fcmp uno X, 0
9524 // fcmp uge X, +inf -> fcmp ueq X, +inf
9525 // fcmp olt X, -inf -> false
9526 // fcmp ole X, -inf -> fcmp oeq X, -inf
9527 // fcmp ogt X, -inf -> fcmp one X, -inf
9528 // fcmp oge X, -inf -> fcmp ord X, 0
9529 // fcmp ult X, -inf -> fcmp uno X, 0
9530 // fcmp ule X, -inf -> fcmp ueq X, -inf
9531 // fcmp ugt X, -inf -> fcmp une X, -inf
9532 // fcmp uge X, -inf -> true
9533 const APFloat *C;
9534 if (match(Op1, m_APFloat(C)) && C->isInfinity()) {
9535 switch (C->isNegative() ? FCmpInst::getSwappedPredicate(Pred) : Pred) {
9536 default:
9537 break;
9538 case FCmpInst::FCMP_ORD:
9539 case FCmpInst::FCMP_UNO:
9542 case FCmpInst::FCMP_OGT:
9543 case FCmpInst::FCMP_ULE:
9544 llvm_unreachable("Should be simplified by InstSimplify");
9545 case FCmpInst::FCMP_OLT:
9546 return new FCmpInst(FCmpInst::FCMP_ONE, Op0, Op1, "", &I);
9547 case FCmpInst::FCMP_OLE:
9548 return new FCmpInst(FCmpInst::FCMP_ORD, Op0, ConstantFP::getZero(OpType),
9549 "", &I);
9550 case FCmpInst::FCMP_OGE:
9551 return new FCmpInst(FCmpInst::FCMP_OEQ, Op0, Op1, "", &I);
9552 case FCmpInst::FCMP_ULT:
9553 return new FCmpInst(FCmpInst::FCMP_UNE, Op0, Op1, "", &I);
9554 case FCmpInst::FCMP_UGT:
9555 return new FCmpInst(FCmpInst::FCMP_UNO, Op0, ConstantFP::getZero(OpType),
9556 "", &I);
9557 case FCmpInst::FCMP_UGE:
9558 return new FCmpInst(FCmpInst::FCMP_UEQ, Op0, Op1, "", &I);
9559 }
9560 }
9561
9562 // Ignore signbit of bitcasted int when comparing equality to FP 0.0:
9563 // fcmp oeq/une (bitcast X), 0.0 --> (and X, SignMaskC) ==/!= 0
9564 if (match(Op1, m_PosZeroFP()) &&
9566 X->getType()->isIntOrIntVectorTy() &&
9567 !F.getDenormalMode(Op1->getType()->getScalarType()->getFltSemantics())
9568 .inputsMayBeZero()) {
9570 if (Pred == FCmpInst::FCMP_OEQ)
9571 IntPred = ICmpInst::ICMP_EQ;
9572 else if (Pred == FCmpInst::FCMP_UNE)
9573 IntPred = ICmpInst::ICMP_NE;
9574
9575 if (IntPred != ICmpInst::BAD_ICMP_PREDICATE) {
9576 Type *IntTy = X->getType();
9577 const APInt &SignMask = ~APInt::getSignMask(IntTy->getScalarSizeInBits());
9578 Value *MaskX = Builder.CreateAnd(X, ConstantInt::get(IntTy, SignMask));
9579 return new ICmpInst(IntPred, MaskX, ConstantInt::getNullValue(IntTy));
9580 }
9581 }
9582
9583 // Handle fcmp with instruction LHS and constant RHS.
9584 Instruction *LHSI;
9585 Constant *RHSC;
9586 if (match(Op0, m_Instruction(LHSI)) && match(Op1, m_Constant(RHSC))) {
9587 switch (LHSI->getOpcode()) {
9588 case Instruction::Select:
9589 // fcmp eq (cond ? x : -x), 0 --> fcmp eq x, 0
9590 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
9591 if (FCmpInst::isEquality(Pred) && match(RHSC, m_AnyZeroFP()) &&
9592 match(LHSI,
9594 return replaceOperand(I, 0, X);
9596 return NV;
9597 break;
9598 case Instruction::FSub:
9599 if (LHSI->hasOneUse())
9600 if (Instruction *NV = foldFCmpFSubIntoFCmp(I, LHSI, RHSC, *this))
9601 return NV;
9602 break;
9603 case Instruction::PHI:
9604 if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI)))
9605 return NV;
9606 break;
9607 case Instruction::SIToFP:
9608 case Instruction::UIToFP:
9609 if (Instruction *NV = foldFCmpIntToFPConst(I, LHSI, RHSC))
9610 return NV;
9611 break;
9612 case Instruction::FDiv:
9613 if (Instruction *NV = foldFCmpReciprocalAndZero(I, LHSI, RHSC))
9614 return NV;
9615 break;
9616 case Instruction::Load:
9618 cast<LoadInst>(LHSI), I.getPredicate(), RHSC, I))
9619 return Res;
9620 break;
9621 case Instruction::FPTrunc:
9622 if (Instruction *NV = foldFCmpFpTrunc(I, *LHSI, *RHSC))
9623 return NV;
9624 break;
9625 }
9626 }
9627
9628 if (Instruction *R = foldFabsWithFcmpZero(I, *this))
9629 return R;
9630
9631 if (Instruction *R = foldFCmpFAbsFSubIntToFP(I, *this))
9632 return R;
9633
9634 if (Instruction *R = foldSqrtWithFcmpZero(I, *this))
9635 return R;
9636
9637 if (Instruction *R = foldFCmpWithFloorAndCeil(I, *this))
9638 return R;
9639
9641 return R;
9642
9643 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
9644 if (match(Op0, m_FNegOrFSubNegZero(m_Value(X)))) {
9645 // fcmp pred (fneg X), C --> fcmp swap(pred) X, -C
9646 Constant *C;
9647 if (match(Op1, m_Constant(C)))
9648 if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL))
9649 return new FCmpInst(I.getSwappedPredicate(), X, NegC, "", &I);
9650 }
9651
9652 // fcmp (fadd X, 0.0), Y --> fcmp X, Y
9653 if (match(Op0, m_FAdd(m_Value(X), m_AnyZeroFP())))
9654 return new FCmpInst(Pred, X, Op1, "", &I);
9655
9656 // fcmp X, (fadd Y, 0.0) --> fcmp X, Y
9657 if (match(Op1, m_FAdd(m_Value(Y), m_AnyZeroFP())))
9658 return new FCmpInst(Pred, Op0, Y, "", &I);
9659
9660 // fcmp ord/uno (fptrunc X), (fptrunc Y) -> fcmp ord/uno X, Y
9661 if ((Pred == FCmpInst::FCMP_ORD || Pred == FCmpInst::FCMP_UNO) &&
9662 match(Op0, m_FPTrunc(m_Value(X))) && match(Op1, m_FPTrunc(m_Value(Y))) &&
9663 X->getType() == Y->getType())
9664 return new FCmpInst(Pred, X, Y, "", &I);
9665
9666 if (match(Op0, m_FPExt(m_Value(X)))) {
9667 // fcmp (fpext X), (fpext Y) -> fcmp X, Y
9668 if (match(Op1, m_FPExt(m_Value(Y))) && X->getType() == Y->getType())
9669 return new FCmpInst(Pred, X, Y, "", &I);
9670
9671 const APFloat *C;
9672 if (match(Op1, m_APFloat(C))) {
9673 const fltSemantics &FPSem =
9674 X->getType()->getScalarType()->getFltSemantics();
9675 bool Lossy;
9676 APFloat TruncC = *C;
9678
9679 if (Lossy) {
9680 // X can't possibly equal the higher-precision constant, so reduce any
9681 // equality comparison.
9682 // TODO: Other predicates can be handled via getFCmpCode().
9683 switch (Pred) {
9684 case FCmpInst::FCMP_OEQ:
9685 // X is ordered and equal to an impossible constant --> false
9686 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
9687 case FCmpInst::FCMP_ONE:
9688 // X is ordered and not equal to an impossible constant --> ordered
9689 return new FCmpInst(FCmpInst::FCMP_ORD, X,
9690 ConstantFP::getZero(X->getType()));
9691 case FCmpInst::FCMP_UEQ:
9692 // X is unordered or equal to an impossible constant --> unordered
9693 return new FCmpInst(FCmpInst::FCMP_UNO, X,
9694 ConstantFP::getZero(X->getType()));
9695 case FCmpInst::FCMP_UNE:
9696 // X is unordered or not equal to an impossible constant --> true
9697 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
9698 default:
9699 break;
9700 }
9701 }
9702
9703 // fcmp (fpext X), C -> fcmp X, (fptrunc C) if fptrunc is lossless
9704 // Avoid lossy conversions and denormals.
9705 // Zero is a special case that's OK to convert.
9706 APFloat Fabs = TruncC;
9707 Fabs.clearSign();
9708 if (!Lossy &&
9709 (Fabs.isZero() || !(Fabs < APFloat::getSmallestNormalized(FPSem)))) {
9710 Constant *NewC = ConstantFP::get(X->getType(), TruncC);
9711 return new FCmpInst(Pred, X, NewC, "", &I);
9712 }
9713 }
9714 }
9715
9716 // Convert a sign-bit test of an FP value into a cast and integer compare.
9717 // TODO: Simplify if the copysign constant is 0.0 or NaN.
9718 // TODO: Handle non-zero compare constants.
9719 // TODO: Handle other predicates.
9721 m_Value(X)))) &&
9722 match(Op1, m_AnyZeroFP()) && !C->isZero() && !C->isNaN()) {
9723 Type *IntType = Builder.getIntNTy(X->getType()->getScalarSizeInBits());
9724 if (auto *VecTy = dyn_cast<VectorType>(OpType))
9725 IntType = VectorType::get(IntType, VecTy->getElementCount());
9726
9727 // copysign(non-zero constant, X) < 0.0 --> (bitcast X) < 0
9728 if (Pred == FCmpInst::FCMP_OLT) {
9729 Value *IntX = Builder.CreateBitCast(X, IntType);
9730 return new ICmpInst(ICmpInst::ICMP_SLT, IntX,
9731 ConstantInt::getNullValue(IntType));
9732 }
9733 }
9734
9735 {
9736 Value *CanonLHS = nullptr;
9738 // (canonicalize(x) == x) => (x == x)
9739 if (CanonLHS == Op1)
9740 return new FCmpInst(Pred, Op1, Op1, "", &I);
9741
9742 Value *CanonRHS = nullptr;
9744 // (x == canonicalize(x)) => (x == x)
9745 if (CanonRHS == Op0)
9746 return new FCmpInst(Pred, Op0, Op0, "", &I);
9747
9748 // (canonicalize(x) == canonicalize(y)) => (x == y)
9749 if (CanonLHS && CanonRHS)
9750 return new FCmpInst(Pred, CanonLHS, CanonRHS, "", &I);
9751 }
9752
9753 if (I.getType()->isVectorTy())
9754 if (Instruction *Res = foldVectorCmp(I, Builder))
9755 return Res;
9756
9757 return Changed ? &I : nullptr;
9758}
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
Rewrite undef for PHI
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
#define Check(C,...)
Hexagon Common GEP
static Instruction * foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI, Constant *RHSC)
Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary.
static Instruction * foldFabsWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC)
Optimize fabs(X) compared with zero.
static void collectOffsetOp(Value *V, SmallVectorImpl< OffsetOp > &Offsets, bool AllowRecursion)
static Value * rewriteGEPAsOffset(Value *Start, Value *Base, GEPNoWrapFlags NW, const DataLayout &DL, SetVector< Value * > &Explored, InstCombiner &IC)
Returns a re-written value of Start as an indexed GEP using Base as a pointer.
static bool isMinMaxCmpSelectEliminable(SelectPatternFlavor Flavor, Value *A, Value *B)
Returns true if a select that implements a min/max is redundant and select result can be replaced wit...
static Instruction * foldICmpEqualityWithOffset(ICmpInst &I, InstCombiner::BuilderTy &Builder, const SimplifyQuery &SQ)
Offset both sides of an equality icmp to see if we can save some instructions: icmp eq/ne X,...
static bool addWithOverflow(APInt &Result, const APInt &In1, const APInt &In2, bool IsSigned=false)
Compute Result = In1+In2, returning true if the result overflowed for this type.
static Instruction * foldICmpOfVectorReduce(ICmpInst &I, const DataLayout &DL, IRBuilderBase &Builder)
static Instruction * foldICmpAndXX(ICmpInst &I, const SimplifyQuery &Q, InstCombinerImpl &IC)
static Instruction * foldVectorCmp(CmpInst &Cmp, InstCombiner::BuilderTy &Builder)
static bool isMaskOrZero(const Value *V, bool Not, const SimplifyQuery &Q, unsigned Depth=0)
static Value * createLogicFromTable(const std::bitset< 4 > &Table, Value *Op0, Value *Op1, IRBuilderBase &Builder, bool HasOneUse)
static Instruction * foldICmpOfUAddOv(ICmpInst &I)
static bool isChainSelectCmpBranch(const SelectInst *SI)
Return true when the instruction sequence within a block is select-cmp-br.
static Instruction * foldICmpInvariantGroup(ICmpInst &I)
std::pair< Instruction::BinaryOps, Value * > OffsetOp
Find all possible pairs (BinOp, RHS) that BinOp V, RHS can be simplified.
static Instruction * foldReductionIdiom(ICmpInst &I, InstCombiner::BuilderTy &Builder, const DataLayout &DL)
This function folds patterns produced by lowering of reduce idioms, such as llvm.vector....
static Instruction * canonicalizeICmpBool(ICmpInst &I, InstCombiner::BuilderTy &Builder)
Integer compare with boolean values can always be turned into bitwise ops.
static Instruction * foldFCmpFSubIntoFCmp(FCmpInst &I, Instruction *LHSI, Constant *RHSC, InstCombinerImpl &CI)
static Value * foldICmpOrXorSubChain(ICmpInst &Cmp, BinaryOperator *Or, InstCombiner::BuilderTy &Builder)
Fold icmp eq/ne (or (xor/sub (X1, X2), xor/sub (X3, X4))), 0.
static bool hasBranchUse(ICmpInst &I)
Given an icmp instruction, return true if any use of this comparison is a branch on sign bit comparis...
static Value * foldICmpWithLowBitMaskedVal(CmpPredicate Pred, Value *Op0, Value *Op1, const SimplifyQuery &Q, InstCombiner &IC)
Some comparisons can be simplified.
static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth)
When performing a comparison against a constant, it is possible that not all the bits in the LHS are ...
static Instruction * foldICmpShlLHSC(ICmpInst &Cmp, Instruction *Shl, const APInt &C)
Fold icmp (shl nuw C2, Y), C.
static Instruction * foldFCmpWithFloorAndCeil(FCmpInst &I, InstCombinerImpl &IC)
static Instruction * foldICmpXorXX(ICmpInst &I, const SimplifyQuery &Q, InstCombinerImpl &IC)
static Instruction * processUZExtIdiom(ICmpInst &I, Value *Val, const APInt *OtherVal, InstCombinerImpl &IC)
Recognize and process idiom involving test for unsigned overflow.
static Instruction * foldICmpOfCmpIntrinsicWithConstant(CmpPredicate Pred, IntrinsicInst *I, const APInt &C, InstCombiner::BuilderTy &Builder)
static Instruction * foldSqrtWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC)
Optimize sqrt(X) compared with zero.
static Instruction * foldFCmpFNegCommonOp(FCmpInst &I)
static bool hasNoWrapProblem(const BinaryOperator &BO, CmpInst::Predicate Pred, bool &HasNSW, bool &HasNUW)
static Instruction * foldICmpWithHighBitMask(ICmpInst &Cmp, InstCombiner::BuilderTy &Builder)
static ICmpInst * canonicalizeCmpWithConstant(ICmpInst &I)
If we have an icmp le or icmp ge instruction with a constant operand, turn it into the appropriate ic...
static Instruction * foldICmpIntrinsicWithIntrinsic(ICmpInst &Cmp, InstCombiner::BuilderTy &Builder)
Fold an icmp with LLVM intrinsics.
static Instruction * foldICmpUSubSatOrUAddSatWithConstant(CmpPredicate Pred, SaturatingInst *II, const APInt &C, InstCombiner::BuilderTy &Builder)
static Instruction * foldICmpPow2Test(ICmpInst &I, InstCombiner::BuilderTy &Builder)
static bool subWithOverflow(APInt &Result, const APInt &In1, const APInt &In2, bool IsSigned=false)
Compute Result = In1-In2, returning true if the result overflowed for this type.
static bool canRewriteGEPAsOffset(Value *Start, Value *Base, GEPNoWrapFlags &NW, const DataLayout &DL, SetVector< Value * > &Explored)
Returns true if we can rewrite Start as a GEP with pointer Base and some integer offset.
static Instruction * foldFCmpFpTrunc(FCmpInst &I, const Instruction &FPTrunc, const Constant &C)
static Instruction * foldICmpXNegX(ICmpInst &I, InstCombiner::BuilderTy &Builder)
static Instruction * processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B, ConstantInt *CI2, ConstantInt *CI1, InstCombinerImpl &IC)
The caller has matched a pattern of the form: I = icmp ugt (add (add A, B), CI2), CI1 If this is of t...
static Value * foldShiftIntoShiftInAnotherHandOfAndInICmp(ICmpInst &I, const SimplifyQuery SQ, InstCombiner::BuilderTy &Builder)
static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C)
Returns true if the exploded icmp can be expressed as a signed comparison to zero and updates the pre...
static Instruction * transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS, CmpPredicate Cond, const DataLayout &DL, InstCombiner &IC)
Converts (CMP GEPLHS, RHS) if this change would make RHS a constant.
static Instruction * foldCtpopPow2Test(ICmpInst &I, IntrinsicInst *CtpopLhs, const APInt &CRhs, InstCombiner::BuilderTy &Builder, const SimplifyQuery &Q)
static Instruction * foldFCmpFAbsFSubIntToFP(FCmpInst &I, InstCombinerImpl &IC)
Fold: fabs(uitofp(a) - uitofp(b)) pred C --> a == b where 'pred' is olt, ult, ogt,...
static void setInsertionPoint(IRBuilder<> &Builder, Value *V, bool Before=true)
static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS, bool IsSigned)
static bool isMultipleOf(Value *X, const APInt &C, const SimplifyQuery &Q)
Return true if X is a multiple of C.
static Value * foldICmpWithTruncSignExtendedVal(ICmpInst &I, InstCombiner::BuilderTy &Builder)
Some comparisons can be simplified.
static Instruction * foldICmpOrXX(ICmpInst &I, const SimplifyQuery &Q, InstCombinerImpl &IC)
This file provides internal interfaces used to implement the InstCombine.
This file provides the interface for the instcombine pass implementation.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:550
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define T1
uint64_t IntrinsicInst * II
#define P(N)
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
int64_t C1
int64_t C0
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
SI Fold Operands
This file implements a set that has insertion order iteration characteristics.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Value * RHS
Value * LHS
cmpResult
IEEE-754R 5.11: Floating Point Comparison Relations.
Definition APFloat.h:351
static constexpr roundingMode rmTowardZero
Definition APFloat.h:365
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:361
static LLVM_ABI ExponentType semanticsMaxExponent(const fltSemantics &)
Definition APFloat.cpp:339
opStatus
IEEE-754R 7: Default exception handling.
Definition APFloat.h:377
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition APFloat.cpp:6045
void clearSign()
Definition APFloat.h:1402
bool isNaN() const
Definition APFloat.h:1581
static APFloat getOne(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative One.
Definition APFloat.h:1192
bool isZero() const
Definition APFloat.h:1579
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
Definition APFloat.h:1262
APInt bitcastToAPInt() const
Definition APFloat.h:1475
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition APFloat.h:1242
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition APFloat.h:1436
opStatus next(bool nextDown)
Definition APFloat.h:1358
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition APFloat.h:1202
LLVM_ABI FPClassTest classify() const
Return the FPClassTest which will return true for the value.
Definition APFloat.cpp:5974
opStatus roundToIntegral(roundingMode RM)
Definition APFloat.h:1352
bool isInfinity() const
Definition APFloat.h:1580
Class for arbitrary precision integers.
Definition APInt.h:78
LLVM_ABI APInt udiv(const APInt &RHS) const
Unsigned division operation.
Definition APInt.cpp:1602
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
static LLVM_ABI void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
Definition APInt.cpp:1796
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
Definition APInt.h:445
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1057
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition APInt.h:225
bool isMinSignedValue() const
Determine if this is the smallest signed value.
Definition APInt.h:419
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1532
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:970
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition APInt.h:202
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition APInt.h:1350
APInt abs() const
Get the absolute value.
Definition APInt.h:1815
unsigned ceilLogBase2() const
Definition APInt.h:1784
bool sgt(const APInt &RHS) const
Signed greater than comparison.
Definition APInt.h:1205
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition APInt.h:367
LLVM_ABI APInt usub_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1986
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition APInt.h:1186
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
Definition APInt.h:462
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition APInt.h:1115
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition APInt.h:205
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
Definition APInt.h:212
bool isNegative() const
Determine sign of this APInt.
Definition APInt.h:325
LLVM_ABI APInt sadd_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1966
bool eq(const APInt &RHS) const
Equality comparison.
Definition APInt.h:1083
LLVM_ABI APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
Definition APInt.cpp:1673
LLVM_ABI APInt uadd_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1973
void negate()
Negate this APInt in place.
Definition APInt.h:1488
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition APInt.h:1659
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition APInt.h:1618
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition APInt.h:215
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
Definition APInt.h:352
void flipAllBits()
Toggle every bit to its opposite value.
Definition APInt.h:1472
unsigned countl_one() const
Count the number of leading one bits.
Definition APInt.h:1635
unsigned logBase2() const
Definition APInt.h:1781
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
Definition APInt.h:471
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition APInt.h:829
bool isMaxSignedValue() const
Determine if this is the largest signed value.
Definition APInt.h:401
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition APInt.h:1154
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition APInt.h:875
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:436
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition APInt.h:292
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition APInt.h:196
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
Definition APInt.h:1241
LLVM_ABI APInt ssub_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1979
bool isOne() const
Determine if this is a value of 1.
Definition APInt.h:385
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition APInt.h:282
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition APInt.h:235
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition APInt.h:853
unsigned countr_one() const
Count the number of trailing one bits.
Definition APInt.h:1676
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition APInt.h:1225
An arbitrary precision integer that knows its signedness.
Definition APSInt.h:24
static APSInt getMinValue(uint32_t numBits, bool Unsigned)
Return the APSInt representing the minimum integer value with the given bit width and signedness.
Definition APSInt.h:310
static APSInt getMaxValue(uint32_t numBits, bool Unsigned)
Return the APSInt representing the maximum integer value with the given bit width and signedness.
Definition APSInt.h:302
an instruction to allocate memory on the stack
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
LLVM Basic Block Representation.
Definition BasicBlock.h:62
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
BinaryOps getOpcode() const
Definition InstrTypes.h:409
static LLVM_ABI BinaryOperator * CreateNot(Value *Op, const Twine &Name="", InsertPosition InsertBefore=nullptr)
static LLVM_ABI BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
Value * getArgOperand(unsigned i) const
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
This class is the base class for the comparison instructions.
Definition InstrTypes.h:728
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Predicate getStrictPredicate() const
For example, SGE -> SGT, SLE -> SLT, ULE -> ULT, UGE -> UGT.
Definition InstrTypes.h:921
bool isEquality() const
Determine if this is an equals/not equals predicate.
Definition InstrTypes.h:978
static LLVM_ABI Predicate getFlippedStrictnessPredicate(Predicate pred)
This is a static version that you can use without an instruction available.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition InstrTypes.h:757
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition InstrTypes.h:742
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
bool isSigned() const
Definition InstrTypes.h:993
static LLVM_ABI bool isEquality(Predicate pred)
Determine if this is an equals/not equals predicate.
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition InstrTypes.h:890
bool isTrueWhenEqual() const
This is just a convenience.
static LLVM_ABI CmpInst * Create(OtherOps Op, Predicate Pred, Value *S1, Value *S2, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Construct a compare instruction, given the opcode, the predicate and the two operands.
Predicate getNonStrictPredicate() const
For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
Definition InstrTypes.h:934
static LLVM_ABI bool isStrictPredicate(Predicate predicate)
This is a static version that you can use without an instruction available.
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition InstrTypes.h:852
Predicate getPredicate() const
Return the predicate for this instruction.
Definition InstrTypes.h:828
static bool isIntPredicate(Predicate P)
Definition InstrTypes.h:839
bool isUnsigned() const
Definition InstrTypes.h:999
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
static LLVM_ABI CmpPredicate getSwapped(CmpPredicate P)
Get the swapped predicate of a CmpPredicate.
Conditional Branch instruction.
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static LLVM_ABI Constant * getPointerBitCastOrAddrSpaceCast(Constant *C, Type *Ty)
Create a BitCast or AddrSpaceCast for a pointer type depending on the address space.
static LLVM_ABI Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static LLVM_ABI Constant * getNot(Constant *C)
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static LLVM_ABI Constant * getXor(Constant *C1, Constant *C2)
static LLVM_ABI Constant * getNeg(Constant *C, bool HasNSW=false)
static LLVM_ABI ConstantFP * getZero(Type *Ty, bool Negative=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...
Definition Constants.h:269
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
Definition Constants.h:135
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
Definition Constants.h:162
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
static LLVM_ABI ConstantInt * getBool(LLVMContext &Context, bool V)
This class represents a range of values.
LLVM_ABI ConstantRange add(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an addition of a value in this ran...
LLVM_ABI bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const
Set up Pred and RHS such that ConstantRange::makeExactICmpRegion(Pred, RHS) == *this.
LLVM_ABI ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
const APInt * getSingleElement() const
If this set contains a single element, return it, otherwise return null.
LLVM_ABI ConstantRange difference(const ConstantRange &CR) const
Subtract the specified range from this range (aka relative complement of the sets).
LLVM_ABI bool isEmptySet() const
Return true if this set contains no members.
LLVM_ABI ConstantRange truncate(uint32_t BitWidth, unsigned NoWrapKind=0) const
Return a new range in the specified integer type, which must be strictly smaller than the current typ...
static LLVM_ABI ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
LLVM_ABI ConstantRange inverse() const
Return a new range that is the logical not of the current set.
LLVM_ABI ConstantRange intersectWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the intersection of this range with another range.
static ConstantRange getNonEmpty(APInt Lower, APInt Upper)
Create non-empty constant range with the given bounds.
LLVM_ABI ConstantRange sub(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a subtraction of a value in this r...
static LLVM_ABI ConstantRange makeExactNoWrapRegion(Instruction::BinaryOps BinOp, const APInt &Other, unsigned NoWrapKind)
Produce the range that contains X if and only if "X BinOp Other" does not wrap.
static LLVM_ABI Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
This is an important base class in LLVM.
Definition Constant.h:44
static LLVM_ABI Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition Constant.h:65
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
LLVM_ABI bool isAllOnesValue() const
Return true if this is the value that would be returned by getAllOnesValue.
Definition Constants.cpp:62
LLVM_ABI const APInt & getUniqueInteger() const
If C is a constant integer then return its value, otherwise C must be a vector of constant integers,...
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition DenseMap.h:757
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
iterator end()
Definition DenseMap.h:686
static ExtractValueInst * Create(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
This instruction compares its operands according to the predicate given to the constructor.
static bool isCommutative(Predicate Pred)
static bool isEquality(Predicate Pred)
Represents flags for the getelementptr instruction/expression.
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
GEPNoWrapFlags intersectForOffsetAdd(GEPNoWrapFlags Other) const
Given (gep (gep p, x), y), determine the nowrap flags for (gep p, x+y).
static GEPNoWrapFlags none()
bool isInBounds() const
Test whether this is an inbounds GEP, as defined by LangRef.html.
Definition Operator.h:390
LLVM_ABI Type * getSourceElementType() const
Definition Operator.cpp:86
Value * getPointerOperand()
Definition Operator.h:417
GEPNoWrapFlags getNoWrapFlags() const
Definition Operator.h:385
bool hasAllConstantIndices() const
Return true if all of the indices of this GEP are constant integers.
Definition Operator.h:464
This instruction compares its operands according to the predicate given to the constructor.
static bool isGE(Predicate P)
Return true if the predicate is SGE or UGE.
static LLVM_ABI bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
static bool isLT(Predicate P)
Return true if the predicate is SLT or ULT.
static bool isGT(Predicate P)
Return true if the predicate is SGT or UGT.
Predicate getFlippedSignednessPredicate() const
For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->EQ.
Predicate getSignedPredicate() const
For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
bool isEquality() const
Return true if this predicate is either EQ or NE.
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
bool isRelational() const
Return true if the predicate is relational (not EQ or NE).
Predicate getUnsignedPredicate() const
For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
static bool isLE(Predicate P)
Return true if the predicate is SLE or ULE.
Common base class shared among various IRBuilders.
Definition IRBuilder.h:111
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", bool HasSameSign=false)
Definition IRBuilder.h:2568
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:1572
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition IRBuilder.h:197
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)
Definition IRBuilder.h:1594
ConstantInt * getInt(const APInt &AI)
Get a constant integer value.
Definition IRBuilder.h:486
LLVM_ABI Value * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *Op, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
Instruction * foldICmpShrConstant(ICmpInst &Cmp, BinaryOperator *Shr, const APInt &C)
Fold icmp ({al}shr X, Y), C.
Instruction * foldICmpWithZextOrSext(ICmpInst &ICmp)
Instruction * foldICmpSelectConstant(ICmpInst &Cmp, SelectInst *Select, ConstantInt *C)
Instruction * foldICmpSRemConstant(ICmpInst &Cmp, BinaryOperator *UDiv, const APInt &C)
Instruction * foldICmpBinOpWithConstant(ICmpInst &Cmp, BinaryOperator *BO, const APInt &C)
Fold an icmp with BinaryOp and constant operand: icmp Pred BO, C.
Instruction * foldICmpOrConstant(ICmpInst &Cmp, BinaryOperator *Or, const APInt &C)
Fold icmp (or X, Y), C.
Instruction * foldICmpTruncWithTruncOrExt(ICmpInst &Cmp, const SimplifyQuery &Q)
Fold icmp (trunc nuw/nsw X), (trunc nuw/nsw Y).
Instruction * foldSignBitTest(ICmpInst &I)
Fold equality-comparison between zero and any (maybe truncated) right-shift by one-less-than-bitwidth...
Instruction * foldCmpLoadFromIndexedGlobal(LoadInst *LI, CmpPredicate Pred, Constant *RHSC, Instruction &CtxI, ConstantInt *AndCst=nullptr)
Try to fold cmp pred (load (gep GV, ...)), cmpcst where GV is a global variable with a constant initi...
Instruction * foldOpIntoPhi(Instruction &I, PHINode *PN, bool AllowMultipleUses=false)
Given a binary operator, cast instruction, or select which has a PHI node as operand #0,...
Value * insertRangeTest(Value *V, const APInt &Lo, const APInt &Hi, bool isSigned, bool Inside)
Emit a computation of: (V >= Lo && V < Hi) if Inside is true, otherwise (V < Lo || V >= Hi).
Instruction * foldICmpBinOp(ICmpInst &Cmp, const SimplifyQuery &SQ)
Try to fold icmp (binop), X or icmp X, (binop).
Instruction * foldICmpSubConstant(ICmpInst &Cmp, BinaryOperator *Sub, const APInt &C)
Fold icmp (sub X, Y), C.
Instruction * foldICmpWithClamp(ICmpInst &Cmp, Value *X, MinMaxIntrinsic *Min)
Match and fold patterns like: icmp eq/ne X, min(max(X, Lo), Hi) which represents a range check and ca...
Instruction * foldICmpInstWithConstantNotInt(ICmpInst &Cmp)
Handle icmp with constant (but not simple integer constant) RHS.
bool matchCommonBinOpOperands(Value *Op0, Value *Op1, CmpInst::Predicate Pred, Value *&LHS, Value *&RHS, const SimplifyQuery &Q)
Match Op0 Pred Op1 on binops with a removable common operand.
bool SimplifyDemandedBits(Instruction *I, unsigned Op, const APInt &DemandedMask, KnownBits &Known, const SimplifyQuery &Q, unsigned Depth=0) override
This form of SimplifyDemandedBits simplifies the specified instruction operand if possible,...
Instruction * foldICmpShlConstConst(ICmpInst &I, Value *ShAmt, const APInt &C1, const APInt &C2)
Handle "(icmp eq/ne (shl AP2, A), AP1)" -> (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)).
Value * reassociateShiftAmtsOfTwoSameDirectionShifts(BinaryOperator *Sh0, const SimplifyQuery &SQ, bool AnalyzeForSignBitExtraction=false)
Instruction * foldICmpEqIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II, const APInt &C)
Fold an equality icmp with LLVM intrinsic and constant operand.
Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI, bool FoldWithMultiUse=false, bool SimplifyBothArms=false)
Given an instruction with a select as one operand and a constant as the other operand,...
Value * foldMultiplicationOverflowCheck(ICmpInst &Cmp)
Fold (-1 u/ x) u< y ((x * y) ?
Instruction * foldICmpWithConstant(ICmpInst &Cmp)
Fold icmp Pred X, C.
OverflowResult computeOverflow(Instruction::BinaryOps BinaryOp, bool IsSigned, Value *LHS, Value *RHS, Instruction *CtxI) const
CmpInst * canonicalizeICmpPredicate(CmpInst &I)
If we have a comparison with a non-canonical predicate, if we can update all the users,...
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Instruction * foldICmpWithZero(ICmpInst &Cmp)
Instruction * foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp, BinaryOperator *BO, const APInt &C)
Fold an icmp equality instruction with binary operator LHS and constant RHS: icmp eq/ne BO,...
Instruction * foldICmpUsingBoolRange(ICmpInst &I)
If one operand of an icmp is effectively a bool (value range of {0,1}), then try to reduce patterns b...
Instruction * foldICmpWithTrunc(ICmpInst &Cmp)
Instruction * foldCmpSelectOfConstants(CmpInst &I)
Fold fcmp/icmp pred (select C1, TV1, FV1), (select C2, TV2, FV2) where all true/false values are cons...
Instruction * foldICmpIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II, const APInt &C)
Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C.
bool matchThreeWayIntCompare(SelectInst *SI, Value *&LHS, Value *&RHS, ConstantInt *&Less, ConstantInt *&Equal, ConstantInt *&Greater)
Match a select chain which produces one of three values based on whether the LHS is less than,...
const InstCombineCLOptions & CLOpts
Instruction * visitFCmpInst(FCmpInst &I)
Instruction * foldICmpUsingKnownBits(ICmpInst &Cmp)
Try to fold the comparison based on range information we can get by checking whether bits are known t...
Instruction * foldICmpDivConstant(ICmpInst &Cmp, BinaryOperator *Div, const APInt &C)
Fold icmp ({su}div X, Y), C.
Instruction * foldIRemByPowerOfTwoToBitTest(ICmpInst &I)
If we have: icmp eq/ne (urem/srem x, y), 0 iff y is a power-of-two, we can replace this with a bit te...
Instruction * foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, Constant *RHSC)
Fold fcmp ([us]itofp x, cst) if possible.
Instruction * foldICmpUDivConstant(ICmpInst &Cmp, BinaryOperator *UDiv, const APInt &C)
Fold icmp (udiv X, Y), C.
Instruction * foldICmpAddOpConst(Value *X, const APInt &C, CmpPredicate Pred)
Fold "icmp pred (X+C), X".
Instruction * foldICmpWithCastOp(ICmpInst &ICmp)
Handle icmp (cast x), (cast or constant).
Instruction * foldICmpTruncConstant(ICmpInst &Cmp, TruncInst *Trunc, const APInt &C)
Fold icmp (trunc X), C.
Instruction * foldICmpAddConstant(ICmpInst &Cmp, BinaryOperator *Add, const APInt &C)
Fold icmp (add X, Y), C.
Instruction * foldICmpMulConstant(ICmpInst &Cmp, BinaryOperator *Mul, const APInt &C)
Fold icmp (mul X, Y), C.
Instruction * foldICmpCommutative(CmpPredicate Pred, Value *Op0, Value *Op1, ICmpInst &CtxI)
Instruction * tryFoldInstWithCtpopWithNot(Instruction *I)
Instruction * foldICmpXorConstant(ICmpInst &Cmp, BinaryOperator *Xor, const APInt &C)
Fold icmp (xor X, Y), C.
Instruction * foldSelectICmp(CmpPredicate Pred, SelectInst *SI, Value *RHS, const ICmpInst &I)
Instruction * foldICmpInstWithConstantAllowPoison(ICmpInst &Cmp, const APInt &C)
Try to fold integer comparisons with a constant operand: icmp Pred X, C where X is some kind of instr...
Instruction * foldIsMultipleOfAPowerOfTwo(ICmpInst &Cmp)
Fold icmp eq (num + mask) & ~mask, num to icmp eq (and num, mask), 0 Where mask is a low bit mask.
Instruction * foldICmpAndShift(ICmpInst &Cmp, BinaryOperator *And, const APInt &C1, const APInt &C2)
Fold icmp (and (sh X, Y), C2), C1.
Instruction * foldICmpBinOpWithConstantViaTruthTable(ICmpInst &Cmp, BinaryOperator *BO, const APInt &C)
Instruction * foldICmpInstWithConstant(ICmpInst &Cmp)
Try to fold integer comparisons with a constant operand: icmp Pred X, C where X is some kind of instr...
Instruction * foldICmpXorShiftConst(ICmpInst &Cmp, BinaryOperator *Xor, const APInt &C)
For power-of-2 C: ((X s>> ShiftC) ^ X) u< C --> (X + C) u< (C << 1) ((X s>> ShiftC) ^ X) u> (C - 1) -...
Instruction * foldICmpShlConstant(ICmpInst &Cmp, BinaryOperator *Shl, const APInt &C)
Fold icmp (shl X, Y), C.
Instruction * foldICmpAndConstant(ICmpInst &Cmp, BinaryOperator *And, const APInt &C)
Fold icmp (and X, Y), C.
Instruction * foldICmpEquality(ICmpInst &Cmp)
Instruction * foldICmpWithMinMax(Instruction &I, MinMaxIntrinsic *MinMax, Value *Z, CmpPredicate Pred)
Fold icmp Pred min|max(X, Y), Z.
bool dominatesAllUses(const Instruction *DI, const Instruction *UI, const BasicBlock *DB) const
True when DB dominates all uses of DI except UI.
bool foldAllocaCmp(AllocaInst *Alloca)
Instruction * visitICmpInst(ICmpInst &I)
Instruction * foldICmpWithDominatingICmp(ICmpInst &Cmp)
Canonicalize icmp instructions based on dominating conditions.
bool replacedSelectWithOperand(SelectInst *SI, const ICmpInst *Icmp, const unsigned SIOpd)
Try to replace select with select operand SIOpd in SI-ICmp sequence.
Instruction * foldICmpShrConstConst(ICmpInst &I, Value *ShAmt, const APInt &C1, const APInt &C2)
Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" -> (icmp eq/ne A, Log2(AP2/AP1)) -> (icmp eq/ne A,...
void freelyInvertAllUsersOf(Value *V, Value *IgnoredUser=nullptr)
Freely adapt every user of V as-if V was changed to !V.
Instruction * foldICmpAndConstConst(ICmpInst &Cmp, BinaryOperator *And, const APInt &C1)
Fold icmp (and X, C2), C1.
Instruction * foldICmpBitCast(ICmpInst &Cmp)
Instruction * foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, CmpPredicate Cond, Instruction &I)
Fold comparisons between a GEP instruction and something else.
The core instruction combiner logic.
SimplifyQuery SQ
OverflowResult computeOverflowForSignedMul(const Value *LHS, const Value *RHS, const Instruction *CtxI) const
Module & getModule() const
LLVM_ABI bool canBeCastedExactlyIntToFP(Value *V, Type *FPTy, bool IsSigned, const Instruction *CtxI=nullptr) const
bool isFreeToInvert(Value *V, bool WillInvertAllUses, bool &DoesConsume)
Return true if the specified value is free to invert (apply ~ to).
static unsigned getComplexity(Value *V)
Assign a complexity or rank value to LLVM Values.
unsigned ComputeMaxSignificantBits(const Value *Op, const Instruction *CtxI=nullptr, unsigned Depth=0) const
bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero=false, const Instruction *CtxI=nullptr, unsigned Depth=0)
TargetLibraryInfo & TLI
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
static Constant * SubOne(Constant *C)
Subtract one from a Constant.
OverflowResult computeOverflowForUnsignedSub(const Value *LHS, const Value *RHS, const Instruction *CtxI) const
static bool isCanonicalPredicate(CmpPredicate Pred)
Predicate canonicalization reduces the number of patterns that need to be matched by other transforms...
const DataLayout & DL
DomConditionCache DC
IRBuilder< TargetFolder, IRBuilderInstCombineInserter > BuilderTy
An IRBuilder that automatically inserts new instructions into the worklist.
bool canFreelyInvertAllUsersOf(Instruction *V, Value *IgnoredUser)
Given i1 V, can every user of V be freely adapted if V is changed to !V ?
OverflowResult computeOverflowForSignedAdd(const WithCache< const Value * > &LHS, const WithCache< const Value * > &RHS, const Instruction *CtxI) const
void addToWorklist(Instruction *I)
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
OverflowResult computeOverflowForUnsignedMul(const Value *LHS, const Value *RHS, const Instruction *CtxI, bool IsNSW=false) const
OverflowResult computeOverflowForSignedSub(const Value *LHS, const Value *RHS, const Instruction *CtxI) const
DominatorTree & DT
OverflowResult computeOverflowForUnsignedAdd(const WithCache< const Value * > &LHS, const WithCache< const Value * > &RHS, const Instruction *CtxI) const
void computeKnownBits(const Value *V, KnownBits &Known, const Instruction *CtxI, unsigned Depth=0) const
Value * getFreelyInverted(Value *V, bool WillInvertAllUses, BuilderTy *Builder, bool &DoesConsume)
const SimplifyQuery & getSimplifyQuery() const
LLVM_ABI bool hasNoNaNs() const LLVM_READONLY
Determine whether the no-NaNs flag is set.
LLVM_ABI bool hasNoUnsignedWrap() const LLVM_READONLY
Determine whether the no unsigned wrap flag is set.
LLVM_ABI bool hasNoInfs() const LLVM_READONLY
Determine whether the no-infs flag is set.
bool isArithmeticShift() const
Return true if this is an arithmetic shift right.
LLVM_ABI bool hasNoSignedWrap() const LLVM_READONLY
Determine whether the no signed wrap flag is set.
LLVM_ABI bool isCommutative() const LLVM_READONLY
Return true if the instruction is commutative:
LLVM_ABI bool isExact() const LLVM_READONLY
Determine whether the exact flag is set.
iterator_range< user_iterator > users()
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
bool isShift() const
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
An instruction for reading from memory.
Value * getPointerOperand()
bool isVolatile() const
Return true if this is a load from a volatile memory location.
This class represents min/max intrinsics.
Value * getLHS() const
Value * getRHS() const
static bool isMin(Intrinsic::ID ID)
Whether the intrinsic is a smin or umin.
static bool isSigned(Intrinsic::ID ID)
Whether the intrinsic is signed or unsigned.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Represents a saturating add/sub intrinsic.
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, const Instruction *MDFrom=nullptr)
A vector that has set insertion semantics.
Definition SetVector.h:57
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:103
bool contains(const_arg_type key) const
Check if the SetVector contains the given key.
Definition SetVector.h:258
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:157
This instruction constructs a fixed permutation of two input vectors.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class represents a truncation of integer types.
bool hasNoSignedWrap() const
Test whether this operation is known to never undergo signed overflow, aka the nsw property.
bool hasNoUnsignedWrap() const
Test whether this operation is known to never undergo unsigned overflow, aka the nuw property.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition Type.h:258
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
bool isPPC_FP128Ty() const
Return true if this is powerpc long double.
Definition Type.h:167
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
LLVM_ABI Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:291
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
LLVM_ABI int getFPMantissaWidth() const
Return the width of the mantissa of this type.
Definition Type.cpp:222
LLVM_ABI const fltSemantics & getFltSemantics() const
Definition Type.cpp:96
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
void setOperand(unsigned i, Value *Val)
Definition User.h:212
Value * getOperand(unsigned i) const
Definition User.h:207
unsigned getNumOperands() const
Definition User.h:229
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
iterator_range< user_iterator > users()
Definition Value.h:428
LLVM_ABI bool hasNUsesOrMore(unsigned N) const
Return true if this value has N uses or more.
Definition Value.cpp:150
LLVM_ABI const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr, bool LookThroughIntToPtr=false) const
Accumulate the constant offset this value has compared to a base pointer.
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:707
iterator_range< use_iterator > uses()
Definition Value.h:382
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:395
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
const ParentTy * getParent() const
Definition ilist_node.h:34
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_ABI APInt RoundingUDiv(const APInt &A, const APInt &B, APInt::Rounding RM)
Return A unsign-divided by B, rounded by the given rounding mode.
Definition APInt.cpp:2810
LLVM_ABI APInt RoundingSDiv(const APInt &A, const APInt &B, APInt::Rounding RM)
Return A sign-divided by B, rounded by the given rounding mode.
Definition APInt.cpp:2828
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
auto m_PosZeroFP()
Matches a floating-point positive zero.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
AllOnesConstantMatch m_AllOnes()
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
match_unless< Pattern > m_Unless(const Pattern &P)
Match if the inner matcher does NOT match.
match_combine_or< Ty... > m_CombineOr(const Ty &...Ps)
Combine pattern matchers matching any of Ps patterns.
match_combine_and< Ty... > m_CombineAnd(const Ty &...Ps)
Combine pattern matchers matching all of Ps patterns.
cst_pred_ty< is_lowbit_mask > m_LowBitMask()
Match an integer or vector with only the low bit(s) set.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
cst_pred_ty< is_negative > m_Negative()
Match an integer or vector of negative values.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
cst_pred_ty< is_sign_mask > m_SignMask()
Match an integer or vector with only the sign bit(s) set.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::FSub > m_FSub(const LHS &L, const RHS &R)
cst_pred_ty< is_power2 > m_Power2()
Match an integer or vector power-of-2.
BinaryOp_match< LHS, RHS, Instruction::URem > m_URem(const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, TruncInst >, OpTy > m_TruncOrSelf(const OpTy &Op)
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)
Matches Trunc.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
auto m_Sqrt(const Opnd0 &Op0)
ap_match< APInt > m_APIntAllowPoison(const APInt *&Res)
Match APInt while allowing poison in splat vector constants.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
match_combine_or< CastInst_match< OpTy, ZExtInst >, OpTy > m_ZExtOrSelf(const OpTy &Op)
bool match(Val *V, const Pattern &P)
BinOpPred_match< LHS, RHS, is_idiv_op > m_IDiv(const LHS &L, const RHS &R)
Matches integer division operations.
match_bind< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
match_deferred< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
cstfp_pred_ty< is_any_zero_fp > m_AnyZeroFP()
Match a floating-point negative zero or positive zero.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
BinOpPred_match< LHS, RHS, is_right_shift_op > m_Shr(const LHS &L, const RHS &R)
Matches logical shift operations.
specific_intval< true > m_SpecificIntAllowPoison(const APInt &V)
ap_match< APFloat > m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
CmpClass_match< LHS, RHS, ICmpInst, true > m_c_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
Matches an ICmp with a predicate over LHS and RHS in either order.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap, true > m_c_NUWAdd(const LHS &L, const RHS &R)
OverflowingBinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub, OverflowingBinaryOperator::NoSignedWrap > m_NSWNeg(const ValTy &V)
Matches a 'Neg' as 'sub nsw 0, V'.
cst_pred_ty< is_nonnegative > m_NonNegative()
Match an integer or vector of non-negative values.
auto m_SMax(const Opnd0 &Op0, const Opnd1 &Op1)
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
auto m_BinOp()
Match an arbitrary binary operation and ignore it.
auto m_UMax(const Opnd0 &Op0, const Opnd1 &Op1)
ExtractValue_match< Ind, Val_t > m_ExtractValue(const Val_t &V)
Match a single index ExtractValue instruction.
BinOpPred_match< LHS, RHS, is_logical_shift_op > m_LogicalShift(const LHS &L, const RHS &R)
Matches logical shift operations.
match_combine_or< CastInst_match< OpTy, UIToFPInst >, CastInst_match< OpTy, SIToFPInst > > m_IToFP(const OpTy &Op)
auto m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Xor, true > m_c_Xor(const LHS &L, const RHS &R)
Matches an Xor with LHS and RHS in either order.
BinaryOp_match< LHS, RHS, Instruction::FAdd > m_FAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
auto m_Constant()
Match an arbitrary Constant and ignore it.
NoWrapTrunc_match< OpTy, TruncInst::NoSignedWrap > m_NSWTrunc(const OpTy &Op)
Matches trunc nsw.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
cst_pred_ty< is_non_zero_int > m_NonZeroInt()
Match a non-zero integer or a vector with all non-zero elements.
ThreeOps_match< decltype(m_Value()), LHS, RHS, Instruction::Select, true > m_c_Select(const LHS &L, const RHS &R)
Match Select(C, LHS, RHS) or Select(C, RHS, LHS)
CastInst_match< OpTy, FPExtInst > m_FPExt(const OpTy &Op)
OverflowingBinaryOp_match< LHS, RHS, Instruction::Shl, OverflowingBinaryOperator::NoSignedWrap > m_NSWShl(const LHS &L, const RHS &R)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Shl, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWShl(const LHS &L, const RHS &R)
OverflowingBinaryOp_match< LHS, RHS, Instruction::Mul, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWMul(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::UDiv > m_UDiv(const LHS &L, const RHS &R)
is_zero< true > m_Zero()
Match any null constant or a vector with all elements equal to 0.
match_immconstant_ty m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
cst_pred_ty< is_negated_power2_or_zero > m_NegatedPower2OrZero()
Match a integer or vector negated power-of-2.
NoWrapTrunc_match< OpTy, TruncInst::NoUnsignedWrap > m_NUWTrunc(const OpTy &Op)
Matches trunc nuw.
cst_pred_ty< custom_checkfn< APInt > > m_CheckedInt(function_ref< bool(const APInt &)> CheckFn)
Match an integer or vector where CheckFn(ele) for each element is true.
SelectLike_match< CondTy, LTy, RTy > m_SelectLike(const CondTy &C, const LTy &TrueC, const RTy &FalseC)
Matches a value that behaves like a boolean-controlled select, i.e.
cst_pred_ty< is_lowbit_mask_or_zero > m_LowBitMaskOrZero()
Match an integer or vector with only the low bit(s) set.
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
match_combine_or< BinaryOp_match< LHS, RHS, Instruction::Add >, DisjointOr_match< LHS, RHS > > m_AddLike(const LHS &L, const RHS &R)
Match either "add" or "or disjoint".
CastInst_match< OpTy, UIToFPInst > m_UIToFP(const OpTy &Op)
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
cstfp_pred_ty< is_finitenonzero > m_FiniteNonZero()
Match a finite non-zero FP constant.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
BinaryOp_match< LHS, RHS, Instruction::SDiv > m_SDiv(const LHS &L, const RHS &R)
auto m_SMin(const Opnd0 &Op0, const Opnd1 &Op1)
FNeg_match< OpTy, true > m_FNegOrFSubNegZero(const OpTy &X)
Match fneg X or fsub -0.0, X.
auto m_FAbs(const Opnd0 &Op0)
Signum_match< Val_t > m_Signum(const Val_t &V)
Matches a signum pattern.
CastInst_match< OpTy, SIToFPInst > m_SIToFP(const OpTy &Op)
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
UAddWithOverflow_match< LHS_t, RHS_t, Sum_t > m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S)
Match an icmp instruction checking for unsigned overflow on addition.
BinOpPred_match< LHS, RHS, is_irem_op > m_IRem(const LHS &L, const RHS &R)
Matches integer remainder operations.
auto m_MaxOrMin(const Opnd0 &Op0, const Opnd1 &Op1)
CastInst_match< OpTy, FPTruncInst > m_FPTrunc(const OpTy &Op)
auto m_Undef()
Match an arbitrary undef constant.
auto m_VecReverse(const Opnd0 &Op0)
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
BinaryOp_match< LHS, RHS, Instruction::Or, true > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
ElementWiseBitCast_match< OpTy > m_ElementWiseBitCast(const OpTy &Op)
BinaryOp_match< LHS, RHS, Instruction::Mul, true > m_c_Mul(const LHS &L, const RHS &R)
Matches a Mul with LHS and RHS in either order.
CastOperator_match< OpTy, Instruction::PtrToInt > m_PtrToInt(const OpTy &Op)
Matches PtrToInt.
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
cst_pred_ty< icmp_pred_with_threshold > m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold)
Match an integer or vector with every element comparing 'pred' (eg/ne/...) to Threshold.
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition STLExtras.h:846
@ NeverOverflows
Never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
LLVM_ABI bool isKnownNeverInfinity(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Return true if the floating-point scalar value is not an infinity or if the floating-point vector val...
LLVM_ABI bool isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS, bool &TrueIfSigned)
Given an exploded icmp instruction, return true if the comparison only checks the sign bit.
@ Known
Known to have no common set bits.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ BinaryOp
One of the operands is a binary op.
LLVM_ABI Value * stripNullTest(Value *V)
Returns the inner value X if the expression has the form f(X) where f(X) == 0 if and only if X == 0,...
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
LLVM_ABI Value * simplifyFCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q)
Given operands for an FCmpInst, fold the result or return null.
int ilogb(const APFloat &Arg)
Returns the exponent of the internal representation of the APFloat.
Definition APFloat.h:1707
LLVM_ABI bool isKnownNegative(const WithCache< const Value * > &V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the given value is known be negative (i.e.
LLVM_ABI bool isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL, bool OrZero=false, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Return true if the given value is known to have exactly one bit set when defined.
LLVM_ABI bool MaskedValueIsZero(const Value *V, const APInt &Mask, const SimplifyQuery &SQ, unsigned Depth=0)
Return true if 'V & Mask' is known to be zero.
LLVM_ABI Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Function *CtxF=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
LLVM_ABI Value * simplifyAddInst(Value *LHS, Value *RHS, bool IsNSW, bool IsNUW, const SimplifyQuery &Q)
Given operands for an Add, fold the result or return null.
LLVM_ABI Constant * ConstantFoldConstant(const Constant *C, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldConstant - Fold the constant using the specified DataLayout.
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
LLVM_ABI bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition bit.h:263
LLVM_ABI Value * emitGEPOffset(IRBuilderBase *Builder, const DataLayout &DL, User *GEP, bool NoAssumptions=false)
Given a getelementptr instruction/constantexpr, emit the code necessary to compute the offset from th...
Definition Local.cpp:22
constexpr unsigned MaxAnalysisRecursionDepth
LLVM_ABI Constant * ConstantFoldUnaryOpOperand(unsigned Opcode, Constant *Op, const DataLayout &DL)
Attempt to constant fold a unary operation with the specified operand.
SelectPatternFlavor
Specific patterns of select instructions we can match.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UNKNOWN
@ SPF_FMINNUM
Unsigned maximum.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
LLVM_ABI bool isKnownPositive(const WithCache< const Value * > &V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the given value is known be positive (i.e.
LLVM_ABI bool impliesPoison(const Value *ValAssumedPoison, const Value *V)
Return true if V is poison given that ValAssumedPoison is already poison.
LLVM_ABI LinearExpression decomposeLinearExpression(const DataLayout &DL, Value *Ptr)
Decompose a pointer into a linear expression.
Definition Loads.cpp:978
LLVM_ABI bool isFinite(const Loop *L)
Return true if this loop can be assumed to run for a finite number of iterations.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
Returns: X * 2^Exp for integral exponents.
Definition APFloat.h:1716
LLVM_ABI bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
LLVM_ABI Value * simplifyICmpInst(CmpPredicate Pred, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for an ICmpInst, fold the result or return null.
LLVM_ABI Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
LLVM_ABI Constant * ConstantFoldLoadFromConst(Constant *C, Type *Ty, const APInt &Offset, const DataLayout &DL)
Extract value of C at the given Offset reinterpreted as Ty.
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth, bool MustPreserveProvenance=false)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI Constant * ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL)
Attempt to constant fold a binary operation with the specified operands.
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:389
@ Other
Any other memory.
Definition ModRef.h:68
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:74
LLVM_ABI Value * simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a BinaryOperator, fold the result or return null.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ SMin
Signed integer min implemented in terms of select(cmp()).
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
@ UMax
Unsigned integer max implemented in terms of select(cmp()).
IntPtrTy
Definition InstrProf.h:82
LLVM_ABI bool isKnownNonEqual(const Value *V1, const Value *V2, const SimplifyQuery &SQ, unsigned Depth=0)
Return true if the given values are known to be non-equal when defined.
DWARFExpression::Operation Op
LLVM_ABI bool PointerMayBeCaptured(const Value *V, bool ReturnCaptures, unsigned MaxUsesToExplore=0)
PointerMayBeCaptured - Return true if this pointer value may be captured by the enclosing function (w...
constexpr unsigned BitWidth
LLVM_ABI Constant * getLosslessInvCast(Constant *C, Type *InvCastTo, unsigned CastOp, const DataLayout &DL, PreservedCastFlags *Flags=nullptr)
Try to cast C to InvC losslessly, satisfying CastOp(InvC) equals C, or CastOp(InvC) is a refined valu...
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition STLExtras.h:2035
LLVM_ABI bool isKnownNeverInfOrNaN(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Return true if the floating-point value can never contain a NaN or infinity.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI bool isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...
LLVM_ABI std::optional< std::pair< CmpPredicate, Constant * > > getFlippedStrictnessPredicateAndConstant(CmpPredicate Pred, Constant *C)
Convert an integer comparison with a constant RHS into an equivalent form with the strictness flipped...
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition STLExtras.h:2182
@ Continue
Definition DWP.h:26
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
LLVM_ABI bool isKnownNonNegative(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the give value is known to be non-negative.
LLVM_ABI SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
constexpr detail::IsaCheckPredicate< Types... > IsaPred
Function object wrapper for the llvm::isa type check.
Definition Casting.h:866
LLVM_ABI std::optional< bool > isImpliedCondition(const Value *LHS, const Value *RHS, const DataLayout &DL, bool LHSIsTrue=true, unsigned Depth=0)
Return true if RHS is known to be implied true by LHS.
LLVM_ABI std::optional< DecomposedBitTest > decomposeBitTestICmp(Value *LHS, Value *RHS, CmpInst::Predicate Pred, bool LookThroughTrunc=true, bool AllowNonZeroC=false, bool DecomposeAnd=false)
Decompose an icmp into the form ((X & Mask) pred C) if possible.
LLVM_ABI ConstantRange computeConstantRange(const Value *V, bool ForSigned, const SimplifyQuery &SQ, unsigned Depth=0)
Determine the possible constant range of an integer or vector of integer value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define NC
Definition regutils.h:42
Value * materialize(InstCombiner::BuilderTy &Builder) const
static OffsetResult select(Value *Cond, Value *TrueV, Value *FalseV, Instruction *MDFrom)
static OffsetResult value(Value *V)
static OffsetResult invalid()
This callback is used in conjunction with PointerMayBeCaptured.
static CommonPointerBase compute(Value *LHS, Value *RHS)
Represent subnormal handling kind for floating point instruction inputs and outputs.
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ PositiveZero
Denormals are flushed to positive zero.
static constexpr DenormalMode getIEEE()
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition KnownBits.h:106
bool isZero() const
Returns true if value is all zero.
Definition KnownBits.h:78
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition KnownBits.h:256
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
Definition KnownBits.h:288
APInt getSignedMaxValue() const
Return the maximal signed value possible given these KnownBits.
Definition KnownBits.h:152
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition KnownBits.h:303
bool isConstant() const
Returns true if we know the value of all bits.
Definition KnownBits.h:54
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:262
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition KnownBits.h:146
APInt getMinValue() const
Return the minimal unsigned value possible given these KnownBits.
Definition KnownBits.h:130
bool isNegative() const
Returns true if this value is known to be negative.
Definition KnownBits.h:103
unsigned countMinPopulation() const
Returns the number of bits known to be one.
Definition KnownBits.h:300
APInt getSignedMinValue() const
Return the minimal signed value possible given these KnownBits.
Definition KnownBits.h:136
const APInt & getConstant() const
Returns the value when all bits have a known value.
Definition KnownBits.h:58
Linear expression BasePtr + Index * Scale + Offset.
Definition Loads.h:226
GEPNoWrapFlags Flags
Definition Loads.h:231
Matching combinators.
SelectPatternFlavor Flavor
static bool isMinOrMax(SelectPatternFlavor SPF)
When implementing this min/max pattern as fcmp; select, does the fcmp have to be ordered?
const DataLayout & DL
const DominatorTree * DT
SimplifyQuery getWithInstruction(const Instruction *I) const
AssumptionCache * AC
const Instruction * CtxI
A MapVector that performs no allocations if smaller than a certain size.
Definition MapVector.h:344
Capture information for a specific Use.